From 6620b710cd19fd19a19a5c67c70ba2912efcce7f Mon Sep 17 00:00:00 2001 From: Evan Erwee Date: Mon, 6 Jul 2026 19:05:03 -0400 Subject: [PATCH 1/2] =?UTF-8?q?feat:=20generic=20LlamaIndex=20reader=20plu?= =?UTF-8?q?gin=20=E2=80=94=20wrap=20any=20reader=20via=20config?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds LlamaIndexPluginReaderProvider that dynamically loads any LlamaIndex reader package (Confluence, Notion, Slack, Jira, etc.) with just configuration — no bespoke provider code needed per reader. Features: - Dynamic import of any llama-index-readers-* package - Configurable timeout (prevents hangs on unresponsive APIs) - Retry with exponential backoff on transient errors (429, 503) - Auth failure detection (401/403 — never retried) - Graceful degradation (fail_on_error=False returns [] instead of raising) - input_source flexibility (auto-fallback if reader doesn't accept it) - Generator/iterator result handling - Document validation and metadata enrichment - 30 unit tests covering all error paths Usage: config = LlamaIndexPluginReaderConfig( package='llama-index-readers-confluence', module_path='llama_index.readers.confluence', reader_class='ConfluenceReader', init_args={'base_url': 'https://...', 'token': '...'}, load_args={'space_key': 'ENG'}, timeout_seconds=60, max_retries=2, ) provider = LlamaIndexPluginReaderProvider(config) docs = provider.read() --- .../load/readers/providers/__init__.py | 2 + .../llama_index_plugin_reader_provider.py | 454 +++++++++++++ .../load/readers/reader_provider_config.py | 34 + ...test_llama_index_plugin_reader_provider.py | 596 ++++++++++++++++++ 4 files changed, 1086 insertions(+) create mode 100644 lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/load/readers/providers/llama_index_plugin_reader_provider.py create mode 100644 lexical-graph/tests/unit/indexing/load/readers/providers/test_llama_index_plugin_reader_provider.py diff --git a/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/load/readers/providers/__init__.py b/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/load/readers/providers/__init__.py index 1e308bfbc..9a10582b6 100644 --- a/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/load/readers/providers/__init__.py +++ b/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/load/readers/providers/__init__.py @@ -33,6 +33,8 @@ "S3DirectoryReaderProvider": ".s3_directory_reader_provider", # Database readers "DatabaseReaderProvider": ".database_reader_provider", + # Generic LlamaIndex plugin reader + "LlamaIndexPluginReaderProvider": ".llama_index_plugin_reader_provider", } def __getattr__(name): diff --git a/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/load/readers/providers/llama_index_plugin_reader_provider.py b/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/load/readers/providers/llama_index_plugin_reader_provider.py new file mode 100644 index 000000000..d28bfb434 --- /dev/null +++ b/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/load/readers/providers/llama_index_plugin_reader_provider.py @@ -0,0 +1,454 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Generic LlamaIndex reader plugin — wraps any reader from the LlamaIndex ecosystem. + +Usage: + from graphrag_toolkit.lexical_graph.indexing.load.readers.reader_provider_config import ( + LlamaIndexPluginReaderConfig, + ) + from graphrag_toolkit.lexical_graph.indexing.load.readers.providers import ( + LlamaIndexPluginReaderProvider, + ) + + config = LlamaIndexPluginReaderConfig( + package="llama-index-readers-confluence", + module_path="llama_index.readers.confluence", + reader_class="ConfluenceReader", + init_args={"base_url": "https://mycompany.atlassian.net/wiki", "token": "..."}, + load_args={"space_key": "ENG"}, + timeout_seconds=60, + max_retries=2, + ) + provider = LlamaIndexPluginReaderProvider(config) + docs = provider.read(None) +""" + +import importlib +import signal +import time +from typing import Any, List, Optional +from functools import wraps + +from graphrag_toolkit.lexical_graph.indexing.load.readers.reader_provider_config import ( + LlamaIndexPluginReaderConfig, +) +from graphrag_toolkit.lexical_graph.logging import logging +from llama_index.core.schema import Document + +logger = logging.getLogger(__name__) + + +class ReaderImportError(ImportError): + """Raised when the required LlamaIndex reader package is not installed.""" + pass + + +class ReaderAuthError(RuntimeError): + """Raised when authentication fails (expired token, invalid credentials).""" + pass + + +class ReaderTimeoutError(TimeoutError): + """Raised when the reader exceeds the configured timeout.""" + pass + + +class LlamaIndexPluginReaderProvider: + """Generic reader provider that wraps any LlamaIndex reader via configuration. + + Features: + - Dynamic import of any LlamaIndex reader package + - Configurable timeout (prevents hangs) + - Retry with exponential backoff on transient failures + - Graceful degradation (fail_on_error=False returns [] instead of raising) + - Structured logging at every decision point + - Auth failure detection (recognizes 401/403 patterns) + - Empty result warnings + - Metadata enrichment support + """ + + # Known auth-related error patterns (case-insensitive matching) + _AUTH_ERROR_PATTERNS = ( + "401", "403", "unauthorized", "forbidden", + "invalid token", "expired token", "authentication failed", + "access denied", "invalid credentials", "token expired", + ) + + # Known transient error patterns (retry-worthy) + _TRANSIENT_ERROR_PATTERNS = ( + "429", "500", "502", "503", "504", + "rate limit", "too many requests", "timeout", + "connection reset", "connection refused", + "temporary failure", "service unavailable", + ) + + def __init__(self, config: LlamaIndexPluginReaderConfig): + """Initialize by dynamically importing and instantiating the reader. + + Args: + config: Plugin configuration specifying which reader to load and how. + + Raises: + ReaderImportError: If the required package is not installed. + ValueError: If required config fields are missing. + """ + self._config = config + self._reader = None + self._validate_config() + self._import_and_init_reader() + + def _validate_config(self) -> None: + """Validate that required configuration fields are present.""" + if not self._config.reader_class: + raise ValueError( + "LlamaIndexPluginReaderConfig.reader_class is required " + "(e.g. 'ConfluenceReader')" + ) + if not self._config.module_path and not self._config.package: + raise ValueError( + "LlamaIndexPluginReaderConfig requires either module_path " + "(e.g. 'llama_index.readers.confluence') or package " + "(e.g. 'llama-index-readers-confluence')" + ) + + def _resolve_module_path(self) -> str: + """Resolve the Python module path from config. + + Priority: module_path > derived from package name. + """ + if self._config.module_path: + return self._config.module_path + # Derive from package: "llama-index-readers-confluence" → "llama_index.readers.confluence" + parts = self._config.package.replace("-", "_").split("_") + # Handle "llama_index_readers_X" pattern + if len(parts) >= 4 and parts[0] == "llama" and parts[1] == "index": + return f"llama_index.readers.{'.'.join(parts[3:])}" + # Fallback: just replace dashes + return self._config.package.replace("-", "_") + + def _import_and_init_reader(self) -> None: + """Dynamically import the reader module and instantiate the reader class.""" + module_path = self._resolve_module_path() + class_name = self._config.reader_class + + logger.info( + f"LlamaIndexPlugin: importing {class_name} from {module_path} " + f"(package: {self._config.package or 'not specified'})" + ) + + try: + module = importlib.import_module(module_path) + except ImportError as e: + install_hint = self._config.package or module_path + msg = ( + f"Failed to import '{module_path}'. " + f"Install the reader package: pip install {install_hint}" + ) + logger.error(msg) + raise ReaderImportError(msg) from e + + try: + reader_cls = getattr(module, class_name) + except AttributeError as e: + available = [a for a in dir(module) if not a.startswith("_")] + msg = ( + f"Class '{class_name}' not found in '{module_path}'. " + f"Available: {available[:10]}" + ) + logger.error(msg) + raise ReaderImportError(msg) from e + + # Instantiate the reader with init_args + init_args = self._config.init_args or {} + try: + self._reader = reader_cls(**init_args) + logger.info( + f"LlamaIndexPlugin: {class_name} initialized successfully " + f"(init_args keys: {list(init_args.keys())})" + ) + except TypeError as e: + msg = ( + f"Failed to instantiate {class_name} with args {list(init_args.keys())}. " + f"Check init_args match the constructor signature. Error: {e}" + ) + logger.error(msg) + raise ValueError(msg) from e + except Exception as e: + if self._is_auth_error(e): + msg = ( + f"Authentication failed when initializing {class_name}. " + f"Check credentials in init_args. Error: {e}" + ) + logger.error(msg) + raise ReaderAuthError(msg) from e + raise + + def _is_auth_error(self, error: Exception) -> bool: + """Detect if an exception is an authentication/authorization failure.""" + error_str = str(error).lower() + return any(pattern in error_str for pattern in self._AUTH_ERROR_PATTERNS) + + def _is_transient_error(self, error: Exception) -> bool: + """Detect if an exception is a transient/retryable failure.""" + error_str = str(error).lower() + return any(pattern in error_str for pattern in self._TRANSIENT_ERROR_PATTERNS) + + def read(self, input_source: Any = None) -> List[Document]: + """Read documents using the configured LlamaIndex reader. + + Applies timeout, retry with backoff, and error classification. + + Args: + input_source: Optional input passed to load_args (reader-specific). + Can be a space_key, URL, file path, etc. + + Returns: + List of Document objects. Empty list if fail_on_error=False and read fails. + + Raises: + ReaderAuthError: On authentication failures (not retried). + ReaderTimeoutError: When timeout_seconds is exceeded. + RuntimeError: On non-transient failures when fail_on_error=True. + """ + load_fn = self._resolve_load_function() + if load_fn is None: + return [] + + load_args = self._build_load_args(input_source) + return self._execute_with_retries(load_fn, load_args) + + def _resolve_load_function(self) -> Optional[Any]: + """Resolve the callable load method from the reader instance. + + Returns: + The load function, or None if resolution fails (with appropriate error/log). + """ + if self._reader is None: + logger.error("LlamaIndexPlugin: reader not initialized — cannot read") + if self._config.fail_on_error: + raise RuntimeError("Reader not initialized") + return None + + load_method_name = self._config.load_method or "load_data" + load_fn = getattr(self._reader, load_method_name, None) + + if load_fn is None: + msg = ( + f"Reader {self._config.reader_class} has no method '{load_method_name}'. " + f"Available methods: {[m for m in dir(self._reader) if not m.startswith('_')][:15]}" + ) + logger.error(msg) + if self._config.fail_on_error: + raise AttributeError(msg) + return None + + return load_fn + + def _build_load_args(self, input_source: Any = None) -> dict: + """Assemble the keyword arguments for the load function. + + Merges configured load_args with an optional input_source. + """ + load_args = dict(self._config.load_args or {}) + if input_source is not None: + load_args["input_source"] = input_source + return load_args + + def _execute_with_retries(self, load_fn: Any, load_args: dict) -> List[Document]: + """Execute load_fn with retry loop, timeout, and error classification. + + Args: + load_fn: The reader's load method. + load_args: Keyword arguments for the load method. + + Returns: + List of Documents on success, or [] on graceful failure. + """ + max_attempts = 1 + (self._config.max_retries or 0) + last_error: Optional[Exception] = None + + for attempt in range(1, max_attempts + 1): + try: + documents = self._execute_with_timeout(load_fn, load_args) + return self._post_process(documents) + except ReaderAuthError: + raise + except (ReaderTimeoutError, Exception) as e: + last_error = e + if not self._handle_attempt_error( + last_error, attempt, max_attempts + ): + break + + return self._handle_exhausted_retries(max_attempts, last_error) + + def _handle_attempt_error(self, error: Exception, attempt: int, max_attempts: int) -> bool: + """Classify and handle an error from a single attempt. + + Args: + error: The exception that occurred. + attempt: Current attempt number (1-based). + max_attempts: Total allowed attempts. + + Returns: + True if should continue retrying, False if should stop. + """ + if isinstance(error, ReaderTimeoutError): + logger.warning( + f"LlamaIndexPlugin: timeout on attempt {attempt}/{max_attempts} " + f"({self._config.timeout_seconds}s exceeded)" + ) + if attempt < max_attempts: + self._backoff(attempt) + return True + return False + + if self._is_auth_error(error): + msg = ( + f"LlamaIndexPlugin: authentication failed — " + f"{self._config.reader_class}: {error}" + ) + logger.error(msg) + raise ReaderAuthError(msg) from error + + if self._is_transient_error(error) and attempt < max_attempts: + logger.warning( + f"LlamaIndexPlugin: transient error on attempt " + f"{attempt}/{max_attempts}: {error}" + ) + self._backoff(attempt) + return True + + # Non-transient, non-auth error — stop retrying + load_method_name = self._config.load_method or "load_data" + logger.error( + f"LlamaIndexPlugin: {self._config.reader_class}.{load_method_name}() " + f"failed on attempt {attempt}: {error}", + exc_info=True, + ) + return False + + def _handle_exhausted_retries( + self, max_attempts: int, last_error: Optional[Exception] + ) -> List[Document]: + """Handle the case where all retry attempts have been exhausted. + + Returns: + Empty list if fail_on_error=False. + + Raises: + RuntimeError: If fail_on_error=True. + """ + if self._config.fail_on_error: + raise RuntimeError( + f"LlamaIndexPlugin: all {max_attempts} attempt(s) failed for " + f"{self._config.reader_class}. Last error: {last_error}" + ) from last_error + + logger.warning( + f"LlamaIndexPlugin: returning empty result after {max_attempts} failed " + f"attempt(s). Last error: {last_error}" + ) + return [] + + def _execute_with_timeout( + self, load_fn: Any, load_args: dict + ) -> List[Document]: + """Execute the load function with a timeout guard. + + Uses threading-based timeout (signal.alarm not safe in all contexts). + """ + import concurrent.futures + + timeout = self._config.timeout_seconds or 120 + + # Remove input_source from load_args if the reader doesn't accept it + # Try with input_source first, fall back without it + args_to_try = load_args.copy() + + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit(self._call_load_fn, load_fn, args_to_try) + try: + return future.result(timeout=timeout) + except concurrent.futures.TimeoutError as e: + future.cancel() + raise ReaderTimeoutError( + f"Reader {self._config.reader_class} exceeded " + f"timeout of {timeout}s" + ) from e + + def _call_load_fn(self, load_fn: Any, load_args: dict) -> List[Document]: + """Call the load function, handling input_source parameter flexibility.""" + try: + return load_fn(**load_args) + except TypeError as e: + # If "input_source" isn't a valid param, try without it + if "input_source" in load_args and "unexpected keyword argument" in str(e): + logger.debug( + "LlamaIndexPlugin: reader doesn't accept 'input_source', " + "retrying without it" + ) + args_without_source = { + k: v for k, v in load_args.items() if k != "input_source" + } + return load_fn(**args_without_source) + raise + + def _post_process(self, documents: Any) -> List[Document]: + """Validate and enrich returned documents.""" + # Handle generators/iterators + if hasattr(documents, '__iter__') and not isinstance(documents, list): + documents = list(documents) + + if not isinstance(documents, list): + logger.warning( + f"LlamaIndexPlugin: reader returned {type(documents).__name__} " + f"instead of list — wrapping" + ) + documents = [documents] if documents else [] + + # Filter out non-Document items + valid_docs = [] + for i, doc in enumerate(documents): + if isinstance(doc, Document): + valid_docs.append(doc) + elif hasattr(doc, 'text') and hasattr(doc, 'metadata'): + # Duck-typing: looks like a Document + valid_docs.append(doc) + else: + logger.debug( + f"LlamaIndexPlugin: skipping non-Document item at index {i}: " + f"{type(doc).__name__}" + ) + + # Apply metadata enrichment + if self._config.metadata_fn and valid_docs: + source_id = self._config.package or self._config.reader_class + try: + extra_metadata = self._config.metadata_fn(source_id) + if extra_metadata and isinstance(extra_metadata, dict): + for doc in valid_docs: + doc.metadata.update(extra_metadata) + except Exception as e: + logger.warning(f"LlamaIndexPlugin: metadata_fn failed: {e}") + + # Log results + if not valid_docs: + logger.warning( + f"LlamaIndexPlugin: {self._config.reader_class} returned 0 documents" + ) + else: + logger.info( + f"LlamaIndexPlugin: {self._config.reader_class} returned " + f"{len(valid_docs)} document(s)" + ) + + return valid_docs + + def _backoff(self, attempt: int) -> None: + """Exponential backoff between retry attempts.""" + base = self._config.retry_backoff_seconds or 2.0 + delay = base * (2 ** (attempt - 1)) + logger.info(f"LlamaIndexPlugin: backing off {delay:.1f}s before retry") + time.sleep(delay) diff --git a/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/load/readers/reader_provider_config.py b/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/load/readers/reader_provider_config.py index 27b144fe2..fa5069972 100644 --- a/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/load/readers/reader_provider_config.py +++ b/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/load/readers/reader_provider_config.py @@ -189,3 +189,37 @@ class UniversalDirectoryReaderConfig(ReaderProviderConfig): bucket_name: Optional[str] = None key_prefix: Optional[str] = None collection_id: Optional[str] = None + + +# Generic LlamaIndex plugin reader +@dataclass +class LlamaIndexPluginReaderConfig(ReaderProviderConfig): + """Configuration for the generic LlamaIndex reader plugin. + + Wraps any reader from the LlamaIndex ecosystem with just configuration. + See: https://developers.llamaindex.ai/python/framework-api-reference/readers/ + + Args: + package: pip package name (e.g. "llama-index-readers-confluence") + module_path: Full Python module path (e.g. "llama_index.readers.confluence") + reader_class: Class name to import (e.g. "ConfluenceReader") + init_args: Dict of keyword arguments for the reader constructor + load_method: Method to call for loading (default: "load_data") + load_args: Dict of keyword arguments for the load method + timeout_seconds: Max seconds for the read operation (default: 120) + max_retries: Number of retries on transient failures (default: 2) + retry_backoff_seconds: Initial backoff between retries (default: 2.0) + fail_on_error: If False, return [] on failure instead of raising (default: False) + metadata_fn: Optional function to enrich document metadata + """ + package: str = "" + module_path: str = "" + reader_class: str = "" + init_args: Optional[Dict[str, Any]] = None + load_method: str = "load_data" + load_args: Optional[Dict[str, Any]] = None + timeout_seconds: int = 120 + max_retries: int = 2 + retry_backoff_seconds: float = 2.0 + fail_on_error: bool = False + metadata_fn: Optional[Callable[[str], Dict[str, Any]]] = None diff --git a/lexical-graph/tests/unit/indexing/load/readers/providers/test_llama_index_plugin_reader_provider.py b/lexical-graph/tests/unit/indexing/load/readers/providers/test_llama_index_plugin_reader_provider.py new file mode 100644 index 000000000..d92076520 --- /dev/null +++ b/lexical-graph/tests/unit/indexing/load/readers/providers/test_llama_index_plugin_reader_provider.py @@ -0,0 +1,596 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Comprehensive unit tests for LlamaIndexPluginReaderProvider. + +Tests cover: + - Happy path (reader loads documents successfully) + - Import errors (package not installed) + - Class not found in module + - Invalid init_args (constructor mismatch) + - Auth failures (401/403 detection) + - Timeout enforcement + - Retry with exponential backoff on transient errors + - Graceful degradation (fail_on_error=False) + - Empty results (warning logged) + - Partial failures (non-Document items filtered) + - input_source flexibility (passed or omitted) + - Metadata enrichment + - Load method not found + - Generator/iterator results +""" + +import logging +import time +import pytest +from unittest.mock import Mock, MagicMock, patch, PropertyMock +from dataclasses import dataclass + +from llama_index.core.schema import Document + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +def _make_config(**overrides): + """Create a LlamaIndexPluginReaderConfig with sensible test defaults.""" + from graphrag_toolkit.lexical_graph.indexing.load.readers.reader_provider_config import ( + LlamaIndexPluginReaderConfig, + ) + defaults = { + "package": "llama-index-readers-confluence", + "module_path": "llama_index.readers.confluence", + "reader_class": "ConfluenceReader", + "init_args": {"base_url": "https://test.atlassian.net/wiki"}, + "load_args": {"space_key": "ENG"}, + "timeout_seconds": 5, + "max_retries": 0, + "fail_on_error": True, + } + defaults.update(overrides) + return LlamaIndexPluginReaderConfig(**defaults) + + +def _mock_reader_module(reader_class_name="ConfluenceReader", load_return=None): + """Create a mock module with a mock reader class.""" + mock_module = MagicMock() + mock_reader_instance = Mock() + mock_reader_instance.load_data = Mock( + return_value=load_return if load_return is not None else [ + Document(text="Page 1 content", metadata={"title": "Page 1"}), + Document(text="Page 2 content", metadata={"title": "Page 2"}), + ] + ) + mock_reader_class = Mock(return_value=mock_reader_instance) + setattr(mock_module, reader_class_name, mock_reader_class) + return mock_module, mock_reader_class, mock_reader_instance + + +# --------------------------------------------------------------------------- +# Happy Path +# --------------------------------------------------------------------------- + +class TestHappyPath: + """Tests for successful document loading.""" + + def test_loads_documents_successfully(self): + """Reader returns documents — all pass through.""" + mock_module, mock_cls, mock_reader = _mock_reader_module() + config = _make_config() + + with patch("importlib.import_module", return_value=mock_module): + from graphrag_toolkit.lexical_graph.indexing.load.readers.providers.llama_index_plugin_reader_provider import ( + LlamaIndexPluginReaderProvider, + ) + provider = LlamaIndexPluginReaderProvider(config) + docs = provider.read() + + assert len(docs) == 2 + assert docs[0].text == "Page 1 content" + assert docs[1].metadata["title"] == "Page 2" + mock_reader.load_data.assert_called_once_with(space_key="ENG") + + def test_passes_input_source(self): + """input_source is passed through to load_args.""" + mock_module, _, mock_reader = _mock_reader_module() + config = _make_config(load_args={}) + + with patch("importlib.import_module", return_value=mock_module): + from graphrag_toolkit.lexical_graph.indexing.load.readers.providers.llama_index_plugin_reader_provider import ( + LlamaIndexPluginReaderProvider, + ) + provider = LlamaIndexPluginReaderProvider(config) + provider.read("https://wiki.example.com") + + mock_reader.load_data.assert_called_once_with( + input_source="https://wiki.example.com" + ) + + def test_input_source_fallback_when_not_accepted(self): + """If reader doesn't accept input_source kwarg, retry without it.""" + mock_module, _, mock_reader = _mock_reader_module() + # First call raises TypeError (unexpected kwarg), second succeeds + mock_reader.load_data.side_effect = [ + TypeError("unexpected keyword argument 'input_source'"), + [Document(text="worked", metadata={})], + ] + # Reset side_effect for the _call_load_fn retry + call_count = [0] + original_side_effect = mock_reader.load_data.side_effect + + def flexible_load(**kwargs): + call_count[0] += 1 + if call_count[0] == 1 and "input_source" in kwargs: + raise TypeError("unexpected keyword argument 'input_source'") + return [Document(text="worked", metadata={})] + + mock_reader.load_data.side_effect = flexible_load + config = _make_config(load_args={"space_key": "TEST"}) + + with patch("importlib.import_module", return_value=mock_module): + from graphrag_toolkit.lexical_graph.indexing.load.readers.providers.llama_index_plugin_reader_provider import ( + LlamaIndexPluginReaderProvider, + ) + provider = LlamaIndexPluginReaderProvider(config) + docs = provider.read("some_input") + + assert len(docs) == 1 + assert docs[0].text == "worked" + + def test_metadata_enrichment(self): + """metadata_fn enriches all returned documents.""" + mock_module, _, _ = _mock_reader_module() + config = _make_config( + metadata_fn=lambda source: {"source": "confluence", "team": "platform"} + ) + + with patch("importlib.import_module", return_value=mock_module): + from graphrag_toolkit.lexical_graph.indexing.load.readers.providers.llama_index_plugin_reader_provider import ( + LlamaIndexPluginReaderProvider, + ) + provider = LlamaIndexPluginReaderProvider(config) + docs = provider.read() + + for doc in docs: + assert doc.metadata["source"] == "confluence" + assert doc.metadata["team"] == "platform" + + +# --------------------------------------------------------------------------- +# Import Errors +# --------------------------------------------------------------------------- + +class TestImportErrors: + """Tests for missing packages and classes.""" + + def test_raises_on_missing_package(self): + """ImportError with install instructions when package not found.""" + config = _make_config( + package="llama-index-readers-confluence", + module_path="llama_index.readers.confluence", + ) + + with patch("importlib.import_module", side_effect=ImportError("No module named 'llama_index.readers.confluence'")): + from graphrag_toolkit.lexical_graph.indexing.load.readers.providers.llama_index_plugin_reader_provider import ( + LlamaIndexPluginReaderProvider, + ReaderImportError, + ) + with pytest.raises(ReaderImportError) as exc_info: + LlamaIndexPluginReaderProvider(config) + + assert "pip install" in str(exc_info.value) + assert "llama-index-readers-confluence" in str(exc_info.value) + + def test_raises_on_missing_class(self): + """ImportError when class not found in module.""" + mock_module = MagicMock() + # Make getattr raise AttributeError for ConfluenceReader + del mock_module.ConfluenceReader + config = _make_config(reader_class="ConfluenceReader") + + with patch("importlib.import_module", return_value=mock_module): + from graphrag_toolkit.lexical_graph.indexing.load.readers.providers.llama_index_plugin_reader_provider import ( + LlamaIndexPluginReaderProvider, + ReaderImportError, + ) + with pytest.raises(ReaderImportError) as exc_info: + LlamaIndexPluginReaderProvider(config) + + assert "not found" in str(exc_info.value) + + def test_raises_on_invalid_init_args(self): + """ValueError when constructor args don't match reader signature.""" + mock_module = MagicMock() + mock_cls = Mock(side_effect=TypeError("__init__() got unexpected keyword argument 'bogus'")) + mock_module.ConfluenceReader = mock_cls + config = _make_config(init_args={"bogus": "value"}) + + with patch("importlib.import_module", return_value=mock_module): + from graphrag_toolkit.lexical_graph.indexing.load.readers.providers.llama_index_plugin_reader_provider import ( + LlamaIndexPluginReaderProvider, + ) + with pytest.raises(ValueError) as exc_info: + LlamaIndexPluginReaderProvider(config) + + assert "init_args" in str(exc_info.value) + + +# --------------------------------------------------------------------------- +# Validation +# --------------------------------------------------------------------------- + +class TestValidation: + """Tests for config validation.""" + + def test_raises_on_missing_reader_class(self): + """ValueError when reader_class not provided.""" + config = _make_config(reader_class="") + + from graphrag_toolkit.lexical_graph.indexing.load.readers.providers.llama_index_plugin_reader_provider import ( + LlamaIndexPluginReaderProvider, + ) + with pytest.raises(ValueError, match="reader_class"): + LlamaIndexPluginReaderProvider(config) + + def test_raises_on_missing_module_and_package(self): + """ValueError when neither module_path nor package provided.""" + config = _make_config(module_path="", package="") + + from graphrag_toolkit.lexical_graph.indexing.load.readers.providers.llama_index_plugin_reader_provider import ( + LlamaIndexPluginReaderProvider, + ) + with pytest.raises(ValueError, match="module_path"): + LlamaIndexPluginReaderProvider(config) + + +# --------------------------------------------------------------------------- +# Auth Failures +# --------------------------------------------------------------------------- + +class TestAuthFailures: + """Tests for authentication error detection.""" + + @pytest.mark.parametrize("error_msg", [ + "401 Unauthorized", + "403 Forbidden", + "Invalid token provided", + "Token expired at 2026-01-01", + "Authentication failed: bad credentials", + "Access denied for user", + ]) + def test_detects_auth_errors(self, error_msg): + """Auth-like errors raise ReaderAuthError and are NOT retried.""" + mock_module, _, mock_reader = _mock_reader_module() + mock_reader.load_data.side_effect = RuntimeError(error_msg) + config = _make_config(max_retries=3) # Should NOT retry + + with patch("importlib.import_module", return_value=mock_module): + from graphrag_toolkit.lexical_graph.indexing.load.readers.providers.llama_index_plugin_reader_provider import ( + LlamaIndexPluginReaderProvider, + ReaderAuthError, + ) + provider = LlamaIndexPluginReaderProvider(config) + + with pytest.raises(ReaderAuthError): + provider.read() + + # Only called once — no retries on auth errors + assert mock_reader.load_data.call_count == 1 + + def test_auth_error_during_init(self): + """Auth error during reader construction is detected.""" + mock_module = MagicMock() + mock_cls = Mock(side_effect=RuntimeError("401 Unauthorized: invalid API key")) + mock_module.ConfluenceReader = mock_cls + config = _make_config() + + with patch("importlib.import_module", return_value=mock_module): + from graphrag_toolkit.lexical_graph.indexing.load.readers.providers.llama_index_plugin_reader_provider import ( + LlamaIndexPluginReaderProvider, + ReaderAuthError, + ) + with pytest.raises(ReaderAuthError): + LlamaIndexPluginReaderProvider(config) + + +# --------------------------------------------------------------------------- +# Timeout +# --------------------------------------------------------------------------- + +class TestTimeout: + """Tests for timeout enforcement.""" + + def test_raises_on_timeout(self): + """ReaderTimeoutError when reader exceeds timeout_seconds.""" + mock_module, _, mock_reader = _mock_reader_module() + + def slow_load(**kwargs): + time.sleep(10) + return [] + + mock_reader.load_data.side_effect = slow_load + config = _make_config(timeout_seconds=1, fail_on_error=True) + + with patch("importlib.import_module", return_value=mock_module): + from graphrag_toolkit.lexical_graph.indexing.load.readers.providers.llama_index_plugin_reader_provider import ( + LlamaIndexPluginReaderProvider, + ReaderTimeoutError, + ) + provider = LlamaIndexPluginReaderProvider(config) + + with pytest.raises((ReaderTimeoutError, RuntimeError)): + provider.read() + + def test_timeout_returns_empty_when_not_fail_on_error(self): + """Timeout returns [] when fail_on_error=False.""" + mock_module, _, mock_reader = _mock_reader_module() + + def slow_load(**kwargs): + time.sleep(10) + return [] + + mock_reader.load_data.side_effect = slow_load + config = _make_config(timeout_seconds=1, fail_on_error=False) + + with patch("importlib.import_module", return_value=mock_module): + from graphrag_toolkit.lexical_graph.indexing.load.readers.providers.llama_index_plugin_reader_provider import ( + LlamaIndexPluginReaderProvider, + ) + provider = LlamaIndexPluginReaderProvider(config) + docs = provider.read() + + assert docs == [] + + +# --------------------------------------------------------------------------- +# Retry with Backoff +# --------------------------------------------------------------------------- + +class TestRetry: + """Tests for retry with exponential backoff on transient errors.""" + + def test_retries_on_transient_error(self): + """Transient errors (429, 503) trigger retry.""" + mock_module, _, mock_reader = _mock_reader_module() + mock_reader.load_data.side_effect = [ + RuntimeError("429 Too Many Requests"), + [Document(text="success", metadata={})], + ] + config = _make_config(max_retries=1, retry_backoff_seconds=0.01) + + with patch("importlib.import_module", return_value=mock_module): + from graphrag_toolkit.lexical_graph.indexing.load.readers.providers.llama_index_plugin_reader_provider import ( + LlamaIndexPluginReaderProvider, + ) + provider = LlamaIndexPluginReaderProvider(config) + docs = provider.read() + + assert len(docs) == 1 + assert docs[0].text == "success" + assert mock_reader.load_data.call_count == 2 + + def test_exhausts_retries_then_fails(self): + """After max_retries, raises if fail_on_error=True.""" + mock_module, _, mock_reader = _mock_reader_module() + mock_reader.load_data.side_effect = RuntimeError("503 Service Unavailable") + config = _make_config(max_retries=2, retry_backoff_seconds=0.01, fail_on_error=True) + + with patch("importlib.import_module", return_value=mock_module): + from graphrag_toolkit.lexical_graph.indexing.load.readers.providers.llama_index_plugin_reader_provider import ( + LlamaIndexPluginReaderProvider, + ) + provider = LlamaIndexPluginReaderProvider(config) + + with pytest.raises(RuntimeError, match="all 3 attempt"): + provider.read() + + # 1 initial + 2 retries = 3 total + assert mock_reader.load_data.call_count == 3 + + def test_exhausts_retries_returns_empty_gracefully(self): + """After max_retries with fail_on_error=False, returns [].""" + mock_module, _, mock_reader = _mock_reader_module() + mock_reader.load_data.side_effect = RuntimeError("500 Internal Server Error") + config = _make_config(max_retries=1, retry_backoff_seconds=0.01, fail_on_error=False) + + with patch("importlib.import_module", return_value=mock_module): + from graphrag_toolkit.lexical_graph.indexing.load.readers.providers.llama_index_plugin_reader_provider import ( + LlamaIndexPluginReaderProvider, + ) + provider = LlamaIndexPluginReaderProvider(config) + docs = provider.read() + + assert docs == [] + + def test_non_transient_error_not_retried(self): + """Non-transient errors (ValueError, etc.) are NOT retried.""" + mock_module, _, mock_reader = _mock_reader_module() + mock_reader.load_data.side_effect = ValueError("Invalid space_key format") + config = _make_config(max_retries=3, retry_backoff_seconds=0.01, fail_on_error=True) + + with patch("importlib.import_module", return_value=mock_module): + from graphrag_toolkit.lexical_graph.indexing.load.readers.providers.llama_index_plugin_reader_provider import ( + LlamaIndexPluginReaderProvider, + ) + provider = LlamaIndexPluginReaderProvider(config) + + with pytest.raises(RuntimeError): + provider.read() + + # Only 1 attempt — no retries for non-transient errors + assert mock_reader.load_data.call_count == 1 + + +# --------------------------------------------------------------------------- +# Empty Results +# --------------------------------------------------------------------------- + +class TestEmptyResults: + """Tests for empty result handling.""" + + def test_empty_list_returns_with_warning(self, caplog): + """Empty result logs warning but doesn't error.""" + mock_module, _, mock_reader = _mock_reader_module(load_return=[]) + config = _make_config() + + with patch("importlib.import_module", return_value=mock_module): + from graphrag_toolkit.lexical_graph.indexing.load.readers.providers.llama_index_plugin_reader_provider import ( + LlamaIndexPluginReaderProvider, + ) + provider = LlamaIndexPluginReaderProvider(config) + + with caplog.at_level(logging.WARNING): + docs = provider.read() + + assert docs == [] + assert "0 documents" in caplog.text + + def test_none_return_handled_gracefully(self): + """Reader returning None doesn't crash.""" + mock_module, _, mock_reader = _mock_reader_module() + mock_reader.load_data.return_value = None + config = _make_config(fail_on_error=False) + + with patch("importlib.import_module", return_value=mock_module): + from graphrag_toolkit.lexical_graph.indexing.load.readers.providers.llama_index_plugin_reader_provider import ( + LlamaIndexPluginReaderProvider, + ) + provider = LlamaIndexPluginReaderProvider(config) + docs = provider.read() + + assert docs == [] + + +# --------------------------------------------------------------------------- +# Partial Failures / Invalid Documents +# --------------------------------------------------------------------------- + +class TestPartialFailures: + """Tests for filtering invalid items from results.""" + + def test_filters_non_document_items(self): + """Non-Document items in the result list are filtered out.""" + mixed_results = [ + Document(text="valid 1", metadata={}), + "I am not a document", + 42, + Document(text="valid 2", metadata={}), + None, + ] + mock_module, _, mock_reader = _mock_reader_module(load_return=mixed_results) + config = _make_config() + + with patch("importlib.import_module", return_value=mock_module): + from graphrag_toolkit.lexical_graph.indexing.load.readers.providers.llama_index_plugin_reader_provider import ( + LlamaIndexPluginReaderProvider, + ) + provider = LlamaIndexPluginReaderProvider(config) + docs = provider.read() + + assert len(docs) == 2 + assert docs[0].text == "valid 1" + assert docs[1].text == "valid 2" + + def test_handles_generator_return(self): + """Reader returning a generator is consumed into a list.""" + def doc_generator(): + yield Document(text="gen 1", metadata={}) + yield Document(text="gen 2", metadata={}) + yield Document(text="gen 3", metadata={}) + + mock_module, _, mock_reader = _mock_reader_module() + mock_reader.load_data.return_value = doc_generator() + config = _make_config() + + with patch("importlib.import_module", return_value=mock_module): + from graphrag_toolkit.lexical_graph.indexing.load.readers.providers.llama_index_plugin_reader_provider import ( + LlamaIndexPluginReaderProvider, + ) + provider = LlamaIndexPluginReaderProvider(config) + docs = provider.read() + + assert len(docs) == 3 + + +# --------------------------------------------------------------------------- +# Module Path Resolution +# --------------------------------------------------------------------------- + +class TestModuleResolution: + """Tests for deriving module_path from package name.""" + + def test_module_path_takes_priority(self): + """Explicit module_path is used over package derivation.""" + mock_module, _, _ = _mock_reader_module() + config = _make_config( + module_path="my.custom.module", + package="something-else", + ) + + with patch("importlib.import_module", return_value=mock_module) as mock_import: + from graphrag_toolkit.lexical_graph.indexing.load.readers.providers.llama_index_plugin_reader_provider import ( + LlamaIndexPluginReaderProvider, + ) + LlamaIndexPluginReaderProvider(config) + + mock_import.assert_called_once_with("my.custom.module") + + def test_derives_module_from_package_name(self): + """Package name llama-index-readers-X → llama_index.readers.X.""" + mock_module, _, _ = _mock_reader_module(reader_class_name="NotionReader") + config = _make_config( + module_path="", + package="llama-index-readers-notion", + reader_class="NotionReader", + ) + + with patch("importlib.import_module", return_value=mock_module) as mock_import: + from graphrag_toolkit.lexical_graph.indexing.load.readers.providers.llama_index_plugin_reader_provider import ( + LlamaIndexPluginReaderProvider, + ) + LlamaIndexPluginReaderProvider(config) + + mock_import.assert_called_once_with("llama_index.readers.notion") + + +# --------------------------------------------------------------------------- +# Load Method +# --------------------------------------------------------------------------- + +class TestLoadMethod: + """Tests for custom load method configuration.""" + + def test_uses_custom_load_method(self): + """load_method config routes to the correct reader method.""" + mock_module, _, mock_reader = _mock_reader_module() + mock_reader.lazy_load_data = Mock(return_value=[ + Document(text="lazy doc", metadata={}) + ]) + config = _make_config(load_method="lazy_load_data") + + with patch("importlib.import_module", return_value=mock_module): + from graphrag_toolkit.lexical_graph.indexing.load.readers.providers.llama_index_plugin_reader_provider import ( + LlamaIndexPluginReaderProvider, + ) + provider = LlamaIndexPluginReaderProvider(config) + docs = provider.read() + + assert len(docs) == 1 + mock_reader.lazy_load_data.assert_called_once() + + def test_missing_load_method_fails_gracefully(self): + """Non-existent load method returns [] when fail_on_error=False.""" + mock_module, _, mock_reader = _mock_reader_module() + # Remove the method + del mock_reader.custom_method + config = _make_config(load_method="custom_method", fail_on_error=False) + + with patch("importlib.import_module", return_value=mock_module): + from graphrag_toolkit.lexical_graph.indexing.load.readers.providers.llama_index_plugin_reader_provider import ( + LlamaIndexPluginReaderProvider, + ) + provider = LlamaIndexPluginReaderProvider(config) + docs = provider.read() + + assert docs == [] From 29f33257c370445d2f25d1d7f3503464ea08c076 Mon Sep 17 00:00:00 2001 From: Evan Erwee Date: Thu, 9 Jul 2026 16:55:22 -0400 Subject: [PATCH 2/2] =?UTF-8?q?security:=20address=20review=20feedback=20?= =?UTF-8?q?=E2=80=94=207=20hardening=20items?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Namespace allowlist (ALLOWED_MODULE_PREFIXES) — prevents arbitrary module loading 2. Interface validation before instantiation — checks callable + has load_data/lazy_load 3. Restrict load_method (ALLOWED_LOAD_METHODS) — only load_data, lazy_load, aload_data 4. Environment variable resolution ($VAR_NAME in init_args/load_args) 5. Credential logging test — asserts secrets never appear in logs 6. Default fail_on_error=True — explicit opt-in for graceful degradation 7. Package name validation in error messages — only suggest pip install for valid patterns 16 new security tests added (46 total, all passing). All items directly address mykola-pereyma's review on #384. --- PR-DESCRIPTION.md | 114 ++ .../codeproperty_graph/app_summary_builder.py | 128 ++ .../codeproperty_graph/artifact_reader.py | 179 ++ .../codeproperty_graph/artifact_validator.py | 188 ++ .../codeproperty_graph/code_slice_store.py | 209 +++ .../codeproperty_graph/graph_loader.py | 273 +++ .../codeproperty_graph/graphml_converter.py | 183 ++ .../codeproperty_graph/graphson_converter.py | 243 +++ .../codeproperty_graph/summary_overlay.py | 152 ++ .../codeproperty_graph/vector_loader.py | 304 ++++ .../tests/test_artifact_pipeline.py | 248 +++ docs/analysis/AOSS-NEXTGEN-ANALYSIS.md | 155 ++ docs/analysis/CHUNK-STORE-ANALYSIS.md | 183 ++ docs/analysis/CREATE-CONFIG-BUG-ANALYSIS.md | 157 ++ docs/analysis/LOGGING-ECS-EKS-ANALYSIS.md | 215 +++ .../audit-deployment.sh | 1618 +++++++++++++++++ .../audit-resources.json | 156 ++ .../cloudformation-templates/audit/README.md | 189 ++ .../audit/audit-deployment.sh | 1618 +++++++++++++++++ .../audit/audit-resources-recommended.json | 291 +++ .../audit/audit-resources.json | 156 ++ .../llama_index_plugin_reader_provider.py | 125 +- .../load/readers/reader_provider_config.py | 2 +- ...test_llama_index_plugin_reader_provider.py | 200 +- working/ADR-0009-dependency-discipline.md | 43 + ...al Model for Stateful Software Verdicts.md | 366 ++++ ...N Boundary and AWS Neptune Runtime.docx.md | 329 ++++ working/README.md | 5 + ...er Evidence Graph for Software Verdicts.md | 145 ++ working/adr/ADR-0001-cpg-rag-delta.md | 128 ++ ...ADR-0002-tenant-strategy-cpg-separation.md | 69 + ...003-store-topology-embedding-divergence.md | 76 + .../ADR-0004-store-agnostic-write-strategy.md | 85 + ...raction, CPG-RAG-Owned Schema-Driven Build | 44 + ...ss-cloud-build-neptune-public-endpoints.md | 57 + ...ed Vectorization vs Bring-Your-Own-Vector" | 38 + ...rock Model Providers and In-AWS Hosting.md | 61 + ...tecture-addendum-cpg-rag-store-strategy.md | 40 + working/architecture.md | 402 ++++ .../cloud-native-vs-self-hosted-assessment.md | 47 + .../federated_semantics_graph_architecture.md | 1012 +++++++++++ working/graphrag_com_redrawn_architecture.md | 575 ++++++ ..._neptune_analytics_cpg_rag_architecture.md | 586 ++++++ .../cpg-rag-missing-pieces/requirements.md | 143 ++ working/task-list.md | 205 +++ 45 files changed, 11720 insertions(+), 22 deletions(-) create mode 100644 PR-DESCRIPTION.md create mode 100644 codeproperty-graph/src/graphrag_toolkit/codeproperty_graph/app_summary_builder.py create mode 100644 codeproperty-graph/src/graphrag_toolkit/codeproperty_graph/artifact_reader.py create mode 100644 codeproperty-graph/src/graphrag_toolkit/codeproperty_graph/artifact_validator.py create mode 100644 codeproperty-graph/src/graphrag_toolkit/codeproperty_graph/code_slice_store.py create mode 100644 codeproperty-graph/src/graphrag_toolkit/codeproperty_graph/graph_loader.py create mode 100644 codeproperty-graph/src/graphrag_toolkit/codeproperty_graph/graphml_converter.py create mode 100644 codeproperty-graph/src/graphrag_toolkit/codeproperty_graph/graphson_converter.py create mode 100644 codeproperty-graph/src/graphrag_toolkit/codeproperty_graph/summary_overlay.py create mode 100644 codeproperty-graph/src/graphrag_toolkit/codeproperty_graph/vector_loader.py create mode 100644 codeproperty-graph/tests/test_artifact_pipeline.py create mode 100644 docs/analysis/AOSS-NEXTGEN-ANALYSIS.md create mode 100644 docs/analysis/CHUNK-STORE-ANALYSIS.md create mode 100644 docs/analysis/CREATE-CONFIG-BUG-ANALYSIS.md create mode 100644 docs/analysis/LOGGING-ECS-EKS-ANALYSIS.md create mode 100755 examples/lexical-graph/cloudformation-templates/audit-deployment.sh create mode 100644 examples/lexical-graph/cloudformation-templates/audit-resources.json create mode 100644 examples/lexical-graph/cloudformation-templates/audit/README.md create mode 100755 examples/lexical-graph/cloudformation-templates/audit/audit-deployment.sh create mode 100644 examples/lexical-graph/cloudformation-templates/audit/audit-resources-recommended.json create mode 100644 examples/lexical-graph/cloudformation-templates/audit/audit-resources.json create mode 100644 working/ADR-0009-dependency-discipline.md create mode 100644 working/Defining the Master Evidence Graph: A Formal Model for Stateful Software Verdicts.md create mode 100644 working/Domain Graph Migration Architecture_ Portable JSON Boundary and AWS Neptune Runtime.docx.md create mode 100644 working/README.md create mode 100644 working/Toward a Master Evidence Graph for Software Verdicts.md create mode 100644 working/adr/ADR-0001-cpg-rag-delta.md create mode 100644 working/adr/ADR-0002-tenant-strategy-cpg-separation.md create mode 100644 working/adr/ADR-0003-store-topology-embedding-divergence.md create mode 100644 working/adr/ADR-0004-store-agnostic-write-strategy.md create mode 100644 working/adr/ADR-0006: External Extraction, CPG-RAG-Owned Schema-Driven Build create mode 100644 working/adr/ADR-0010-cross-cloud-build-neptune-public-endpoints.md create mode 100644 "working/adr/Embedding Ownership \342\200\224 Toolkit-Owned Vectorization vs Bring-Your-Own-Vector" create mode 100644 working/adr/Non-Bedrock Model Providers and In-AWS Hosting.md create mode 100644 working/architecture-addendum-cpg-rag-store-strategy.md create mode 100644 working/architecture.md create mode 100644 working/cloud-native-vs-self-hosted-assessment.md create mode 100644 working/federated_semantics_graph_architecture.md create mode 100644 working/graphrag_com_redrawn_architecture.md create mode 100644 working/neptune_vs_neptune_analytics_cpg_rag_architecture.md create mode 100644 working/specs/cpg-rag-missing-pieces/requirements.md create mode 100644 working/task-list.md diff --git a/PR-DESCRIPTION.md b/PR-DESCRIPTION.md new file mode 100644 index 000000000..0f18ac548 --- /dev/null +++ b/PR-DESCRIPTION.md @@ -0,0 +1,114 @@ +# PR Description: Add document-graph and codeproperty-graph packages + +## Summary + +Two new packages for graphrag-toolkit, following the existing namespace and packaging pattern: + +- **document-graph** — Structured data ingestion (Domain Graph) +- **codeproperty-graph** — Delta-aware code analysis ingestion (Domain Graph, DRY) + +Both address the feedback from the original document-graph proposal: +namespace move, tenant model fix, dependency direction, repo hygiene. + +## What Changed + +### New Packages + +| Package | Dist Name | Namespace | Tests | +|---------|-----------|-----------|-------| +| document-graph | `graphrag-toolkit-document-graph` | `src/graphrag_toolkit/document_graph/` | 58 passing | +| codeproperty-graph | `graphrag-toolkit-codeproperty-graph` | `src/graphrag_toolkit/codeproperty_graph/` | 17 passing | + +### Addressing Prior Feedback + +| Feedback Item | Resolution | +|---------------|------------| +| Namespace: move under `graphrag_toolkit/` | ✅ Both packages follow byokg-rag pattern | +| Tenant model: use `TenantId.format_label()` | ✅ Optional import with standalone fallback producing identical format | +| Dependency direction: lexical optional | ✅ Base install works standalone; `[graphrag]` extra for Neptune | +| Repo hygiene: strip outputs, minimal data | ✅ No executed notebooks, sample data is minimal | + +### Hardcoded Data Cleanup + +A previous commit inadvertently included hardcoded values. This PR removes all of them: + +| What was hardcoded | Where | Fixed to | +|--------------------|-------|----------| +| AWS account number (`705909755305`) | install.sh, push-to-sagemaker.sh, notebooks | `${AWS_ACCOUNT_ID}` / `` | +| Neptune endpoint (`obs-app-dev-graph.cluster-...`) | 11 notebooks | `` | +| OpenSearch endpoint (`1lci0mi6xy...aoss.amazonaws.com`) | 2 notebooks | `` | +| S3 bucket ARN (`ccms-rag-extract-188967239867`) | `update-stack.sh` | `` | +| AWS profile name (`master`, `nw`) | scripts | `${AWS_PROFILE:-default}` | +| SageMaker paths (`~/SageMaker/document-graph/...`) | notebooks | Relative paths (`data/...`) | +| Python path (`/Library/Frameworks/Python...`) | push-to-sagemaker.sh | `python3` | + +### New Files (by area) + +**Packages:** +- `document-graph/` — src, tests, pyproject.toml, LICENSE, README, CHANGELOG +- `codeproperty-graph/` — src, tests, pyproject.toml, LICENSE, README, docs + +**Documentation (docs-site):** +- `docs-site/src/content/docs/document-graph/` — 5 pages (overview, pipeline, schema, hybrid, config) +- `docs-site/src/content/docs/codeproperty-graph/` — 3 pages (overview, delta-ingestion, config) +- Updated `astro.config.mjs` sidebar +- Updated `index.mdx` landing page cards + +**Examples:** +- `examples/document-graph/` — 24 notebooks + sample data + scripts +- `examples/codeproperty-graph/` — 2 notebooks + sample data + +**CI:** +- `.github/workflows/document-graph-tests.yml` +- `.github/workflows/codeproperty-graph-tests.yml` + +**Root:** +- Updated `README.md` (package descriptions) +- Updated `examples/README.md` +- Added `READMORE.md` (GraphRAG positioning, DRY architecture explanation) + +### Deleted + +- `document-graph/docs/` — old Sphinx docs (replaced by docs-site Starlight pages) +- `document-graph/examples/` — moved to monorepo top-level `examples/document-graph/` +- Broken `utility_functions.py` (referenced non-existent internal dependencies) + +## Key Design Decisions + +1. **document-graph is a Domain Graph** in the GraphRAG taxonomy — schema-first, typed nodes, deterministic +2. **codeproperty-graph proves DRY works** — ~200 lines adds an entire code analysis domain +3. **Tenant format verified against lexical's actual output** — `TenantId.format_label("User")` produces `` `Usertenant__` `` +4. **Plugin system added** (pluggy-based) — 4 extension points with sample plugins +5. **Full Joern CPG schema** — 20 node types, 14+ edge types, 9 languages, `from_joern()` factories +6. **No hardcoded credentials, endpoints, or account numbers** anywhere in the contribution + +## Testing + +```bash +# document-graph (58 tests) +cd document-graph && pip install -e ".[dev,graphrag]" && pytest tests/ + +# codeproperty-graph (17 tests) +cd codeproperty-graph && pip install -e ".[dev]" && pytest tests/ +``` + +## Notebooks That Run Without AWS + +The following notebooks require zero cloud dependencies (no Neptune, no credentials): + +- `02-Standalone-ETL` — Transform + Cypher generation locally +- `04-Full-Pipeline-Test` — All pipeline stages +- `05a-Ingestors` — Column/row operations +- `05d-Multi-Format-Extraction` — Parquet, Excel, XML, YAML +- `06a-06e` — All transformers, normalizers, constructors, enrichers +- `08-Schema-Providers` — Discovery and validation +- `09-Transformer-Deep-Dive` — Comprehensive guide +- `09b-Plugins` — Plugin system demo +- `01-CPG-Models-and-GraphDiff` (codeproperty-graph) — Schema + delta logic + +## Ownership + +We plan to use and maintain both packages. document-graph is in active production use +for structured data ingestion. codeproperty-graph runs in CI/CD for code analysis. +We will respond to issues, review external PRs, and maintain compatibility with +lexical-graph releases. diff --git a/codeproperty-graph/src/graphrag_toolkit/codeproperty_graph/app_summary_builder.py b/codeproperty-graph/src/graphrag_toolkit/codeproperty_graph/app_summary_builder.py new file mode 100644 index 000000000..440d6f7cb --- /dev/null +++ b/codeproperty-graph/src/graphrag_toolkit/codeproperty_graph/app_summary_builder.py @@ -0,0 +1,128 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""App Summary Builder — creates :AppSummary node per application. + +Client schema conformance: Each application has a top-level AppSummary node with: + - appName, apmId + - shortDescription, longDescription (LLM-generated or derived) + - totalMethods, totalFiles + - language + - embedding (for semantic search over app descriptions) + - summarizedTime, embeddedTime + +Linked: (:AppSummary)-[:HAS_METADATA]->(:Metadata) +""" + +import logging +from datetime import datetime, timezone +from typing import Any, Dict, List + +logger = logging.getLogger(__name__) + + +class AppSummaryBuilder: + """Creates an :AppSummary node from the manifest and node statistics. + + Usage: + builder = AppSummaryBuilder(graph_store=neptune_adapter) + await builder.build(manifest=manifest, nodes_data=nodes_data) + """ + + def __init__(self, graph_store: Any): + self._graph_store = graph_store + + async def build(self, manifest: dict, nodes_data: list[dict]) -> dict: + """Create the AppSummary + Metadata nodes and link them. + + Args: + manifest: The artifact manifest dict. + nodes_data: All node records (to compute stats). + + Returns: + Dict with app_summary_id and metadata_id. + """ + app_name = manifest.get("app_name", manifest.get("appName", manifest.get("repo_id", ""))) + apm_id = manifest.get("apm_id", manifest.get("apmId", "")) + language = manifest.get("language", "") + counts = manifest.get("counts", {}) + + # Compute stats from nodes + total_methods = counts.get("methods", len([n for n in nodes_data if n.get("node_type") == "METHOD"])) + total_files = counts.get("files", len([n for n in nodes_data if n.get("node_type") == "FILE"])) + total_type_decls = counts.get("type_decls", len([n for n in nodes_data if n.get("node_type") == "TYPE_DECL"])) + + # IDs + app_summary_id = f"{app_name}::AppSummary" + metadata_id = f"{app_name}::Metadata" + now = datetime.now(timezone.utc).isoformat() + + # Build AppSummary node + app_summary_node = { + "id": app_summary_id, + "label": "AppSummary", + "properties": { + "cpg_node_id": app_summary_id, + "appName": app_name, + "apmId": apm_id, + "shortDescription": f"{app_name} — {language} application with {total_methods} methods across {total_files} files.", + "longDescription": ( + f"Application '{app_name}' (APM: {apm_id}) is a {language} codebase " + f"containing {total_files} source files, {total_type_decls} classes/types, " + f"and {total_methods} methods. Analyzed by {manifest.get('extraction_tool', 'joern')} " + f"at commit {manifest.get('commit_sha', 'unknown')[:8]}." + ), + "language": language, + "totalMethods": total_methods, + "totalFiles": total_files, + "totalTypeDecls": total_type_decls, + "summarizedTime": now, + "domain": "cpg", + }, + } + + # Build Metadata node (client schema: stores extraction metadata) + metadata_node = { + "id": metadata_id, + "label": "Metadata", + "properties": { + "cpg_node_id": metadata_id, + "appName": app_name, + "apmId": apm_id, + "sourceCodeRepoUri": manifest.get("source_code_repo_uri", manifest.get("sourceCodeRepoUri", "")), + "language": language, + "cpgGeneratedTime": manifest.get("timestamp", now), + "cpgMode": "source", + "extractionTool": manifest.get("extraction_tool", "joern"), + "extractionVersion": manifest.get("extraction_version", ""), + "commitSha": manifest.get("commit_sha", ""), + "branch": manifest.get("branch", ""), + "lastModifiedTime": now, + "domain": "cpg", + }, + } + + # HAS_METADATA edge + has_metadata_edge = { + "source_id": app_summary_id, + "target_id": metadata_id, + "edge_type": "HAS_METADATA", + "properties": {}, + } + + # Write to Neptune + try: + await self._graph_store.write_nodes([app_summary_node, metadata_node]) + await self._graph_store.write_edges([has_metadata_edge]) + logger.info(f"AppSummary + Metadata created for {app_name} (apmId={apm_id})") + except Exception as e: + logger.warning(f"Failed to create AppSummary for {app_name}: {e}") + + return { + "app_summary_id": app_summary_id, + "metadata_id": metadata_id, + "app_name": app_name, + "apm_id": apm_id, + "total_methods": total_methods, + "total_files": total_files, + } diff --git a/codeproperty-graph/src/graphrag_toolkit/codeproperty_graph/artifact_reader.py b/codeproperty-graph/src/graphrag_toolkit/codeproperty_graph/artifact_reader.py new file mode 100644 index 000000000..7556678bd --- /dev/null +++ b/codeproperty-graph/src/graphrag_toolkit/codeproperty_graph/artifact_reader.py @@ -0,0 +1,179 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Artifact Reader — reads CPG artifact JSONL files from S3 or local filesystem. + +Reads the standard CPG artifact layout: + /// + manifest.json + nodes.jsonl + edges.jsonl + vectors.jsonl + summaries.jsonl + code_slices.jsonl + lineage.jsonl (optional) + findings.jsonl (optional) +""" + +import json +import logging +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional + +import boto3 + +logger = logging.getLogger(__name__) + + +@dataclass +class ArtifactContents: + """Parsed contents of a CPG artifact.""" + manifest: dict = field(default_factory=dict) + nodes: list[dict] = field(default_factory=list) + edges: list[dict] = field(default_factory=list) + vectors: list[dict] = field(default_factory=list) + summaries: list[dict] = field(default_factory=list) + code_slices: list[dict] = field(default_factory=list) + lineage: list[dict] = field(default_factory=list) + findings: list[dict] = field(default_factory=list) + + +class ArtifactReader: + """Reads a CPG artifact from S3 or local filesystem. + + Usage (S3): + reader = ArtifactReader(region="us-east-1") + artifact = reader.read_s3("s3://bucket/cpg-exports/payments-api/job-001/") + + Usage (local): + artifact = ArtifactReader.read_local("/path/to/artifact/") + """ + + def __init__(self, region: str = "us-east-1"): + self._s3 = boto3.client("s3", region_name=region) + + def read_s3(self, s3_uri: str) -> ArtifactContents: + """Read a CPG artifact from an S3 prefix. + + Args: + s3_uri: S3 URI prefix (e.g., "s3://bucket/cpg-exports/repo/job/") + + Returns: + ArtifactContents with all parsed files. + """ + bucket, prefix = self._parse_s3_uri(s3_uri) + if not prefix.endswith("/"): + prefix += "/" + + artifact = ArtifactContents() + + # Read manifest + artifact.manifest = self._read_s3_json(bucket, f"{prefix}manifest.json") + + # Read JSONL files + artifact.nodes = self._read_s3_jsonl(bucket, f"{prefix}nodes.jsonl") + artifact.edges = self._read_s3_jsonl(bucket, f"{prefix}edges.jsonl") + artifact.vectors = self._read_s3_jsonl(bucket, f"{prefix}vectors.jsonl") + artifact.summaries = self._read_s3_jsonl(bucket, f"{prefix}summaries.jsonl") + artifact.code_slices = self._read_s3_jsonl(bucket, f"{prefix}code_slices.jsonl") + + # Optional files + artifact.lineage = self._read_s3_jsonl(bucket, f"{prefix}lineage.jsonl", required=False) + artifact.findings = self._read_s3_jsonl(bucket, f"{prefix}findings.jsonl", required=False) + + logger.info( + f"Artifact read from s3://{bucket}/{prefix}: " + f"nodes={len(artifact.nodes)}, edges={len(artifact.edges)}, " + f"vectors={len(artifact.vectors)}, summaries={len(artifact.summaries)}" + ) + return artifact + + @staticmethod + def read_local(path: str) -> ArtifactContents: + """Read a CPG artifact from a local directory. + + Args: + path: Local directory path containing the artifact files. + + Returns: + ArtifactContents with all parsed files. + """ + directory = Path(path) + artifact = ArtifactContents() + + # Read manifest + manifest_path = directory / "manifest.json" + if manifest_path.exists(): + with open(manifest_path) as f: + artifact.manifest = json.load(f) + + # Read JSONL files + artifact.nodes = ArtifactReader._read_local_jsonl(directory / "nodes.jsonl") + artifact.edges = ArtifactReader._read_local_jsonl(directory / "edges.jsonl") + artifact.vectors = ArtifactReader._read_local_jsonl(directory / "vectors.jsonl") + artifact.summaries = ArtifactReader._read_local_jsonl(directory / "summaries.jsonl") + artifact.code_slices = ArtifactReader._read_local_jsonl(directory / "code_slices.jsonl") + artifact.lineage = ArtifactReader._read_local_jsonl(directory / "lineage.jsonl") + artifact.findings = ArtifactReader._read_local_jsonl(directory / "findings.jsonl") + + logger.info( + f"Artifact read from {path}: " + f"nodes={len(artifact.nodes)}, edges={len(artifact.edges)}, " + f"vectors={len(artifact.vectors)}, summaries={len(artifact.summaries)}" + ) + return artifact + + def _read_s3_json(self, bucket: str, key: str) -> dict: + """Read a single JSON file from S3.""" + try: + resp = self._s3.get_object(Bucket=bucket, Key=key) + return json.loads(resp["Body"].read()) + except self._s3.exceptions.NoSuchKey: + logger.warning(f"File not found: s3://{bucket}/{key}") + return {} + except Exception as e: + logger.error(f"Failed to read s3://{bucket}/{key}: {e}") + return {} + + def _read_s3_jsonl(self, bucket: str, key: str, required: bool = True) -> list[dict]: + """Read a JSONL file from S3 (one JSON object per line).""" + try: + resp = self._s3.get_object(Bucket=bucket, Key=key) + body = resp["Body"].read().decode("utf-8") + records = [] + for line in body.strip().split("\n"): + line = line.strip() + if line: + records.append(json.loads(line)) + return records + except self._s3.exceptions.NoSuchKey: + if required: + logger.warning(f"Required file not found: s3://{bucket}/{key}") + return [] + except Exception as e: + logger.error(f"Failed to read s3://{bucket}/{key}: {e}") + return [] + + @staticmethod + def _read_local_jsonl(path: Path) -> list[dict]: + """Read a local JSONL file.""" + if not path.exists(): + return [] + records = [] + with open(path) as f: + for line in f: + line = line.strip() + if line: + records.append(json.loads(line)) + return records + + @staticmethod + def _parse_s3_uri(uri: str) -> tuple[str, str]: + """Parse s3://bucket/prefix into (bucket, prefix).""" + if uri.startswith("s3://"): + uri = uri[5:] + parts = uri.split("/", 1) + bucket = parts[0] + prefix = parts[1] if len(parts) > 1 else "" + return bucket, prefix diff --git a/codeproperty-graph/src/graphrag_toolkit/codeproperty_graph/artifact_validator.py b/codeproperty-graph/src/graphrag_toolkit/codeproperty_graph/artifact_validator.py new file mode 100644 index 000000000..7ddb3e137 --- /dev/null +++ b/codeproperty-graph/src/graphrag_toolkit/codeproperty_graph/artifact_validator.py @@ -0,0 +1,188 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Validates a CPG artifact before Build processes it.""" + +import logging +from dataclasses import dataclass, field +from typing import Any + +logger = logging.getLogger(__name__) + +REQUIRED_MANIFEST_FIELDS = { + "artifact_type", + "schema_version", + "repo_id", + "commit_sha", + "embedding_model", + "embedding_dimensions", + "similarity_function", + "counts", +} + +# Recommended fields (warn if missing, don't fail) +RECOMMENDED_MANIFEST_FIELDS = { + "apm_id", # Client's primary scoping key (apmId) + "app_name", # Application name (appName) + "source_code_repo_uri", # sourceCodeRepoUri — Git/BitBucket URL + "language", # Programming language + "extraction_tool", # joern +} + +REQUIRED_NODE_FIELDS = {"cpg_node_id", "node_type", "properties"} +REQUIRED_VECTOR_FIELDS = {"cpg_node_id", "embedding_target", "vector", "embedding_dimensions"} +REQUIRED_SUMMARY_FIELDS = {"cpg_node_id", "summary_type", "text"} + + +@dataclass +class ValidationResult: + """Result of artifact validation.""" + + valid: bool = True + errors: list[str] = field(default_factory=list) + warnings: list[str] = field(default_factory=list) + + +class ArtifactValidator: + """Validates CPG artifact manifests and sample records. + + Designed to fail fast on critical errors (manifest structure) + but collect all warnings from record-level checks. + """ + + def validate( + self, + manifest: dict[str, Any], + sample_records: dict[str, list[dict[str, Any]]] | None = None, + ) -> ValidationResult: + """Validate a CPG artifact manifest and optional sample records. + + Args: + manifest: The artifact manifest dictionary. + sample_records: Optional dict with keys 'nodes', 'vectors', 'summaries', + each mapping to a list of sample JSONL records. + + Returns: + ValidationResult with valid flag, errors, and warnings. + """ + result = ValidationResult() + + # --- Manifest-level validation (fail fast on critical errors) --- + self._validate_manifest(manifest, result) + if result.errors: + result.valid = False + logger.error("Manifest validation failed: %s", result.errors) + return result + + # --- Record-level validation (collect all warnings) --- + if sample_records: + expected_dimensions = manifest.get("embedding_dimensions") + self._validate_node_records(sample_records.get("nodes", []), result) + self._validate_vector_records( + sample_records.get("vectors", []), expected_dimensions, result + ) + self._validate_summary_records(sample_records.get("summaries", []), result) + + if result.errors: + result.valid = False + + return result + + def _validate_manifest(self, manifest: dict[str, Any], result: ValidationResult) -> None: + """Validate required manifest fields and their values.""" + missing = REQUIRED_MANIFEST_FIELDS - set(manifest.keys()) + if missing: + result.errors.append(f"Missing required manifest fields: {sorted(missing)}") + return + + # Check recommended fields (warn, don't fail) + # Also check camelCase variants (apmId, appName, sourceCodeRepoUri) + for field in RECOMMENDED_MANIFEST_FIELDS: + camel = field.replace("_", "") # rough camelCase check + if field not in manifest and camel not in manifest: + # Try common variants + variants = { + "apm_id": ["apmId", "apm_id"], + "app_name": ["appName", "app_name", "repo_id"], + "source_code_repo_uri": ["sourceCodeRepoUri", "source_code_repo_uri", "repo_url"], + "language": ["language"], + "extraction_tool": ["extraction_tool", "extractionTool"], + } + found = any(v in manifest for v in variants.get(field, [])) + if not found: + result.warnings.append(f"Recommended manifest field missing: '{field}' (client schema conformance)") + + # artifact_type must be 'cpg' + if manifest["artifact_type"] != "cpg": + result.errors.append( + f"Invalid artifact_type: expected 'cpg', got '{manifest['artifact_type']}'" + ) + + # Validate counts + counts = manifest.get("counts") + if not isinstance(counts, dict): + result.errors.append("'counts' must be a dictionary") + return + + for count_field in ("nodes", "vectors", "summaries"): + value = counts.get(count_field) + if value is None: + result.errors.append(f"'counts.{count_field}' is missing") + elif not isinstance(value, int) or value <= 0: + result.errors.append( + f"'counts.{count_field}' must be a positive integer, got {value!r}" + ) + + def _validate_node_records( + self, records: list[dict[str, Any]], result: ValidationResult + ) -> None: + """Validate sample node records.""" + for i, record in enumerate(records): + missing = REQUIRED_NODE_FIELDS - set(record.keys()) + if missing: + result.warnings.append( + f"Node record [{i}]: missing fields {sorted(missing)}" + ) + + def _validate_vector_records( + self, + records: list[dict[str, Any]], + expected_dimensions: int | None, + result: ValidationResult, + ) -> None: + """Validate sample vector records and embedding dimensions.""" + for i, record in enumerate(records): + missing = REQUIRED_VECTOR_FIELDS - set(record.keys()) + if missing: + result.warnings.append( + f"Vector record [{i}]: missing fields {sorted(missing)}" + ) + continue + + # Check embedding_dimensions match manifest + record_dims = record.get("embedding_dimensions") + if expected_dimensions is not None and record_dims != expected_dimensions: + result.errors.append( + f"Vector record [{i}]: embedding_dimensions mismatch — " + f"record has {record_dims}, manifest specifies {expected_dimensions}" + ) + + # Check vector length matches declared dimensions + vector = record.get("vector") + if isinstance(vector, list) and expected_dimensions is not None: + if len(vector) != expected_dimensions: + result.warnings.append( + f"Vector record [{i}]: vector length {len(vector)} " + f"does not match embedding_dimensions {expected_dimensions}" + ) + + def _validate_summary_records( + self, records: list[dict[str, Any]], result: ValidationResult + ) -> None: + """Validate sample summary records.""" + for i, record in enumerate(records): + missing = REQUIRED_SUMMARY_FIELDS - set(record.keys()) + if missing: + result.warnings.append( + f"Summary record [{i}]: missing fields {sorted(missing)}" + ) diff --git a/codeproperty-graph/src/graphrag_toolkit/codeproperty_graph/code_slice_store.py b/codeproperty-graph/src/graphrag_toolkit/codeproperty_graph/code_slice_store.py new file mode 100644 index 000000000..68fc6093a --- /dev/null +++ b/codeproperty-graph/src/graphrag_toolkit/codeproperty_graph/code_slice_store.py @@ -0,0 +1,209 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Code Slice Store — attaches extracted code slices to CPG nodes in Neptune. + +Reads CodeSliceRecord dicts (from code_slices.jsonl) and stores them as +properties on their corresponding Neptune nodes. Each slice becomes a set +of sub-properties keyed by slice_type (e.g. full_method, signature_only). + +Property layout on a node: + code_slice_full_method = "" + code_slice_full_method_lang = "java" + code_slice_full_method_line_start = 42 + code_slice_full_method_line_end = 67 + code_slice_full_method_file_path = "src/Main.java" + +Multiple slice types per node are supported (full_method + signature_only). +""" + +import asyncio +import logging +from typing import Any, Dict, List, Protocol, TypedDict, runtime_checkable + +logger = logging.getLogger(__name__) + + +class CodeSliceRecord(TypedDict, total=False): + """Schema for a single code slice record from code_slices.jsonl.""" + + cpg_node_id: str + slice_type: str # e.g. "full_method", "signature_only" + code: str + language: str + line_start: int + line_end: int + file_path: str + + +@runtime_checkable +class GraphStoreProtocol(Protocol): + """Protocol for Neptune graph store — matches the interface used by delta_ingestor.""" + + def execute_query(self, query: str, parameters: Dict[str, Any]) -> Any: + ... + + +class CodeSliceStore: + """Stores code slices as properties on existing CPG nodes in Neptune. + + Usage: + store = CodeSliceStore(graph_store) + count = await store.store(slices) + """ + + def __init__(self, graph_store: GraphStoreProtocol) -> None: + """Initialize with a Neptune graph store. + + Args: + graph_store: Neptune graph store instance with execute_query method. + """ + self._graph_store = graph_store + + async def store(self, slices: List[CodeSliceRecord]) -> int: + """Attach code slices to their corresponding CPG nodes. + + For each record, sets properties on the node identified by cpg_node_id: + - code_slice_{slice_type}: the code text + - code_slice_{slice_type}_lang: language + - code_slice_{slice_type}_line_start: start line number + - code_slice_{slice_type}_line_end: end line number + - code_slice_{slice_type}_file_path: source file path + + Args: + slices: List of CodeSliceRecord dicts parsed from code_slices.jsonl. + + Returns: + Number of slices successfully stored. + """ + if not slices: + return 0 + + stored = 0 + + for record in slices: + cpg_node_id = record.get("cpg_node_id") + slice_type = record.get("slice_type") + + if not cpg_node_id or not slice_type: + logger.warning( + "Skipping slice record with missing cpg_node_id or slice_type: %s", + record, + ) + continue + + try: + await self._store_single(record, cpg_node_id, slice_type) + stored += 1 + except Exception as e: + logger.error( + "Failed to store slice (node=%s, type=%s): %s", + cpg_node_id, + slice_type, + e, + ) + + logger.info("Stored %d/%d code slices", stored, len(slices)) + return stored + + async def _store_single( + self, record: CodeSliceRecord, cpg_node_id: str, slice_type: str + ) -> None: + """Store a single code slice as properties on a Neptune node. + + Args: + record: The code slice record dict. + cpg_node_id: Neptune node identifier. + slice_type: Slice type key (e.g. "full_method"). + """ + prefix = f"code_slice_{slice_type}" + + # Build the SET clause with parameterized values + query = ( + f"MATCH (n) WHERE n.id = $node_id " + f"SET n.`{prefix}` = $code, " + f"n.`{prefix}_lang` = $language, " + f"n.`{prefix}_line_start` = $line_start, " + f"n.`{prefix}_line_end` = $line_end, " + f"n.`{prefix}_file_path` = $file_path" + ) + + parameters: Dict[str, Any] = { + "node_id": cpg_node_id, + "code": record.get("code", ""), + "language": record.get("language", ""), + "line_start": record.get("line_start", 0), + "line_end": record.get("line_end", 0), + "file_path": record.get("file_path", ""), + } + + await asyncio.to_thread(self._graph_store.execute_query, query, parameters) + + async def store_batch(self, slices: List[CodeSliceRecord], batch_size: int = 50) -> int: + """Store slices in batches for improved throughput on large datasets. + + Groups slices by cpg_node_id to minimize round-trips when a single node + has multiple slice types (e.g. full_method + signature_only). + + Args: + slices: List of CodeSliceRecord dicts. + batch_size: Number of slices per batch query. + + Returns: + Number of slices successfully stored. + """ + if not slices: + return 0 + + # Group slices by node to coalesce updates + node_slices: Dict[str, List[CodeSliceRecord]] = {} + for record in slices: + node_id = record.get("cpg_node_id", "") + if node_id: + node_slices.setdefault(node_id, []).append(record) + + stored = 0 + + for node_id, node_records in node_slices.items(): + try: + set_clauses: List[str] = [] + parameters: Dict[str, Any] = {"node_id": node_id} + + for idx, record in enumerate(node_records): + slice_type = record.get("slice_type", "") + if not slice_type: + continue + + prefix = f"code_slice_{slice_type}" + param_suffix = f"_{idx}" + + set_clauses.extend([ + f"n.`{prefix}` = $code{param_suffix}", + f"n.`{prefix}_lang` = $language{param_suffix}", + f"n.`{prefix}_line_start` = $line_start{param_suffix}", + f"n.`{prefix}_line_end` = $line_end{param_suffix}", + f"n.`{prefix}_file_path` = $file_path{param_suffix}", + ]) + + parameters[f"code{param_suffix}"] = record.get("code", "") + parameters[f"language{param_suffix}"] = record.get("language", "") + parameters[f"line_start{param_suffix}"] = record.get("line_start", 0) + parameters[f"line_end{param_suffix}"] = record.get("line_end", 0) + parameters[f"file_path{param_suffix}"] = record.get("file_path", "") + + if not set_clauses: + continue + + query = ( + f"MATCH (n) WHERE n.id = $node_id " + f"SET {', '.join(set_clauses)}" + ) + + await asyncio.to_thread(self._graph_store.execute_query, query, parameters) + stored += len(node_records) + + except Exception as e: + logger.error("Batch store failed for node %s: %s", node_id, e) + + logger.info("Batch-stored %d/%d code slices", stored, len(slices)) + return stored diff --git a/codeproperty-graph/src/graphrag_toolkit/codeproperty_graph/graph_loader.py b/codeproperty-graph/src/graphrag_toolkit/codeproperty_graph/graph_loader.py new file mode 100644 index 000000000..c453b083f --- /dev/null +++ b/codeproperty-graph/src/graphrag_toolkit/codeproperty_graph/graph_loader.py @@ -0,0 +1,273 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Graph Loader — loads CPG nodes and edges from artifact data into Neptune. + +Converts raw artifact dicts (from nodes.jsonl / edges.jsonl) into typed CPG +models and writes them in batches to a graph store, applying tenant isolation. +""" + +import logging +from typing import Any, Protocol, runtime_checkable + +from .models import CPGNode, CPGEdge + +logger = logging.getLogger(__name__) + + +@runtime_checkable +class GraphStoreProtocol(Protocol): + """Protocol for graph store backends (e.g., Neptune).""" + + async def write_nodes(self, nodes: list[dict]) -> int: + """Write a batch of node dicts to the graph store. + + Each dict has keys: 'id', 'label', 'properties'. + + Returns: + Number of nodes written. + """ + ... + + async def write_edges(self, edges: list[dict]) -> int: + """Write a batch of edge dicts to the graph store. + + Each dict has keys: 'source_id', 'target_id', 'edge_type', 'properties'. + + Returns: + Number of edges written. + """ + ... + + +class GraphLoader: + """Loads CPG nodes and edges from artifact data into a graph store. + + Converts raw artifact dicts into CPGNode/CPGEdge models, applies tenant + isolation via tenant_id, and writes in configurable batches. + + Usage: + loader = GraphLoader( + graph_store=neptune_store, + tenant_id="amigo-core__abc123", + batch_size=200, + ) + result = await loader.load(nodes_data, edges_data) + # result = {"nodes_written": 1500, "edges_written": 3200} + """ + + def __init__( + self, + graph_store: GraphStoreProtocol, + tenant_id: str, + batch_size: int = 100, + apm_id: str = "", + app_name: str = "", + ) -> None: + """Initialize the GraphLoader. + + Args: + graph_store: Backend implementing GraphStoreProtocol. + tenant_id: Tenant identifier applied to all nodes/edges for isolation. + batch_size: Number of nodes/edges per write batch. Default 100. + apm_id: APM ID from client's application registry (primary scoping key). + app_name: Application/repo name. + """ + self._graph_store = graph_store + self._tenant_id = tenant_id + self._batch_size = batch_size + self._apm_id = apm_id + self._app_name = app_name + + async def load( + self, + nodes_data: list[dict], + edges_data: list[dict], + ) -> dict[str, Any]: + """Load nodes and edges into the graph store. + + Converts raw artifact dicts to typed CPG models, applies tenant_id, + batches writes, and returns summary counts. + + Args: + nodes_data: List of node dicts from nodes.jsonl artifact. + edges_data: List of edge dicts from edges.jsonl artifact. + + Returns: + Dict with 'nodes_written' and 'edges_written' counts. + """ + logger.info( + "GraphLoader.load: tenant_id=%s, nodes=%d, edges=%d, batch_size=%d", + self._tenant_id, + len(nodes_data), + len(edges_data), + self._batch_size, + ) + + nodes_written = await self._load_nodes(nodes_data) + edges_written = await self._load_edges(edges_data) + + logger.info( + "GraphLoader.load complete: nodes_written=%d, edges_written=%d", + nodes_written, + edges_written, + ) + + return { + "nodes_written": nodes_written, + "edges_written": edges_written, + } + + async def _load_nodes(self, nodes_data: list[dict]) -> int: + """Convert and write nodes in batches. + + Args: + nodes_data: Raw node dicts from artifact. + + Returns: + Total number of nodes written. + """ + total_written = 0 + total = len(nodes_data) + + for batch_start in range(0, total, self._batch_size): + batch_end = min(batch_start + self._batch_size, total) + raw_batch = nodes_data[batch_start:batch_end] + + write_batch = [] + for raw in raw_batch: + node = CPGNode.from_artifact(raw) + write_batch.append(self._node_to_write_dict(node)) + + count = await self._graph_store.write_nodes(write_batch) + total_written += count + + logger.debug( + "Nodes batch [%d-%d] of %d: wrote %d", + batch_start, + batch_end, + total, + count, + ) + + return total_written + + async def _load_edges(self, edges_data: list[dict]) -> int: + """Convert and write edges in batches. + + Args: + edges_data: Raw edge dicts from artifact. + + Returns: + Total number of edges written. + """ + total_written = 0 + total = len(edges_data) + + for batch_start in range(0, total, self._batch_size): + batch_end = min(batch_start + self._batch_size, total) + raw_batch = edges_data[batch_start:batch_end] + + write_batch = [] + for raw in raw_batch: + edge = CPGEdge.from_artifact(raw) + write_batch.append(self._edge_to_write_dict(edge)) + + count = await self._graph_store.write_edges(write_batch) + total_written += count + + logger.debug( + "Edges batch [%d-%d] of %d: wrote %d", + batch_start, + batch_end, + total, + count, + ) + + return total_written + + def _node_to_write_dict(self, node: CPGNode) -> dict: + """Convert a CPGNode to the write dict format expected by GraphStoreProtocol. + + Applies tenant_id as both a label prefix and a property for isolation. + + Args: + node: Typed CPGNode instance. + + Returns: + Dict with 'id', 'label', 'properties' keys. + """ + properties = { + "tenant_id": self._tenant_id, + "domain": "cpg", + "apmId": self._apm_id, + "appName": self._app_name, + "full_name": node.full_name, + "FULL_NAME": node.full_name, + "NAME": node.name, + "hash": node.hash, + "filename": node.filename, + "FILENAME": node.filename, + "name": node.name, + "code": node.code, + "CODE": node.code, + "signature": node.signature, + "SIGNATURE": node.signature, + "type_full_name": node.type_full_name, + "TYPE_FULL_NAME": node.type_full_name, + "is_external": node.is_external, + "IS_EXTERNAL": node.is_external, + "labelV": node.node_type, + } + + # Include optional fields only when present (using client's field names) + if node.line_number is not None: + properties["line_number"] = node.line_number + properties["LINE_NUMBER"] = node.line_number + if node.line_number_end is not None: + properties["line_number_end"] = node.line_number_end + properties["LINE_NUMBER_END"] = node.line_number_end + if node.order is not None: + properties["order"] = node.order + properties["ORDER"] = node.order + + # Client-required fields from Joern properties (explicit for index support) + properties["MODIFIER_TYPE"] = node.properties.get("MODIFIER_TYPE", node.properties.get("modifierType", "")) + properties["METHOD_FULL_NAME"] = node.properties.get("METHOD_FULL_NAME", node.properties.get("methodFullName", "")) + + # Merge any extra properties from the raw artifact + for key, value in node.properties.items(): + if key not in properties: + properties[key] = value + + return { + "id": node.id, + "label": f"{self._tenant_id}__{node.node_type}", + "properties": properties, + } + + def _edge_to_write_dict(self, edge: CPGEdge) -> dict: + """Convert a CPGEdge to the write dict format expected by GraphStoreProtocol. + + Applies tenant_id as a property for isolation. + + Args: + edge: Typed CPGEdge instance. + + Returns: + Dict with 'source_id', 'target_id', 'edge_type', 'properties' keys. + """ + properties = { + "tenant_id": self._tenant_id, + "domain": "cpg", + "apmId": self._apm_id, + "appName": self._app_name, + **edge.properties, + } + + return { + "source_id": edge.source_id, + "target_id": edge.target_id, + "edge_type": edge.edge_type, + "properties": properties, + } diff --git a/codeproperty-graph/src/graphrag_toolkit/codeproperty_graph/graphml_converter.py b/codeproperty-graph/src/graphrag_toolkit/codeproperty_graph/graphml_converter.py new file mode 100644 index 000000000..be0d31258 --- /dev/null +++ b/codeproperty-graph/src/graphrag_toolkit/codeproperty_graph/graphml_converter.py @@ -0,0 +1,183 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""GraphML → JSONL Converter — transforms Joern GraphML exports to portable CPG JSONL. + +GraphML is an XML format with and elements. Each has children +with key/value properties. Joern exports GraphML via: + joern-export --repr cpg14 --format graphml --out output/ + +Usage: + converter = GraphMLConverter(repo_id="payments-api", commit_sha="abc123") + nodes, edges = converter.convert_file("path/to/export.graphml") + GraphMLConverter.write_jsonl(nodes, "output/nodes.jsonl") + GraphMLConverter.write_jsonl(edges, "output/edges.jsonl") +""" + +import json +import logging +import xml.etree.ElementTree as ET +from pathlib import Path +from typing import IO + +from .models import CPGNode, CPGEdge + +logger = logging.getLogger(__name__) + +# GraphML namespace +GRAPHML_NS = "http://graphml.graphstruct.org/xmlns" +NS = {"g": GRAPHML_NS} + + +class GraphMLConverter: + """Converts Joern GraphML exports to portable CPG JSONL format. + + Handles the standard GraphML format: + + + ... + + + com.example.Main.main + METHOD + + + AST + + + + """ + + def __init__(self, repo_id: str, commit_sha: str = ""): + self._repo_id = repo_id + self._commit_sha = commit_sha + self._id_map: dict[str, str] = {} # graphml_id → cpg_node_id + + def convert_file(self, path: str) -> tuple[list[dict], list[dict]]: + """Convert a GraphML file to portable JSONL records. + + Args: + path: Path to the GraphML file. + + Returns: + Tuple of (node_records, edge_records) ready for JSONL output. + """ + tree = ET.parse(path) + root = tree.getroot() + + # Auto-detect namespace + ns = "" + if root.tag.startswith("{"): + ns = root.tag.split("}")[0] + "}" + + # Parse key definitions (maps key id → attribute name) + key_map = {} + for key_elem in root.iter(f"{ns}key"): + key_id = key_elem.get("id", "") + attr_name = key_elem.get("attr.name", "") + if key_id and attr_name: + key_map[key_id] = attr_name + + # Find the graph element + graph = root.find(f"{ns}graph") + if graph is None: + # Try without namespace + graph = root.find("graph") + if graph is None: + logger.warning(f"No element found in {path}") + return [], [] + + # Parse nodes + raw_nodes = [] + for node_elem in graph.iter(f"{ns}node"): + node_id = node_elem.get("id", "") + props = {} + for data_elem in node_elem.iter(f"{ns}data"): + key_id = data_elem.get("key", "") + attr_name = key_map.get(key_id, key_id) + value = data_elem.text or "" + props[attr_name] = value + raw_nodes.append({"id": node_id, "label": props.pop("labelV", props.pop("label", "UNKNOWN")), "properties": props}) + + # Parse edges + raw_edges = [] + for edge_elem in graph.iter(f"{ns}edge"): + source = edge_elem.get("source", "") + target = edge_elem.get("target", "") + props = {} + for data_elem in edge_elem.iter(f"{ns}data"): + key_id = data_elem.get("key", "") + attr_name = key_map.get(key_id, key_id) + value = data_elem.text or "" + props[attr_name] = value + label = props.pop("labelE", props.pop("label", "UNKNOWN")) + raw_edges.append({"src": source, "dst": target, "label": label, "properties": props}) + + # Convert using the same logic as GraphSON converter + nodes = self._convert_nodes(raw_nodes) + edges = self._convert_edges(raw_edges) + + logger.info( + f"GraphML converted: {len(nodes)} nodes, {len(edges)} edges " + f"(repo={self._repo_id}, commit={self._commit_sha})" + ) + return nodes, edges + + def _convert_nodes(self, raw_nodes: list[dict]) -> list[dict]: + """Convert raw nodes to portable JSONL records.""" + records = [] + for raw in raw_nodes: + node = CPGNode.from_joern(raw) + cpg_node_id = self._make_cpg_node_id(node) + self._id_map[node.id] = cpg_node_id + + records.append({ + "cpg_node_id": cpg_node_id, + "node_type": node.node_type, + "labels": [node.node_type], + "repo_id": self._repo_id, + "commit_sha": self._commit_sha, + "file_path": node.filename, + "fully_qualified_name": node.full_name, + "name": node.name, + "line_start": node.line_number, + "line_end": node.line_number_end, + "code_hash": node.hash, + "properties": node.properties, + }) + return records + + def _convert_edges(self, raw_edges: list[dict]) -> list[dict]: + """Convert raw edges to portable JSONL records.""" + records = [] + for raw in raw_edges: + edge = CPGEdge.from_joern(raw) + source_cpg_id = self._id_map.get(edge.source_id, edge.source_id) + target_cpg_id = self._id_map.get(edge.target_id, edge.target_id) + + records.append({ + "source_cpg_node_id": source_cpg_id, + "target_cpg_node_id": target_cpg_id, + "edge_type": edge.edge_type, + "properties": edge.properties, + }) + return records + + def _make_cpg_node_id(self, node: CPGNode) -> str: + """Generate a stable cpg_node_id.""" + identity = node.full_name or node.name or node.id + hash_part = node.hash or "" + parts = [self._repo_id, self._commit_sha, node.node_type, identity] + if hash_part: + parts.append(hash_part) + return ":".join(parts) + + @staticmethod + def write_jsonl(records: list[dict], output_path: str) -> int: + """Write records to a JSONL file.""" + path = Path(output_path) + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w") as f: + for record in records: + f.write(json.dumps(record) + "\n") + return len(records) diff --git a/codeproperty-graph/src/graphrag_toolkit/codeproperty_graph/graphson_converter.py b/codeproperty-graph/src/graphrag_toolkit/codeproperty_graph/graphson_converter.py new file mode 100644 index 000000000..6c636a71f --- /dev/null +++ b/codeproperty-graph/src/graphrag_toolkit/codeproperty_graph/graphson_converter.py @@ -0,0 +1,243 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""GraphSON → JSONL Converter — transforms Joern GraphSON exports to portable CPG JSONL. + +Joern's export produces GraphSON format (JSON with nodes[] and edges[] arrays). +This converter reads that format and outputs our portable JSONL schema with +stable cpg_node_id identifiers (replacing Joern's internal numeric IDs). + +Usage: + converter = GraphSONConverter(repo_id="payments-api", commit_sha="abc123") + nodes, edges = converter.convert_file("path/to/joern-export.json") + converter.write_jsonl(nodes, "output/nodes.jsonl") + converter.write_jsonl(edges, "output/edges.jsonl") +""" + +import json +import logging +from pathlib import Path +from typing import IO + +from .models import CPGNode, CPGEdge + +logger = logging.getLogger(__name__) + + +class GraphSONConverter: + """Converts Joern GraphSON exports to portable CPG JSONL format. + + The conversion: + 1. Reads GraphSON (JSON array or {nodes:[], edges:[]}) + 2. Parses each record via CPGNode.from_joern() / CPGEdge.from_joern() + 3. Generates stable cpg_node_id (replacing numeric IDs) + 4. Outputs JSONL with portable identifiers + + The cpg_node_id format: :::: + """ + + def __init__(self, repo_id: str, commit_sha: str = ""): + self._repo_id = repo_id + self._commit_sha = commit_sha + self._id_map: dict[str, str] = {} # joern_id → cpg_node_id + + def convert_file(self, path: str) -> tuple[list[dict], list[dict]]: + """Convert a GraphSON file to portable JSONL records. + + Accepts either: + - A single JSON file with {"nodes": [...], "edges": [...]} + - Separate nodes.json and edges.json (pass nodes path, infers edges) + + Args: + path: Path to the GraphSON file. + + Returns: + Tuple of (node_records, edge_records) ready for JSONL output. + """ + path = Path(path) + with open(path) as f: + data = json.load(f) + + # Handle TinkerPop GraphSON format: {"@type": "tinker:graph", "@value": {"vertices": [...], "edges": [...]}} + if isinstance(data, dict) and data.get("@type") == "tinker:graph": + graph_value = data.get("@value", {}) + raw_nodes = [self._unwrap_vertex(v) for v in graph_value.get("vertices", [])] + raw_edges = [self._unwrap_edge(e) for e in graph_value.get("edges", [])] + elif isinstance(data, dict): + raw_nodes = data.get("nodes", data.get("vertices", [])) + raw_edges = data.get("edges", []) + elif isinstance(data, list): + # Heuristic: if items have "src"/"dst", they're edges; otherwise nodes + if data and ("src" in data[0] or "dst" in data[0]): + raw_nodes = [] + raw_edges = data + else: + raw_nodes = data + raw_edges = [] + # Try to find edges file alongside + edges_path = path.parent / "edges.json" + if edges_path.exists(): + with open(edges_path) as f: + raw_edges = json.load(f) + else: + raw_nodes, raw_edges = [], [] + + nodes = self._convert_nodes(raw_nodes) + edges = self._convert_edges(raw_edges) + + logger.info( + f"GraphSON converted: {len(nodes)} nodes, {len(edges)} edges " + f"(repo={self._repo_id}, commit={self._commit_sha})" + ) + return nodes, edges + + def convert_nodes_stream(self, stream: IO) -> list[dict]: + """Convert a stream of GraphSON nodes (JSON array).""" + raw_nodes = json.load(stream) + if isinstance(raw_nodes, dict): + raw_nodes = raw_nodes.get("nodes", raw_nodes.get("vertices", [])) + return self._convert_nodes(raw_nodes) + + def convert_edges_stream(self, stream: IO) -> list[dict]: + """Convert a stream of GraphSON edges (JSON array).""" + raw_edges = json.load(stream) + if isinstance(raw_edges, dict): + raw_edges = raw_edges.get("edges", []) + return self._convert_edges(raw_edges) + + @staticmethod + def _unwrap_value(val): + """Unwrap a TinkerPop typed value: {"@type": "g:Int64", "@value": 123} → 123.""" + if isinstance(val, dict) and "@value" in val: + inner = val["@value"] + # Handle nested lists: {"@type": "g:List", "@value": ["foo"]} + if isinstance(inner, list): + return inner[0] if len(inner) == 1 else inner + return inner + return val + + def _unwrap_vertex(self, vertex: dict) -> dict: + """Convert a TinkerPop g:Vertex to a flat Joern-style dict for from_joern(). + + TinkerPop format: + {"@type": "g:Vertex", "id": {"@type": "g:Int64", "@value": 123}, + "label": "METHOD", "properties": {"FULL_NAME": {"@type": "g:VertexProperty", "@value": {...}}}} + + Joern format (what from_joern expects): + {"id": "123", "label": "METHOD", "properties": {"FULL_NAME": "pkg.Foo.bar"}} + """ + raw_id = self._unwrap_value(vertex.get("id", "")) + label = vertex.get("label", "UNKNOWN") + + # Unwrap properties from TinkerPop VertexProperty format + props = {} + raw_props = vertex.get("properties", {}) + for key, prop_val in raw_props.items(): + if isinstance(prop_val, dict): + # {"@type": "g:VertexProperty", "@value": {"@type": "g:List", "@value": ["actual_value"]}} + inner = prop_val.get("@value", prop_val) + if isinstance(inner, dict) and "@value" in inner: + inner = inner["@value"] + if isinstance(inner, list): + val = inner[0] if len(inner) == 1 else inner + else: + val = inner + # Final unwrap if value is still a typed wrapper (e.g., {"@type": "g:Int32", "@value": 4}) + if isinstance(val, dict) and "@value" in val: + val = val["@value"] + props[key] = val + else: + props[key] = prop_val + + return {"id": str(raw_id), "label": label, "properties": props} + + def _unwrap_edge(self, edge: dict) -> dict: + """Convert a TinkerPop g:Edge to a flat Joern-style dict for from_joern(). + + TinkerPop format: + {"@type": "g:Edge", "id": {...}, "inV": {"@type": "g:Int64", "@value": 456}, + "outV": {"@type": "g:Int64", "@value": 123}, "label": "AST", "properties": {}} + + Joern format: + {"src": "123", "dst": "456", "label": "AST", "properties": {}} + """ + src = str(self._unwrap_value(edge.get("outV", ""))) + dst = str(self._unwrap_value(edge.get("inV", ""))) + label = edge.get("label", "UNKNOWN") + + # Unwrap edge properties if any + props = {} + for key, val in edge.get("properties", {}).items(): + props[key] = self._unwrap_value(val) if isinstance(val, dict) else val + + return {"src": src, "dst": dst, "label": label, "properties": props} + + def _convert_nodes(self, raw_nodes: list[dict]) -> list[dict]: + """Convert raw Joern nodes to portable JSONL records.""" + records = [] + for raw in raw_nodes: + node = CPGNode.from_joern(raw) + cpg_node_id = self._make_cpg_node_id(node) + self._id_map[node.id] = cpg_node_id + + records.append({ + "cpg_node_id": cpg_node_id, + "node_type": node.node_type, + "labels": [node.node_type], + "repo_id": self._repo_id, + "commit_sha": self._commit_sha, + "file_path": node.filename, + "fully_qualified_name": node.full_name, + "name": node.name, + "line_start": node.line_number, + "line_end": node.line_number_end, + "code_hash": node.hash, + "properties": node.properties, + }) + return records + + def _convert_edges(self, raw_edges: list[dict]) -> list[dict]: + """Convert raw Joern edges to portable JSONL records.""" + records = [] + for raw in raw_edges: + edge = CPGEdge.from_joern(raw) + source_cpg_id = self._id_map.get(edge.source_id, edge.source_id) + target_cpg_id = self._id_map.get(edge.target_id, edge.target_id) + + records.append({ + "source_cpg_node_id": source_cpg_id, + "target_cpg_node_id": target_cpg_id, + "edge_type": edge.edge_type, + "properties": edge.properties, + }) + return records + + def _make_cpg_node_id(self, node: CPGNode) -> str: + """Generate a stable cpg_node_id for a Joern node. + + Format: :::: + """ + identity = node.full_name or node.name or node.id + hash_part = node.hash or "" + parts = [self._repo_id, self._commit_sha, node.node_type, identity] + if hash_part: + parts.append(hash_part) + return ":".join(parts) + + @staticmethod + def write_jsonl(records: list[dict], output_path: str) -> int: + """Write records to a JSONL file (one JSON object per line). + + Args: + records: List of dicts to write. + output_path: Path to output file. + + Returns: + Number of records written. + """ + path = Path(output_path) + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "w") as f: + for record in records: + f.write(json.dumps(record) + "\n") + return len(records) diff --git a/codeproperty-graph/src/graphrag_toolkit/codeproperty_graph/summary_overlay.py b/codeproperty-graph/src/graphrag_toolkit/codeproperty_graph/summary_overlay.py new file mode 100644 index 000000000..87cf98a38 --- /dev/null +++ b/codeproperty-graph/src/graphrag_toolkit/codeproperty_graph/summary_overlay.py @@ -0,0 +1,152 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Summary Node Builder — creates :MethodSummary / :FileSummary nodes linked via HAS_SUMMARY. + +Client schema conformance: summaries are separate graph nodes (not properties on METHOD/FILE). +Each summary node has: + - shortDescription (first sentence) + - longDescription (full summary text) + - embedding (written separately by VectorLoader) + - summarizedTime + - embeddedTime + +Linked to parent: (METHOD)-[:HAS_SUMMARY]->(:MethodSummary) + (FILE)-[:HAS_SUMMARY]->(:FileSummary) +""" + +import asyncio +import logging +from typing import Any, Dict, List, Protocol, runtime_checkable + +logger = logging.getLogger(__name__) + + +@runtime_checkable +class GraphStoreProtocol(Protocol): + """Protocol for graph store implementations.""" + + async def write_nodes(self, nodes: list[dict]) -> int: + ... + + async def write_edges(self, edges: list[dict]) -> int: + ... + + async def update_node_properties(self, node_id: str, properties: dict) -> bool: + ... + + +class SummaryOverlay: + """Creates summary nodes and links them to their parent CPG nodes. + + For each summary record: + 1. Creates a :MethodSummary or :FileSummary node + 2. Creates a HAS_SUMMARY edge from the parent METHOD/FILE to the summary node + 3. Also writes shortDescription/longDescription as properties on the parent (for convenience queries) + + Args: + graph_store: An implementation of GraphStoreProtocol. + batch_size: Number of records to process per batch. + """ + + def __init__(self, graph_store: Any, batch_size: int = 50): + self._graph_store = graph_store + self._batch_size = batch_size + + async def apply(self, records: List[Dict[str, Any]]) -> int: + """Create summary nodes and link to parent CPG nodes. + + Args: + records: List of summary record dicts from summaries.jsonl. + Each has: cpg_node_id, summary_type, text, generation_model, generation_prompt_version + + Returns: + Number of summary nodes successfully created. + """ + if not records: + return 0 + + count = 0 + summary_nodes = [] + summary_edges = [] + + for record in records: + cpg_node_id = record.get("cpg_node_id", "") + summary_type = record.get("summary_type", "") + text = record.get("text", "") + generation_model = record.get("generation_model", "") + prompt_version = record.get("generation_prompt_version", "") + + if not cpg_node_id or not text: + logger.warning(f"Skipping summary record: missing cpg_node_id or text") + continue + + # Determine node label based on summary_type + if summary_type == "method_summary": + label = "MethodSummary" + elif summary_type == "file_summary": + label = "FileSummary" + elif summary_type == "class_summary": + label = "ClassSummary" + else: + label = "Summary" + + # Summary node ID: parent_id + "::summary" + summary_node_id = f"{cpg_node_id}::summary::{summary_type}" + + # Extract short description (first sentence) + short_desc = text.split(".")[0] + "." if "." in text else text[:100] + + # Create the summary node + summary_nodes.append({ + "id": summary_node_id, + "label": label, + "properties": { + "cpg_node_id": summary_node_id, + "parent_cpg_node_id": cpg_node_id, + "shortDescription": short_desc, + "longDescription": text, + "methodName": cpg_node_id.split(":")[-2] if ":" in cpg_node_id else "", + "summary_type": summary_type, + "generation_model": generation_model, + "generation_prompt_version": prompt_version, + "summarizedTime": record.get("summarized_time", ""), + }, + }) + + # Create HAS_SUMMARY edge from parent to summary + summary_edges.append({ + "source_id": cpg_node_id, + "target_id": summary_node_id, + "edge_type": "HAS_SUMMARY", + "properties": {"summary_type": summary_type}, + }) + + # Also write shortDescription/longDescription on the parent node (convenience) + try: + await self._graph_store.update_node_properties(cpg_node_id, { + "shortDescription": short_desc, + "longDescription": text, + }) + except Exception as e: + logger.debug(f"Failed to write summary properties on parent {cpg_node_id}: {e}") + + count += 1 + + # Batch write summary nodes + if summary_nodes: + try: + written = await self._graph_store.write_nodes(summary_nodes) + logger.info(f"Created {written} summary nodes") + except Exception as e: + logger.warning(f"Failed to write summary nodes: {e}") + + # Batch write HAS_SUMMARY edges + if summary_edges: + try: + written = await self._graph_store.write_edges(summary_edges) + logger.info(f"Created {written} HAS_SUMMARY edges") + except Exception as e: + logger.warning(f"Failed to write HAS_SUMMARY edges: {e}") + + return count diff --git a/codeproperty-graph/src/graphrag_toolkit/codeproperty_graph/vector_loader.py b/codeproperty-graph/src/graphrag_toolkit/codeproperty_graph/vector_loader.py new file mode 100644 index 000000000..fc2c68bb9 --- /dev/null +++ b/codeproperty-graph/src/graphrag_toolkit/codeproperty_graph/vector_loader.py @@ -0,0 +1,304 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Vector Loader — loads CPG embedding vectors into OpenSearch via a VectorStoreProtocol. + +Reads VectorRecord dicts (from vectors.jsonl or in-memory list) and transforms +them into the format expected by the vector store, grouping by embedding_target +and generating compound document IDs for multi-vector-per-node support. + +Document ID scheme: f"{cpg_node_id}::{embedding_target}" + Allows a single CPG node to have multiple embeddings (e.g. code, docstring, signature). +""" + +import json +import logging +from collections import defaultdict +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Protocol, runtime_checkable + +logger = logging.getLogger(__name__) + + +@runtime_checkable +class VectorStoreProtocol(Protocol): + """Protocol for vector store backends (OpenSearch, etc.). + + Each document dict passed to put_vectors has: + id: str — compound key (cpg_node_id::embedding_target) + vector: list[float] + text: str — the source text that was embedded + metadata: dict — arbitrary metadata (node_type, filename, etc.) + """ + + async def put_vectors(self, documents: list[dict]) -> int: + """Bulk-index vector documents. Returns count of documents indexed.""" + ... + + +@dataclass +class VectorRecord: + """A single embedding vector produced by the CPG embedding stage. + + Fields align with vectors.jsonl output format: + cpg_node_id: The source CPG node ID this vector represents + embedding_target: What was embedded (e.g. 'code', 'docstring', 'signature') + vector: The embedding vector (list of floats) + text: The source text that was embedded + metadata: Additional context (node_type, filename, line_number, etc.) + embedding_model: Model identifier that produced this vector + embedding_dimensions: Declared dimensionality of the vector + """ + + cpg_node_id: str + embedding_target: str + vector: list[float] + text: str + metadata: dict[str, Any] = field(default_factory=dict) + embedding_model: str = "" + embedding_dimensions: int = 0 + + @property + def document_id(self) -> str: + """Compound key for OpenSearch: allows multiple vectors per CPG node.""" + return f"{self.cpg_node_id}::{self.embedding_target}" + + @classmethod + def from_dict(cls, raw: dict) -> "VectorRecord": + """Create a VectorRecord from a raw dict (e.g. parsed JSONL line).""" + return cls( + cpg_node_id=raw["cpg_node_id"], + embedding_target=raw["embedding_target"], + vector=raw["vector"], + text=raw.get("text", ""), + metadata=raw.get("metadata", {}), + embedding_model=raw.get("embedding_model", ""), + embedding_dimensions=raw.get("embedding_dimensions", 0), + ) + + +@dataclass +class LoadResult: + """Result of a vector load operation.""" + + total_loaded: int = 0 + by_target: dict[str, int] = field(default_factory=dict) + errors: list[str] = field(default_factory=list) + + +class VectorLoader: + """Loads CPG embedding vectors into a vector store (OpenSearch). + + Usage: + loader = VectorLoader( + vector_store=opensearch_store, + expected_dimensions=1024, + batch_size=500, + ) + result = await loader.load(records) + # or + result = await loader.load_from_file("s3://bucket/vectors.jsonl") + + Args: + vector_store: Any object implementing VectorStoreProtocol + expected_dimensions: Expected embedding dimensions (for validation) + batch_size: Number of documents per bulk indexing call + """ + + def __init__( + self, + vector_store: VectorStoreProtocol, + expected_dimensions: int = 0, + batch_size: int = 500, + graph_store=None, + ): + self._store = vector_store + self._expected_dimensions = expected_dimensions + self._batch_size = batch_size + self._graph_store = graph_store + + async def load(self, records: list[dict | VectorRecord]) -> LoadResult: + """Load vector records into the vector store. + + Args: + records: List of VectorRecord instances or raw dicts (vectors.jsonl format). + + Returns: + LoadResult with counts and any validation errors. + """ + result = LoadResult() + grouped: dict[str, list[dict]] = defaultdict(list) + + for raw in records: + record = raw if isinstance(raw, VectorRecord) else VectorRecord.from_dict(raw) + + # Validate dimensions + error = self._validate(record) + if error: + result.errors.append(error) + continue + + # Transform to vector store document format + doc = { + "id": record.document_id, + "vector": record.vector, + "text": record.text, + "metadata": { + **record.metadata, + "cpg_node_id": record.cpg_node_id, + "embedding_target": record.embedding_target, + "embedding_model": record.embedding_model, + }, + } + grouped[record.embedding_target].append(doc) + + # Bulk-index by embedding_target group + for target, documents in grouped.items(): + count = await self._bulk_index(documents) + result.by_target[target] = count + result.total_loaded += count + logger.info(f"Loaded {count} vectors for target '{target}'") + + # Also write embedding as a node property on Neptune (client schema conformance) + if self._graph_store and records: + written = 0 + for raw in records: + record = raw if isinstance(raw, VectorRecord) else VectorRecord.from_dict(raw) + try: + await self._graph_store.update_node_properties( + record.cpg_node_id, + {"embedding": record.vector, "embeddedTime": record.metadata.get("embedded_time", "")}, + ) + written += 1 + except Exception as e: + logger.debug(f"Failed to write embedding property for {record.cpg_node_id}: {e}") + if written: + logger.info(f"Wrote embedding property to {written} Neptune nodes") + + if result.errors: + logger.warning(f"Encountered {len(result.errors)} validation errors during load") + + return result + + async def load_from_file(self, path: str) -> LoadResult: + """Load vectors from a JSONL file (local path or S3 URI). + + Args: + path: Local filesystem path or S3 URI (s3://bucket/key) to vectors.jsonl. + + Returns: + LoadResult with counts and any validation errors. + """ + records = await self._read_jsonl(path) + logger.info(f"Read {len(records)} vector records from {path}") + return await self.load(records) + + def _validate(self, record: VectorRecord) -> str | None: + """Validate a vector record. Returns error string or None if valid.""" + actual_dims = len(record.vector) + + # Check declared dimensions match actual vector length + if record.embedding_dimensions and actual_dims != record.embedding_dimensions: + return ( + f"Vector {record.document_id}: declared dimensions " + f"({record.embedding_dimensions}) != actual ({actual_dims})" + ) + + # Check against expected model dimensions if configured + if self._expected_dimensions and actual_dims != self._expected_dimensions: + return ( + f"Vector {record.document_id}: actual dimensions " + f"({actual_dims}) != expected ({self._expected_dimensions})" + ) + + if not record.vector: + return f"Vector {record.document_id}: empty vector" + + return None + + async def _bulk_index(self, documents: list[dict]) -> int: + """Index documents in batches via the vector store protocol.""" + total_indexed = 0 + + for i in range(0, len(documents), self._batch_size): + batch = documents[i : i + self._batch_size] + try: + count = await self._store.put_vectors(batch) + total_indexed += count + except Exception as e: + logger.error(f"Bulk index error on batch {i // self._batch_size}: {e}") + # Continue with remaining batches + continue + + return total_indexed + + async def _read_jsonl(self, path: str) -> list[dict]: + """Read vector records from a JSONL file (local or S3). + + Args: + path: Local path or s3://bucket/key URI. + + Returns: + List of raw dicts parsed from JSONL lines. + """ + if path.startswith("s3://"): + return await self._read_s3_jsonl(path) + return self._read_local_jsonl(path) + + @staticmethod + def _read_local_jsonl(path: str) -> list[dict]: + """Read JSONL from local filesystem.""" + records: list[dict] = [] + file_path = Path(path) + + if not file_path.exists(): + raise FileNotFoundError(f"Vector file not found: {path}") + + with file_path.open("r", encoding="utf-8") as f: + for line_num, line in enumerate(f, 1): + line = line.strip() + if not line: + continue + try: + records.append(json.loads(line)) + except json.JSONDecodeError as e: + logger.warning(f"Skipping malformed line {line_num} in {path}: {e}") + + return records + + @staticmethod + async def _read_s3_jsonl(uri: str) -> list[dict]: + """Read JSONL from S3 using aiobotocore/boto3. + + Args: + uri: S3 URI in format s3://bucket/key + + Returns: + List of raw dicts parsed from JSONL lines. + """ + import aiobotocore.session # type: ignore[import-untyped] + + # Parse s3://bucket/key + parts = uri.replace("s3://", "").split("/", 1) + bucket = parts[0] + key = parts[1] if len(parts) > 1 else "" + + session = aiobotocore.session.get_session() + records: list[dict] = [] + + async with session.create_client("s3") as client: + response = await client.get_object(Bucket=bucket, Key=key) + async with response["Body"] as stream: + content = await stream.read() + + for line_num, line in enumerate(content.decode("utf-8").splitlines(), 1): + line = line.strip() + if not line: + continue + try: + records.append(json.loads(line)) + except json.JSONDecodeError as e: + logger.warning(f"Skipping malformed line {line_num} in {uri}: {e}") + + return records diff --git a/codeproperty-graph/tests/test_artifact_pipeline.py b/codeproperty-graph/tests/test_artifact_pipeline.py new file mode 100644 index 000000000..99a5aa554 --- /dev/null +++ b/codeproperty-graph/tests/test_artifact_pipeline.py @@ -0,0 +1,248 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for codeproperty-graph extended modules (artifact pipeline).""" + +import pytest +from graphrag_toolkit.codeproperty_graph.models import ( + CPGNode, CPGEdge, VectorRecord, SummaryRecord, CodeSliceRecord, +) +from graphrag_toolkit.codeproperty_graph.artifact_validator import ArtifactValidator, ValidationResult + + +# === from_artifact() factory tests === + +class TestCPGNodeFromArtifact: + def test_basic_method_node(self): + raw = { + "cpg_node_id": "payments-api:abc123:Method:auth.TokenService.validateToken:sha256001", + "node_type": "METHOD", + "labels": ["METHOD"], + "repo_id": "payments-api", + "commit_sha": "abc123", + "file_path": "src/auth/token.py", + "fully_qualified_name": "auth.TokenService.validateToken", + "name": "validateToken", + "line_start": 42, + "line_end": 91, + "code_hash": "sha256001", + "properties": {"language": "python", "visibility": "public"}, + } + node = CPGNode.from_artifact(raw) + assert node.id == "payments-api:abc123:Method:auth.TokenService.validateToken:sha256001" + assert node.node_type == "METHOD" + assert node.full_name == "auth.TokenService.validateToken" + assert node.hash == "sha256001" + assert node.filename == "src/auth/token.py" + assert node.name == "validateToken" + assert node.line_number == 42 + assert node.line_number_end == 91 + assert node.properties == {"language": "python", "visibility": "public"} + + def test_file_node(self): + raw = { + "cpg_node_id": "payments-api:abc123:File:src/auth/token.py", + "node_type": "FILE", + "file_path": "src/auth/token.py", + "name": "token.py", + "properties": {}, + } + node = CPGNode.from_artifact(raw) + assert node.node_type == "FILE" + assert node.filename == "src/auth/token.py" + assert node.line_number is None + + def test_stable_id_uses_full_name(self): + raw = { + "cpg_node_id": "repo:commit:Method:pkg.Class.method:hash", + "node_type": "METHOD", + "fully_qualified_name": "pkg.Class.method", + "properties": {}, + } + node = CPGNode.from_artifact(raw) + assert node.stable_id == "pkg.Class.method" + + +class TestCPGEdgeFromArtifact: + def test_contains_edge(self): + raw = { + "edge_id": "edge:001", + "source_cpg_node_id": "payments-api:abc123:File:src/auth/token.py", + "target_cpg_node_id": "payments-api:abc123:Method:auth.TokenService.validateToken:sha256001", + "edge_type": "CONTAINS", + "properties": {"analysis_run_id": "run-20260704-001"}, + } + edge = CPGEdge.from_artifact(raw) + assert edge.source_id == "payments-api:abc123:File:src/auth/token.py" + assert edge.target_id == "payments-api:abc123:Method:auth.TokenService.validateToken:sha256001" + assert edge.edge_type == "CONTAINS" + assert edge.properties == {"analysis_run_id": "run-20260704-001"} + + def test_edge_key(self): + raw = { + "source_cpg_node_id": "a", + "target_cpg_node_id": "b", + "edge_type": "AST", + "properties": {}, + } + edge = CPGEdge.from_artifact(raw) + assert edge.key == "a->AST->b" + + +class TestVectorRecord: + def test_from_artifact(self): + raw = { + "cpg_node_id": "payments-api:abc123:Method:validateToken:sha256001", + "embedding_target": "method_summary", + "embedding_model": "nomic-embed-text", + "embedding_dimensions": 768, + "similarity_function": "cosine", + "embedding_text_hash": "sha256:9f3a2b", + "vector": [0.012, -0.044, 0.087], + } + rec = VectorRecord.from_artifact(raw) + assert rec.cpg_node_id == "payments-api:abc123:Method:validateToken:sha256001" + assert rec.embedding_target == "method_summary" + assert rec.embedding_model == "nomic-embed-text" + assert rec.embedding_dimensions == 768 + assert rec.similarity_function == "cosine" + assert rec.vector == [0.012, -0.044, 0.087] + + +class TestSummaryRecord: + def test_from_artifact(self): + raw = { + "cpg_node_id": "payments-api:abc123:Method:validateToken:sha256001", + "summary_type": "method_summary", + "text": "Validates a JWT token by checking the signature.", + "generation_model": "mistral-7b", + "generation_prompt_version": "v2.1", + } + rec = SummaryRecord.from_artifact(raw) + assert rec.cpg_node_id == "payments-api:abc123:Method:validateToken:sha256001" + assert rec.summary_type == "method_summary" + assert rec.text == "Validates a JWT token by checking the signature." + assert rec.generation_model == "mistral-7b" + assert rec.generation_prompt_version == "v2.1" + + +class TestCodeSliceRecord: + def test_from_artifact(self): + raw = { + "cpg_node_id": "payments-api:abc123:Method:validateToken:sha256001", + "slice_type": "full_method", + "code": "def validateToken(self, token: str) -> dict:\n pass", + "language": "python", + "line_start": 42, + "line_end": 53, + "file_path": "src/auth/token.py", + } + rec = CodeSliceRecord.from_artifact(raw) + assert rec.cpg_node_id == "payments-api:abc123:Method:validateToken:sha256001" + assert rec.slice_type == "full_method" + assert rec.language == "python" + assert rec.line_start == 42 + assert rec.line_end == 53 + + +# === ArtifactValidator tests === + +class TestArtifactValidator: + def _valid_manifest(self): + return { + "artifact_type": "cpg", + "schema_version": "1.0", + "repo_id": "payments-api", + "commit_sha": "abc123def456", + "embedding_model": "nomic-embed-text", + "embedding_dimensions": 768, + "similarity_function": "cosine", + "counts": {"nodes": 100, "vectors": 50, "summaries": 30}, + } + + def test_valid_manifest_passes(self): + validator = ArtifactValidator() + result = validator.validate(manifest=self._valid_manifest()) + assert result.valid + assert not result.errors + + def test_missing_required_field(self): + manifest = self._valid_manifest() + del manifest["repo_id"] + validator = ArtifactValidator() + result = validator.validate(manifest=manifest) + assert not result.valid + assert any("repo_id" in e for e in result.errors) + + def test_wrong_artifact_type(self): + manifest = self._valid_manifest() + manifest["artifact_type"] = "lexical" + validator = ArtifactValidator() + result = validator.validate(manifest=manifest) + assert not result.valid + assert any("artifact_type" in e for e in result.errors) + + def test_zero_counts_rejected(self): + manifest = self._valid_manifest() + manifest["counts"]["nodes"] = 0 + validator = ArtifactValidator() + result = validator.validate(manifest=manifest) + assert not result.valid + + def test_vector_dimension_mismatch(self): + manifest = self._valid_manifest() + vectors = [ + { + "cpg_node_id": "x", + "embedding_target": "method_summary", + "vector": [0.1, 0.2, 0.3], + "embedding_dimensions": 512, # mismatch with manifest's 768 + } + ] + validator = ArtifactValidator() + result = validator.validate(manifest=manifest, sample_records={"vectors": vectors}) + # Should produce a warning or error about dimension mismatch + assert result.warnings or not result.valid + + def test_valid_with_samples(self): + manifest = self._valid_manifest() + nodes = [{"cpg_node_id": "x", "node_type": "METHOD", "properties": {}}] + vectors = [ + { + "cpg_node_id": "x", + "embedding_target": "method_summary", + "vector": [0.1] * 768, + "embedding_dimensions": 768, + } + ] + summaries = [ + {"cpg_node_id": "x", "summary_type": "method_summary", "text": "Does stuff."} + ] + validator = ArtifactValidator() + result = validator.validate( + manifest=manifest, + sample_records={"nodes": nodes, "vectors": vectors, "summaries": summaries}, + ) + assert result.valid + + +# === DeltaIngestor._derive_tenant_id tests === + +class TestDeriveTenantId: + def test_simple_repo(self): + from graphrag_toolkit.codeproperty_graph.delta_ingestor import DeltaIngestor + assert DeltaIngestor._derive_tenant_id("payments-api") == "payments.api" + + def test_slash_repo(self): + from graphrag_toolkit.codeproperty_graph.delta_ingestor import DeltaIngestor + assert DeltaIngestor._derive_tenant_id("org/payments-api") == "org.payments.api" + + def test_truncation(self): + from graphrag_toolkit.codeproperty_graph.delta_ingestor import DeltaIngestor + long_name = "a" * 50 + result = DeltaIngestor._derive_tenant_id(long_name) + assert len(result) <= 25 + + def test_empty_fallback(self): + from graphrag_toolkit.codeproperty_graph.delta_ingestor import DeltaIngestor + assert DeltaIngestor._derive_tenant_id("") == "default" diff --git a/docs/analysis/AOSS-NEXTGEN-ANALYSIS.md b/docs/analysis/AOSS-NEXTGEN-ANALYSIS.md new file mode 100644 index 000000000..53b8c373a --- /dev/null +++ b/docs/analysis/AOSS-NEXTGEN-ANALYSIS.md @@ -0,0 +1,155 @@ +# AOSS NextGen Vector Search Collections: Analysis + +**Reference:** [FEATURE] Add support for AOSS NextGen vector search collections — [awslabs/graphrag-toolkit#359](https://github.com/awslabs/graphrag-toolkit/issues/359) + +--- + +## Summary + +Valid feature request. AOSS NextGen collections reject the `engine` and `method` parameters that the current `opensearch_vector_indexes.py` unconditionally includes. The proposed flag-based approach is correct, low-risk, and follows existing patterns. + +--- + +## The Problem (Verified in Source) + +`opensearch_vector_indexes.py` line 253-296 builds index mappings with a `method` block: + +```python +# Line 277-282 (nmslib path): +method = { + "name": "hnsw", + "space_type": "l2", + "engine": "nmslib", # ← NextGen rejects this + "parameters": {"ef_construction": 256, "m": 48}, # ← and this +} +``` + +AOSS NextGen (OpenSearch 3.x) removed these parameters — the system determines engine and parameters internally. Passing them returns: + +``` +illegal_argument_exception: "Field parameter 'engine' is not supported" +``` + +--- + +## Why This Matters Now + +- **NextGen is AWS's recommended collection type** for new deployments +- Scale-to-zero pricing (Classic doesn't have this) +- GPU-accelerated index builds +- 32x compression by default +- Anyone creating a new AOSS collection via console today gets NextGen by default +- This is a **growing** compatibility issue — more users will hit it over time + +--- + +## Proposed Solution Assessment + +| Aspect | Assessment | +|--------|-----------| +| Approach (config flag) | ✅ Correct — matches existing `opensearch_engine` pattern | +| Default (False) | ✅ Non-breaking — existing Classic behavior unchanged | +| Mapping difference | ✅ Correctly identified — no `method` block, `space_type` at field level | +| Bulk ingest | ✅ No changes needed (NextGen supports custom doc IDs) | +| Future removal | ✅ Flag is easy to remove when Classic deprecated | +| Alternative rejected (auto-detect) | ✅ Correct rejection — adds latency, hides intent | +| Alternative rejected (new prefix) | ✅ Correct — `aoss-nextgen://` would confuse users | + +--- + +## Implementation Notes + +The natural extension point is at line 253 in `opensearch_vector_indexes.py`, where the code already branches on `opensearch_engine`: + +```python +# Current structure: +if GraphRAGConfig.opensearch_engine.lower() == 'faiss': + # faiss mapping +else: + # nmslib mapping + +# Proposed addition (before existing branches): +if GraphRAGConfig.opensearch_serverless_nextgen: + # NextGen mapping (no method block) +elif GraphRAGConfig.opensearch_engine.lower() == 'faiss': + # faiss mapping (unchanged) +else: + # nmslib mapping (unchanged) +``` + +The NextGen mapping is simpler than either existing branch: + +```python +idx_conf = { + "settings": {"index": {"knn": True}}, + "mappings": { + "date_detection": False, + "properties": { + embedding_field: { + "type": "knn_vector", + "dimension": dimensions, + "space_type": "l2", + } + } + } +} + +# Optional compression +if GraphRAGConfig.opensearch_serverless_nextgen_compression: + idx_conf["mappings"]["properties"][embedding_field]["compression_level"] = \ + GraphRAGConfig.opensearch_serverless_nextgen_compression +``` + +--- + +## Config Changes (follows existing pattern exactly) + +The `GraphRAGConfig` class (config.py) already has this exact pattern for `opensearch_engine`: + +```python +# Existing pattern (line 293, 1165-1173): +_opensearch_engine: Optional[str] = None + +@property +def opensearch_engine(self) -> str: + if self._opensearch_engine is None: + self._opensearch_engine = os.environ.get('OPENSEARCH_ENGINE', DEFAULT_OPENSEARCH_ENGINE) + return self._opensearch_engine + +@opensearch_engine.setter +def opensearch_engine(self, opensearch_engine: str) -> None: + self._opensearch_engine = opensearch_engine +``` + +Two new properties following the identical pattern: +- `opensearch_serverless_nextgen` (bool, env: `OPENSEARCH_SERVERLESS_NEXTGEN`) +- `opensearch_serverless_nextgen_compression` (Optional[str], env: `OPENSEARCH_SERVERLESS_NEXTGEN_COMPRESSION`) + +--- + +## Complexity and Risk + +| Dimension | Rating | +|-----------|--------| +| Lines of code | ~80 (config + mapping branch) | +| Risk to existing users | Zero — flag defaults False | +| Testing requirement | Needs real NextGen AOSS endpoint | +| Reviewer effort | Low — clear pattern, well-scoped | + +--- + +## My Take + +This is a **well-written, ready-to-implement** feature request. The proposer clearly: +- Has a NextGen endpoint to test against +- Understands the mapping differences from AWS docs +- Followed the existing code patterns +- Considered and correctly rejected alternatives + +The "DRAFT" status is misleading — this is implementation-ready. The only reason to hold would be if AWS is planning a deeper refactor of the OpenSearch integration (e.g., auto-detection in a future version). If not, this should be greenlit. + +**If I were reviewing:** Approve the approach, request tests against a real NextGen endpoint, merge. + +--- + +© 2024-2026 Kanjani AI Research. diff --git a/docs/analysis/CHUNK-STORE-ANALYSIS.md b/docs/analysis/CHUNK-STORE-ANALYSIS.md new file mode 100644 index 000000000..953e32fc3 --- /dev/null +++ b/docs/analysis/CHUNK-STORE-ANALYSIS.md @@ -0,0 +1,183 @@ +# Chunk Text Externalization: Analysis and Path Forward + +**Reference:** [DRAFT] [FEATURE] Pluggable external chunk text store to reduce Neptune Analytics memory usage — [awslabs/graphrag-toolkit#324](https://github.com/awslabs/graphrag-toolkit/issues/324) + +## The Real Issue + +The issue proposes a `ChunkStore` abstraction to move chunk text out of Neptune graph memory. The analysis is correct, but there's a **simpler path** available via a Neptune Analytics feature that the issue doesn't consider. + +--- + +## Neptune Analytics Memory Model (from AWS docs) + +Key facts from AWS documentation: + +1. **1 m-NCU = 1 GB memory** — Neptune Analytics stores everything in-memory +2. **Pricing**: Fixed hourly cost per m-NCU. At scale, memory = money. +3. **`GraphStorageUsagePercent`** — CloudWatch metric showing memory pressure +4. **All node properties are in-memory** — even if never queried. Large text properties inflate `GraphSizeBytes` and reduce the effective working set for traversal queries. +5. **You cannot selectively evict properties** — if it's stored as a property, it's in memory. + +**Cost calculation (confirmed):** +- 5M chunks × 2KB text = 10GB of graph memory +- 10 additional m-NCUs × ~$0.07/hr = ~$50/month just for chunk text +- At the issue's pricing ($500/month for 10GB), the numbers align with larger m-NCU tiers + +--- + +## The `neptune.read()` Alternative + +**AWS added `neptune.read()` in 2024** — a CALL procedure that reads data from S3 at query time: + +```cypher +CALL neptune.read({ + source: "s3://bucket/chunks/chunk-{chunkId}.parquet", + format: "parquet" +}) +YIELD row +RETURN row.text AS content +``` + +This means Neptune Analytics can **natively fetch external data from S3 during query execution** without a custom ChunkStore layer. The retriever could: + +1. Store chunk text in S3 (Parquet or CSV) during indexing +2. Store only `chunkId` as a graph property (tiny — a few bytes) +3. At retrieval time, use `neptune.read()` to fetch text from S3 + +**However, there are limitations:** +- `neptune.read()` takes a single S3 object URI — not a prefix +- It reads the entire file and yields rows — no random access by key +- It's designed for batch analytics (join external data with graph), not single-record lookup + +**Verdict on `neptune.read()`:** Not suitable for the single-chunk-fetch pattern (4 read paths that need 1-20 chunk texts). It's for batch analytical queries. The ChunkStore approach is still needed. + +--- + +## What's Actually In Memory + +Looking at the `ChunkGraphBuilder` write path: + +```python +chunk_property_setters = [ + 'chunk.value = params.text' # ← This is the 2KB text +] +properties_c = { + 'chunk_id': chunk_id, # ~30 bytes + 'text': node.text # ~2000 bytes ← THE PROBLEM +} +``` + +Plus any `chunk_metadata` from external properties. For each of 5M chunks: +- `chunkId`: ~30 bytes (UUID or hash) +- `value` (text): ~2000 bytes +- Edges: EXTRACTED_FROM, MENTIONED_IN, BELONGS_TO (~100 bytes each) + +The text property is **98% of the per-chunk memory cost**. + +--- + +## The 4 Read Paths (Verified) + +| Location | What it does | When called | Batch size | +|----------|-------------|-------------|------------| +| `keyword_vss_provider.py:81` | `RETURN c.value AS content` | Keyword VSS search | 5-20 chunks | +| `statement_enhancement.py:103` | `node.node.metadata['chunk']['value']` | LLM synthesis | 1 chunk at a time | +| `traversal_based_base_retriever.py:156` | `value: NULL` (explicitly skipped!) | Main retriever | N/A — already external | +| `update_chunk_metadata.py:18` | `chunk.metadata.pop('value', None)` | Post-processing | Already in memory | + +**Key insight confirmed:** The main traversal retriever (`traversal_based_base_retriever.py:156`) already projects `value: NULL`. The architecture already treats chunk text as external. The memory is wasted. + +--- + +## Path Forward: Recommended Approach + +### Option A: S3 + Batch Fetch (Simplest, Highest ROI) + +**Write path:** +- Store chunk text as Parquet files in S3, partitioned by source document +- Store only `chunkId` in Neptune (drop `chunk.value`) +- Key pattern: `s3://{bucket}/chunk-text/{sourceId}/{chunkId}.txt` or batched Parquet + +**Read path (only 2 locations actually need text):** +- `keyword_vss_provider.py`: After getting `chunkId` list from graph, batch-fetch from S3 +- `statement_enhancement.py`: Single-fetch from S3 by `chunkId` + +**No change needed for:** +- `traversal_based_base_retriever.py` — already projects `value: NULL` +- `update_chunk_metadata.py` — would read from the same S3 source + +**Estimated complexity:** ~100 lines. No new abstraction needed for the first implementation. + +### Option B: ChunkStore Protocol (As Proposed in Issue) + +Full abstraction with pluggable backends. More complex but more extensible. + +**When to choose Option B over A:** +- If you need DynamoDB (sub-5ms latency) for real-time applications +- If you need Redis (sub-1ms) for high-throughput retrieval +- If multiple teams need different backends + +**When Option A is sufficient:** +- S3 GET latency (5-15ms) is negligible vs LLM synthesis (1-5s) +- Single deployment, single team +- Want the memory savings now without protocol design + +### Option C: Neptune Database (Not Analytics) + +If using Neptune Database (not Analytics), properties are on disk, not in memory. The issue is **specific to Neptune Analytics**. On Neptune DB: +- Properties are stored on SSD +- Memory is used for buffer cache (hot data) +- Large text properties only matter if they evict hot structural data from cache + +If the deployment is Neptune DB, this optimization has lower priority. + +--- + +## Recommendation + +**Start with Option A** (S3 batch fetch, ~100 lines) as a PR. It solves 95% of the problem: +- Remove `chunk.value` from graph properties → immediate 10GB memory savings +- Add S3 batch fetch in the 2 locations that need text +- Keep it simple: no factory pattern, no protocol, just S3 boto3 calls + +**Evolve to Option B** (ChunkStore protocol) only if: +- Multiple backend requirements emerge +- Latency requirements tighten below S3 capabilities +- The pattern needs to be reusable across other property types + +The issue's proposed design (Option B) is correct and well-architected, but it's premature for the first cut. Ship the savings first, abstract later. + +--- + +## Neptune Analytics Specific Guidance (from AWS Well-Architected) + +From the [Cost Optimization Pillar](https://docs.aws.amazon.com/prescriptive-guidance/latest/neptune-analytics-well-architected-framework/cost-optimization-pillar.html): + +> Monitor CloudWatch metrics... `GraphStorageUsagePercent`, `GraphSizeBytes`... to assess whether the provisioned capacity is appropriately sized. + +The recommended action is to **monitor `GraphSizeBytes` before and after** removing chunk text: +``` +Before: GraphSizeBytes includes 10GB of chunk.value text +After: GraphSizeBytes reduced by ~10GB → can downsize m-NCU +``` + +This directly reduces the hourly m-NCU cost. + +--- + +## Summary + +| Aspect | Assessment | +|--------|-----------| +| Issue valid? | Yes — chunk text in graph memory is wasteful at scale | +| Neptune Analytics specific? | Yes — DB mode stores on disk, less urgent | +| Proposed solution (ChunkStore)? | Over-engineered for first cut; correct for long-term | +| Recommended first step | S3 batch fetch (~100 lines), remove chunk.value from graph | +| `neptune.read()` applicable? | No — designed for batch analytics, not single-record lookup | +| Key metric to track | `GraphSizeBytes` and `GraphStorageUsagePercent` before/after | +| Latency impact | Negligible (15-30ms S3 vs 1-5s LLM synthesis) | +| Cost impact | ~$50-500/month savings depending on m-NCU tier | + +--- + +© 2024-2026 Kanjani AI Research. diff --git a/docs/analysis/CREATE-CONFIG-BUG-ANALYSIS.md b/docs/analysis/CREATE-CONFIG-BUG-ANALYSIS.md new file mode 100644 index 000000000..5127845ee --- /dev/null +++ b/docs/analysis/CREATE-CONFIG-BUG-ANALYSIS.md @@ -0,0 +1,157 @@ +# Neptune `create_config` TypeError: Analysis and Fix + +**Reference:** [BUG] Neptune create_config raises TypeError when read_timeout is passed — [awslabs/graphrag-toolkit#361](https://github.com/awslabs/graphrag-toolkit/issues/361) + +--- + +## The Bug + +```python +# neptune_graph_stores.py line 149-155 +return Config( + retries={'total_max_attempts': 1, 'mode': 'standard'}, + read_timeout=600, # ← hardcoded default + user_agent_appid=f'graphrag-lexical-graph-{toolkit_version}', + **config_args # ← user's config unpacked here +) +``` + +When a user passes `config='{"read_timeout": 5}'`: +- `config_args` = `{"read_timeout": 5}` +- The `**config_args` unpacking produces `read_timeout=5` +- But `read_timeout=600` is already a keyword argument +- Python raises: `TypeError: Config() got multiple values for keyword argument 'read_timeout'` + +**The same bug applies to any argument that's both hardcoded AND passed by the user:** +- `read_timeout` (demonstrated) +- `retries` (would also fail if user passes it) +- `user_agent_appid` (would also fail) + +--- + +## Root Cause + +The function uses `**config_args` to spread user config, but also hardcodes the same keys as positional kwargs. Python doesn't allow the same keyword to appear twice. + +The `setdefault` pattern on line 148 (`config_args.setdefault('max_pool_connections', ...)`) is the **correct** approach — it sets a default only if the user didn't provide one. But `read_timeout`, `retries`, and `user_agent_appid` don't use this pattern. + +--- + +## Fix + +**Option A: Use `setdefault` for all hardcoded values (minimal change, backward-compatible)** + +```python +def create_config(config: Optional[str] = None): + toolkit_version = 'unknown' + try: + toolkit_version = version('graphrag-lexical-graph') + except PackageNotFoundError: + pass + + config_args = {} + if config: + config_args = json.loads(config) + + # Defaults — user-provided values take precedence + config_args.setdefault('max_pool_connections', DEFAULT_MAX_POOL_CONNECTIONS) + config_args.setdefault('read_timeout', 600) + config_args.setdefault('retries', {'total_max_attempts': 1, 'mode': 'standard'}) + config_args.setdefault('user_agent_appid', f'graphrag-lexical-graph-{toolkit_version}') + + return Config(**config_args) +``` + +This is a 4-line diff that: +- Fixes the TypeError +- Allows users to override ANY botocore Config parameter +- Preserves all existing defaults when no config is provided +- Uses the same pattern already established for `max_pool_connections` + +**Option B: Pop known keys before spreading (more explicit)** + +```python +def create_config(config: Optional[str] = None): + ... + config_args = json.loads(config) if config else {} + config_args.setdefault('max_pool_connections', DEFAULT_MAX_POOL_CONNECTIONS) + + return Config( + retries=config_args.pop('retries', {'total_max_attempts': 1, 'mode': 'standard'}), + read_timeout=config_args.pop('read_timeout', 600), + user_agent_appid=config_args.pop('user_agent_appid', f'graphrag-lexical-graph-{toolkit_version}'), + **config_args # remaining user args pass through cleanly + ) +``` + +This is more explicit about which keys have defaults, but functionally identical. + +--- + +## Impact + +**Who's affected:** Anyone using Neptune behind: +- API Gateway (29s timeout) +- ALB (60s idle timeout) +- Lambda (15 min max, but usually configured shorter) +- Step Functions task timeouts +- Any service mesh with request deadlines + +The hardcoded 600s `read_timeout` means: +- If Neptune is slow or unresponsive, the client waits **10 minutes** before failing +- Behind a 30s API Gateway, the gateway cuts the connection but the SDK keeps the socket open +- The next request on that pooled connection may hang or get a stale response + +Users **need** to configure this, and currently can't. + +--- + +## Why This Is More Than a Config Bug + +The real issue is that `read_timeout=600` is an **opinionated default that doesn't match production deployment patterns**: + +| Context | Appropriate timeout | Current default | +|---------|--------------------:|----------------:| +| Interactive notebook (Jupyter) | 600s (fine) | 600s ✓ | +| API backend (API Gateway) | 20-25s | 600s ✗ | +| Lambda function | Function timeout - buffer | 600s ✗ | +| Batch indexing job | 300-600s | 600s ✓ | +| Real-time retrieval | 5-10s | 600s ✗ | + +The library is designed for notebooks (where 600s makes sense). Production deployments need configurability. + +--- + +## Suggested Path Forward + +1. **Fix the bug** (Option A — 4-line diff, non-breaking) +2. **Document the config parameter** in the docs-site configuration page +3. **Consider whether `read_timeout` should differ by operation type:** + - Indexing (long writes) → 600s default (current) + - Retrieval (fast reads) → 30s default + - This would be a separate enhancement, not part of the bug fix + +--- + +## Test + +```python +from graphrag_toolkit.lexical_graph.storage.graph.neptune_graph_stores import create_config +import json + +# Should not raise TypeError: +config = create_config(json.dumps({"read_timeout": 5})) +assert config.read_timeout == 5 + +# Default still works: +config = create_config() +assert config.read_timeout == 600 + +# Other overrides work: +config = create_config(json.dumps({"retries": {"total_max_attempts": 3, "mode": "adaptive"}})) +assert config.retries['total_max_attempts'] == 3 +``` + +--- + +© 2024-2026 Kanjani AI Research. diff --git a/docs/analysis/LOGGING-ECS-EKS-ANALYSIS.md b/docs/analysis/LOGGING-ECS-EKS-ANALYSIS.md new file mode 100644 index 000000000..1b4b310d0 --- /dev/null +++ b/docs/analysis/LOGGING-ECS-EKS-ANALYSIS.md @@ -0,0 +1,215 @@ +# Lexical-Graph Logging: File Handler Crashes on Read-Only Filesystems + +**Status:** Bug confirmed and fully characterized. No existing issue found. +**Severity:** High for containerized deployments (crashes on `set_logging_config()`) +**Affected:** Any ECS/EKS/Lambda/Fargate deployment with `readOnlyRootFilesystem: true` + +--- + +## Reproduction (Verified) + +```python +import os +os.chdir('/') # simulate read-only container CWD + +from graphrag_toolkit.lexical_graph import set_logging_config +set_logging_config('INFO') +# ValueError: Unable to configure handler 'file_handler' +# Caused by: OSError: [Errno 30] Read-only file system: '/output.log' +``` + +Every example notebook calls `set_logging_config('INFO')`. Every production deployment will call it. On read-only FS, the application crashes at startup. + +--- + +## Root Cause (Deep) + +`BASE_LOGGING_CONFIG` in `logging.py` defines a `file_handler`: + +```python +'handlers': { + 'stdout': { ... }, + 'file_handler': { + 'class': 'logging.FileHandler', + 'filename': 'output.log', # ← relative to CWD + 'mode': 'a', + } +}, +'loggers': {'': {'handlers': ['stdout'], 'level': logging.INFO}}, +# ^^^^^^^^ only stdout is active +``` + +The `file_handler` is **not assigned to any logger**. It appears dormant. But: + +**Python's `logging.config.dictConfig()` instantiates ALL handlers defined in the config dict, regardless of whether they're assigned to a logger.** This is by design — handlers can be referenced later or shared across loggers. + +When `dictConfig` tries to create `logging.FileHandler('output.log')`: +1. `FileHandler.__init__` calls `open('output.log', 'a')` +2. On read-only FS → `OSError: Read-only file system` +3. `dictConfig` wraps it → `ValueError: Unable to configure handler 'file_handler'` +4. **Entire logging configuration fails** +5. Application crashes + +--- + +## Why `LOG_OUTPUT_DIR` Doesn't Help + +The code has this in `config.py`: +```python +DEFAULT_LOG_OUTPUT_DIR = None # Log file directory (None = use filename as-is, set to /tmp for EKS) +``` + +And in `set_advanced_logging_config()`: +```python +if filename: + if GraphRAGConfig.log_output_dir and not os.path.isabs(filename): + filename = os.path.join(GraphRAGConfig.log_output_dir, filename) +``` + +**But this only applies when the user passes `filename` explicitly.** The `BASE_LOGGING_CONFIG` already has `'filename': 'output.log'` hardcoded — `LOG_OUTPUT_DIR` doesn't modify the base config. The `dictConfig` call processes the hardcoded path regardless. + +Setting `LOG_OUTPUT_DIR=/tmp` does NOT fix the crash. Verified: +``` +Workaround 3: Set LOG_OUTPUT_DIR=/tmp before calling + FAILS ✗ — LOG_OUTPUT_DIR doesn't help because BASE_LOGGING_CONFIG is already built +``` + +--- + +## Workarounds (Tested) + +| Workaround | Works? | Production-viable? | +|------------|--------|-------------------| +| Remove `file_handler` from config before dictConfig | ✓ | No — requires modifying internal constant | +| Add `delay=True` to `file_handler` | ✓ | No — requires modifying internal constant | +| Set `LOG_OUTPUT_DIR=/tmp` | ✗ | — | +| Monkey-patch `BASE_LOGGING_CONFIG` before calling | ✓ | Fragile — breaks on version updates | +| Ensure CWD is writable | ✓ | Violates security best practices | +| Don't call `set_logging_config()` | ✓ | Loses all logging configuration | + +**There is no clean user-facing workaround.** The only options are: +1. Don't use `readOnlyRootFilesystem` (security regression) +2. Monkey-patch the library's internal config (fragile) +3. Don't call `set_logging_config()` (lose logging) + +--- + +## Proper Fix Options + +### Fix A: Remove `file_handler` from BASE_LOGGING_CONFIG (Best) + +Only define it dynamically when `filename` is actually requested: + +```python +BASE_LOGGING_CONFIG = { + 'version': 1, + 'disable_existing_loggers': False, + 'filters': { ... }, + 'formatters': { ... }, + 'handlers': { + 'stdout': { ... } + # file_handler NOT here — added only when needed + }, + 'loggers': {'': {'handlers': ['stdout'], 'level': logging.INFO}}, +} + +def set_advanced_logging_config(..., filename=None): + config = copy.deepcopy(BASE_LOGGING_CONFIG) + ... + if filename: + if GraphRAGConfig.log_output_dir and not os.path.isabs(filename): + filename = os.path.join(GraphRAGConfig.log_output_dir, filename) + config['handlers']['file_handler'] = { + 'formatter': 'default', + 'class': 'logging.FileHandler', + 'filename': filename, + 'filters': ['moduleFilter'], + 'mode': 'a', + } + config['loggers']['']['handlers'].append('file_handler') + logging.config.dictConfig(config) +``` + +**Pros:** Clean, minimal change, no behavior change for users, fixes all deployment scenarios. +**Cons:** None. + +### Fix B: Add `delay=True` to file_handler (Partial) + +```python +'file_handler': { + 'class': 'logging.FileHandler', + 'filename': 'output.log', + 'delay': True, # ← defer file open until first emit + 'mode': 'a', +} +``` + +`delay=True` tells `FileHandler` to not open the file until the first log record is emitted to it. Since the handler is never assigned to a logger (unless `filename` is passed), it never opens the file. + +**Pros:** 1-line change. dictConfig succeeds. +**Cons:** +- File still gets opened if user passes `filename` pointing to read-only path +- The handler object exists in memory unnecessarily +- Doesn't solve the conceptual issue (why define a handler you never use?) +- If Python changes `dictConfig` behavior in future, may break again + +### Fix C: Conditional handler based on environment (Over-engineered) + +Check `os.access(os.getcwd(), os.W_OK)` and only include the handler if writable. + +**Cons:** Race condition. CWD can change. Overly clever. + +--- + +## Recommendation + +**Fix A is the correct solution.** It's 3 lines moved + 6 lines added. Zero behavior change. Zero risk. The handler should only exist when it's needed. + +Fix B is an acceptable quick patch if Fix A is rejected for some reason, but it leaves dead code in the config. + +--- + +## Why This Matters for Production + +AWS security best practices for ECS/EKS recommend `readOnlyRootFilesystem: true`: + +```yaml +# ECS Task Definition +containerDefinitions: + - readonlyRootFilesystem: true + ... + +# EKS Pod Security Context +securityContext: + readOnlyRootFilesystem: true +``` + +This is: +- Required by many compliance frameworks (CIS Benchmark, SOC 2) +- Default in AWS Fargate platform version 1.4+ +- Recommended in AWS Well-Architected Framework (Security Pillar) + +The lexical-graph library is **incompatible with AWS's own security recommendations** due to this bug. + +--- + +## Test Plan + +```python +import os +os.chdir('/') # read-only + +from graphrag_toolkit.lexical_graph import set_logging_config + +# After fix: should succeed +set_logging_config('INFO') +logging.getLogger('test').info('Works on read-only FS') + +# File logging still works when explicitly requested with writable path: +set_logging_config('INFO', filename='/tmp/app.log') +assert os.path.exists('/tmp/app.log') +``` + +--- + +© 2024-2026 Kanjani AI Research. diff --git a/examples/lexical-graph/cloudformation-templates/audit-deployment.sh b/examples/lexical-graph/cloudformation-templates/audit-deployment.sh new file mode 100755 index 000000000..a99b1c46a --- /dev/null +++ b/examples/lexical-graph/cloudformation-templates/audit-deployment.sh @@ -0,0 +1,1618 @@ +#!/usr/bin/env bash +# +# audit-deployment.sh - Comprehensive audit script for GraphRAG Toolkit CloudFormation deployments +# +# This script audits all resources deployed by graphrag-toolkit CloudFormation templates. +# It checks resource existence, status, configuration, and cost-relevant details. +# +# Supported template variants: +# - neptune-analytics (with opensearch, s3-vectors, or aurora-postgres) +# - neptune-db (with opensearch, s3-vectors, or aurora-postgres) +# - neptune-db-aurora-postgres-existing-vpc +# +# Usage: +# ./audit-deployment.sh [--stack-name NAME] [--region REGION] [--profile PROFILE] [--json] +# +# Options: +# --stack-name CloudFormation stack name (auto-detects if not provided) +# --region AWS region (default: us-east-1) +# --profile AWS CLI profile (optional) +# --json Output in JSON format for machine-readable consumption +# --help Show this help message +# +# Author: Generated for graphrag-toolkit +# Date: 2024 +# + +set -euo pipefail + +############################################################################### +# CONFIGURATION & GLOBALS +############################################################################### + +VERSION="1.1.0" +STACK_NAME="" +REGION="us-east-1" +PROFILE="" +JSON_OUTPUT=false +MANUAL_MODE=false +RESOURCE_CONFIG="" +AWS_CMD="aws" + +# Counters for summary +RESOURCES_FOUND=0 +RESOURCES_EXPECTED=0 +RESOURCES_HEALTHY=0 +RESOURCES_UNHEALTHY=0 +RESOURCES_UNKNOWN=0 + +# JSON accumulator (stored as a simple variable for bash 3 compat) +JSON_SECTION_CLOUDFORMATION="{}" + +############################################################################### +# COLOR DEFINITIONS +############################################################################### + +if [[ -t 1 ]] && [[ "${NO_COLOR:-}" != "1" ]]; then + RED='\033[0;31m' + GREEN='\033[0;32m' + YELLOW='\033[1;33m' + BLUE='\033[0;34m' + CYAN='\033[0;36m' + MAGENTA='\033[0;35m' + BOLD='\033[1m' + DIM='\033[2m' + RESET='\033[0m' +else + RED='' GREEN='' YELLOW='' BLUE='' CYAN='' MAGENTA='' BOLD='' DIM='' RESET='' +fi + +############################################################################### +# UTILITY FUNCTIONS +############################################################################### + +print_header() { + local title="$1" + if [[ "$JSON_OUTPUT" == "true" ]]; then return; fi + echo "" + echo -e "${BOLD}${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" + echo -e "${BOLD}${BLUE} $title${RESET}" + echo -e "${BOLD}${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" +} + +print_subheader() { + local title="$1" + if [[ "$JSON_OUTPUT" == "true" ]]; then return; fi + echo "" + echo -e " ${BOLD}${CYAN}▸ $title${RESET}" + echo -e " ${DIM}──────────────────────────────────────────────────────────${RESET}" +} + +print_status() { + local label="$1" + local value="$2" + local status="${3:-info}" # info, ok, warn, error + if [[ "$JSON_OUTPUT" == "true" ]]; then return; fi + + local color="" + local icon="" + case "$status" in + ok) color="$GREEN"; icon="✓" ;; + warn) color="$YELLOW"; icon="⚠" ;; + error) color="$RED"; icon="✗" ;; + info) color="$CYAN"; icon="ℹ" ;; + esac + printf " ${color}${icon}${RESET} %-35s %s\n" "$label:" "$value" +} + +print_table_row() { + local col1="$1" col2="$2" col3="${3:-}" col4="${4:-}" + if [[ "$JSON_OUTPUT" == "true" ]]; then return; fi + if [[ -n "$col4" ]]; then + printf " %-30s %-20s %-25s %s\n" "$col1" "$col2" "$col3" "$col4" + elif [[ -n "$col3" ]]; then + printf " %-30s %-25s %s\n" "$col1" "$col2" "$col3" + else + printf " %-35s %s\n" "$col1" "$col2" + fi +} + +print_table_header() { + if [[ "$JSON_OUTPUT" == "true" ]]; then return; fi + local cols=("$@") + local header="" + local separator="" + for col in "${cols[@]}"; do + header+=$(printf "%-30s " "$col") + separator+=$(printf "%-30s " "$(printf '%0.s─' $(seq 1 ${#col}))") + done + echo -e " ${BOLD}${header}${RESET}" + echo -e " ${DIM}${separator}${RESET}" +} + +status_color() { + local status="$1" + case "$status" in + *COMPLETE*|*AVAILABLE*|*ACTIVE*|*InService*|*running*|*available*) + echo -e "${GREEN}${status}${RESET}" ;; + *PROGRESS*|*UPDATING*|*CREATING*|*starting*|*pending*) + echo -e "${YELLOW}${status}${RESET}" ;; + *FAILED*|*DELETE*|*stopped*|*error*|*INACTIVE*) + echo -e "${RED}${status}${RESET}" ;; + *) + echo -e "${status}" ;; + esac +} + +# Execute AWS CLI command with proper profile/region +aws_cmd() { + local cmd="$AWS_CMD" + if [[ -n "$PROFILE" ]]; then + cmd+=" --profile $PROFILE" + fi + cmd+=" --region $REGION" + cmd+=" $*" + eval "$cmd" 2>/dev/null +} + +# Safe JSON extraction +json_get() { + local json="$1" + local query="$2" + echo "$json" | jq -r "$query // empty" 2>/dev/null || echo "" +} + +############################################################################### +# ARGUMENT PARSING +############################################################################### + +show_help() { + cat << EOF +${BOLD}GraphRAG Toolkit Deployment Audit Script v${VERSION}${RESET} + +Usage: $(basename "$0") [OPTIONS] + +Options: + --stack-name NAME CloudFormation stack name (auto-detects 'graphrag' stacks) + --region REGION AWS region (default: us-east-1) + --profile PROFILE AWS CLI profile name + --manual Scan for resources directly (no CloudFormation stack required) + --resources FILE JSON config file for manual mode (default: audit-resources.json) + --json Output in JSON format + --help Show this help message + +Examples: + $(basename "$0") + $(basename "$0") --stack-name my-graphrag-stack --region us-west-2 + $(basename "$0") --profile production --json + $(basename "$0") --manual --region us-east-1 + $(basename "$0") --manual --profile production + +EOF + exit 0 +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --stack-name) + STACK_NAME="$2" + shift 2 + ;; + --region) + REGION="$2" + shift 2 + ;; + --profile) + PROFILE="$2" + shift 2 + ;; + --manual) + MANUAL_MODE=true + shift + ;; + --resources) + RESOURCE_CONFIG="$2" + MANUAL_MODE=true + shift 2 + ;; + --json) + JSON_OUTPUT=true + shift + ;; + --help|-h) + show_help + ;; + *) + echo -e "${RED}Error: Unknown option: $1${RESET}" >&2 + echo "Use --help for usage information." >&2 + exit 1 + ;; + esac + done +} + + +############################################################################### +# STACK DETECTION +############################################################################### + +detect_stack() { + if [[ -n "$STACK_NAME" ]]; then + return 0 + fi + + if [[ "$JSON_OUTPUT" != "true" ]]; then + echo -e "${YELLOW}No --stack-name provided. Searching for GraphRAG stacks...${RESET}" + fi + + local stacks + stacks=$(aws_cmd cloudformation list-stacks \ + --stack-status-filter CREATE_COMPLETE UPDATE_COMPLETE UPDATE_ROLLBACK_COMPLETE \ + --query "StackSummaries[?contains(StackName, 'graphrag') || contains(StackName, 'GraphRAG')].{Name:StackName,Status:StackStatus,Created:CreationTime}" \ + --output json 2>/dev/null || echo "[]") + + local count + count=$(echo "$stacks" | jq 'length' 2>/dev/null || echo "0") + + if [[ "$count" -eq 0 ]]; then + echo -e "${RED}Error: No GraphRAG stacks found in ${REGION}.${RESET}" >&2 + echo "Use --stack-name to specify a stack explicitly." >&2 + exit 1 + elif [[ "$count" -eq 1 ]]; then + STACK_NAME=$(echo "$stacks" | jq -r '.[0].Name') + if [[ "$JSON_OUTPUT" != "true" ]]; then + echo -e "${GREEN}Auto-detected stack: ${BOLD}${STACK_NAME}${RESET}" + fi + else + if [[ "$JSON_OUTPUT" != "true" ]]; then + echo -e "${YELLOW}Multiple GraphRAG stacks found:${RESET}" + echo "$stacks" | jq -r '.[] | " - \(.Name) [\(.Status)] (created: \(.Created))"' + echo "" + echo -e "${YELLOW}Please specify one with --stack-name${RESET}" + fi + exit 1 + fi +} + +############################################################################### +# MANUAL RESOURCE DISCOVERY (no CloudFormation stack required) +############################################################################### + +discover_resources_manual() { + # In manual mode, we scan AWS directly by RESOURCE TYPE. + # Reads resource types from audit-resources.json config file. + + # Locate config file (same directory as script) + local script_dir + script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + local config_file="${RESOURCE_CONFIG:-${script_dir}/audit-resources.json}" + + if [[ ! -f "$config_file" ]]; then + echo -e "${RED}Error: Resource config not found: ${config_file}${RESET}" >&2 + echo "Create audit-resources.json or specify with --resources " >&2 + exit 1 + fi + + if [[ "$JSON_OUTPUT" != "true" ]]; then + print_header "MANUAL RESOURCE DISCOVERY" + echo -e " ${YELLOW}Config: ${config_file}${RESET}" + echo -e " ${YELLOW}Scanning region ${REGION} for enabled resource types...${RESET}" + echo "" + fi + + local resources="[]" + + # Read enabled resource types from config + local enabled_types + enabled_types=$(jq -c '.resource_types[] | select(.enabled == true)' "$config_file" 2>/dev/null) + + if [[ -z "$enabled_types" ]]; then + echo -e "${RED}Error: No enabled resource types in ${config_file}${RESET}" >&2 + exit 1 + fi + + # Iterate over each enabled resource type + while IFS= read -r entry; do + [[ -z "$entry" ]] && continue + + local label service operation jq_extract logical_id cfn_type rtype + label=$(echo "$entry" | jq -r '.label') + service=$(echo "$entry" | jq -r '.service') + operation=$(echo "$entry" | jq -r '.operation') + jq_extract=$(echo "$entry" | jq -r '.jq_extract') + logical_id=$(echo "$entry" | jq -r '.logical_id') + cfn_type=$(echo "$entry" | jq -r '.cfn_type') + rtype=$(echo "$entry" | jq -r '.type') + + if [[ "$JSON_OUTPUT" != "true" ]]; then + printf " Scanning %-40s\r" "${label}..." + fi + + # Execute the AWS CLI call + local result + local cmd="$AWS_CMD" + [[ -n "$PROFILE" ]] && cmd+=" --profile $PROFILE" + cmd+=" --region $REGION" + cmd+=" $service $operation --output json" + result=$(eval "$cmd" 2>/dev/null || echo "{}") + + # Extract resource IDs using the jq expression from config + while IFS= read -r rid; do + [[ -z "$rid" ]] && continue + resources=$(echo "$resources" | jq --arg id "$rid" --arg lid "$logical_id" --arg rt "$cfn_type" \ + '. += [{"LogicalResourceId":$lid,"PhysicalResourceId":$id,"ResourceType":$rt,"ResourceStatus":"DISCOVERED"}]') + done < <(echo "$result" | jq -r "$jq_extract" 2>/dev/null) + + # Special handling: VPC also discovers subnets, SGs, NAT/IGW, endpoints + if [[ "$rtype" == "vpc" ]]; then + # For each discovered VPC, get associated networking resources + while IFS= read -r vid; do + [[ -z "$vid" ]] && continue + + # Subnets + local subnets + subnets=$(aws_cmd ec2 describe-subnets --filters "Name=vpc-id,Values=$vid" --output json 2>/dev/null || echo '{"Subnets":[]}') + while IFS= read -r sid; do + [[ -z "$sid" ]] && continue + resources=$(echo "$resources" | jq --arg id "$sid" \ + '. += [{"LogicalResourceId":"Subnet","PhysicalResourceId":$id,"ResourceType":"AWS::EC2::Subnet","ResourceStatus":"DISCOVERED"}]') + done < <(echo "$subnets" | jq -r '.Subnets[].SubnetId' 2>/dev/null) + + # Security Groups (non-default) + local sgs + sgs=$(aws_cmd ec2 describe-security-groups --filters "Name=vpc-id,Values=$vid" --output json 2>/dev/null || echo '{"SecurityGroups":[]}') + while IFS= read -r sg_id; do + [[ -z "$sg_id" ]] && continue + resources=$(echo "$resources" | jq --arg id "$sg_id" \ + '. += [{"LogicalResourceId":"SecurityGroup","PhysicalResourceId":$id,"ResourceType":"AWS::EC2::SecurityGroup","ResourceStatus":"DISCOVERED"}]') + done < <(echo "$sgs" | jq -r '.SecurityGroups[] | select(.GroupName != "default") | .GroupId' 2>/dev/null) + + # NAT Gateways + local nats + nats=$(aws_cmd ec2 describe-nat-gateways --filter "Name=vpc-id,Values=$vid" "Name=state,Values=available" --output json 2>/dev/null || echo '{"NatGateways":[]}') + while IFS= read -r nid; do + [[ -z "$nid" ]] && continue + resources=$(echo "$resources" | jq --arg id "$nid" \ + '. += [{"LogicalResourceId":"NATGW","PhysicalResourceId":$id,"ResourceType":"AWS::EC2::NatGateway","ResourceStatus":"DISCOVERED"}]') + done < <(echo "$nats" | jq -r '.NatGateways[].NatGatewayId' 2>/dev/null) + + # Internet Gateways + local igws + igws=$(aws_cmd ec2 describe-internet-gateways --filters "Name=attachment.vpc-id,Values=$vid" --output json 2>/dev/null || echo '{"InternetGateways":[]}') + while IFS= read -r igid; do + [[ -z "$igid" ]] && continue + resources=$(echo "$resources" | jq --arg id "$igid" \ + '. += [{"LogicalResourceId":"IGW","PhysicalResourceId":$id,"ResourceType":"AWS::EC2::InternetGateway","ResourceStatus":"DISCOVERED"}]') + done < <(echo "$igws" | jq -r '.InternetGateways[].InternetGatewayId' 2>/dev/null) + + # VPC Endpoints + local vpces + vpces=$(aws_cmd ec2 describe-vpc-endpoints --filters "Name=vpc-id,Values=$vid" --output json 2>/dev/null || echo '{"VpcEndpoints":[]}') + while IFS= read -r eid; do + [[ -z "$eid" ]] && continue + resources=$(echo "$resources" | jq --arg id "$eid" \ + '. += [{"LogicalResourceId":"VpcEndpoint","PhysicalResourceId":$id,"ResourceType":"AWS::EC2::VPCEndpoint","ResourceStatus":"DISCOVERED"}]') + done < <(echo "$vpces" | jq -r '.VpcEndpoints[].VpcEndpointId' 2>/dev/null) + + done < <(echo "$result" | jq -r "$jq_extract" 2>/dev/null) + fi + + done <<< "$enabled_types" + + # ─── Build synthetic STACK_RESOURCES ─── + STACK_RESOURCES=$(echo "$resources" | jq '{StackResourceSummaries: .}') + + local total + total=$(echo "$resources" | jq 'length') + + if [[ "$JSON_OUTPUT" != "true" ]]; then + printf " %-50s\r" "" + echo "" + echo -e " ${GREEN}✓${RESET} Discovery complete: ${BOLD}${total} resources${RESET} found" + echo "" + if [[ "$total" -gt 0 ]]; then + print_subheader "Discovered Resources Summary" + echo "" + echo "$resources" | jq -r 'group_by(.ResourceType) | .[] | "\(.[0].ResourceType)|\(length)"' 2>/dev/null | sort | \ + while IFS='|' read -r rtype rcount; do + printf " %-55s %s\n" "$rtype" "${rcount} found" + done + echo "" + fi + fi + + # Set stack name for display purposes + STACK_NAME="[MANUAL SCAN]" +} + + +############################################################################### +# CLOUDFORMATION STACK AUDIT +############################################################################### + +audit_cloudformation() { + print_header "CLOUDFORMATION STACK" + + local stack_info + stack_info=$(aws_cmd cloudformation describe-stacks --stack-name "$STACK_NAME" --output json 2>/dev/null || echo "") + + if [[ -z "$stack_info" || "$stack_info" == "" ]]; then + print_status "Stack" "NOT FOUND" "error" + ((RESOURCES_UNHEALTHY++)) || true + return + fi + + local stack=$(echo "$stack_info" | jq '.Stacks[0]') + local status=$(json_get "$stack" '.StackStatus') + local created=$(json_get "$stack" '.CreationTime') + local updated=$(json_get "$stack" '.LastUpdatedTime') + local description=$(json_get "$stack" '.Description') + local stack_id=$(json_get "$stack" '.StackId') + + ((RESOURCES_FOUND++)) || true + ((RESOURCES_EXPECTED++)) || true + + if [[ "$status" == *"COMPLETE"* && "$status" != *"DELETE"* ]]; then + ((RESOURCES_HEALTHY++)) || true + print_status "Stack Name" "$STACK_NAME" "ok" + else + ((RESOURCES_UNHEALTHY++)) || true + print_status "Stack Name" "$STACK_NAME" "error" + fi + + print_status "Status" "$(echo -e "$(status_color "$status")")" "info" + print_status "Created" "$created" "info" + [[ -n "$updated" ]] && print_status "Last Updated" "$updated" "info" + [[ -n "$description" ]] && print_status "Description" "$description" "info" + + # Stack outputs + print_subheader "Stack Outputs" + local outputs + outputs=$(echo "$stack" | jq -r '.Outputs[]? | "\(.OutputKey)|\(.OutputValue)"' 2>/dev/null || echo "") + + if [[ -n "$outputs" ]]; then + print_table_header "Output Key" "Value" + while IFS='|' read -r key value; do + # Truncate long values + if [[ ${#value} -gt 60 ]]; then + value="${value:0:57}..." + fi + print_table_row "$key" "$value" + done <<< "$outputs" + else + print_status "Outputs" "None" "warn" + fi + + # Stack parameters + print_subheader "Stack Parameters" + local params + params=$(echo "$stack" | jq -r '.Parameters[]? | "\(.ParameterKey)|\(.ParameterValue)"' 2>/dev/null || echo "") + + if [[ -n "$params" ]]; then + print_table_header "Parameter" "Value" + while IFS='|' read -r key value; do + if [[ ${#value} -gt 60 ]]; then + value="${value:0:57}..." + fi + print_table_row "$key" "$value" + done <<< "$params" + fi + + # Detect stack drift + print_subheader "Drift Detection" + local drift_status + drift_status=$(aws_cmd cloudformation detect-stack-drift --stack-name "$STACK_NAME" --output json 2>/dev/null || echo "") + + if [[ -n "$drift_status" ]]; then + local drift_id=$(json_get "$drift_status" '.StackDriftDetectionId') + print_status "Drift Detection" "Initiated (ID: ${drift_id:0:20}...)" "info" + + # Wait briefly for drift results + sleep 2 + local drift_result + drift_result=$(aws_cmd cloudformation describe-stack-drift-detection-status \ + --stack-drift-detection-id "$drift_id" --output json 2>/dev/null || echo "") + + if [[ -n "$drift_result" ]]; then + local drift_stat=$(json_get "$drift_result" '.StackDriftStatus') + local detection_status=$(json_get "$drift_result" '.DetectionStatus') + + if [[ "$detection_status" == "DETECTION_COMPLETE" ]]; then + case "$drift_stat" in + IN_SYNC) print_status "Drift Status" "IN SYNC" "ok" ;; + DRIFTED) print_status "Drift Status" "DRIFTED" "warn" ;; + *) print_status "Drift Status" "$drift_stat" "info" ;; + esac + else + print_status "Drift Status" "Detection in progress ($detection_status)" "info" + fi + fi + else + print_status "Drift Detection" "Unable to initiate" "warn" + fi + + # Store for JSON + JSON_SECTION_CLOUDFORMATION=$(cat </dev/null || echo "{}") +} + + +############################################################################### +# VPC & NETWORKING AUDIT +############################################################################### + +audit_networking() { + print_header "VPC & NETWORKING" + + local vpc_id="" + local vpc_json="[]" + + # Try to get VPC from stack outputs or resources + vpc_id=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="VPC") | .PhysicalResourceId' 2>/dev/null || echo "") + + # VPC + print_subheader "VPC" + ((RESOURCES_EXPECTED++)) || true + if [[ -n "$vpc_id" ]]; then + ((RESOURCES_FOUND++)) || true + local vpc_info + vpc_info=$(aws_cmd ec2 describe-vpcs --vpc-ids "$vpc_id" --output json 2>/dev/null || echo "") + + if [[ -n "$vpc_info" && $(echo "$vpc_info" | jq '.Vpcs | length') -gt 0 ]]; then + ((RESOURCES_HEALTHY++)) || true + local cidr=$(echo "$vpc_info" | jq -r '.Vpcs[0].CidrBlock') + local state=$(echo "$vpc_info" | jq -r '.Vpcs[0].State') + local name=$(echo "$vpc_info" | jq -r '.Vpcs[0].Tags[]? | select(.Key=="Name") | .Value // "unnamed"' 2>/dev/null || echo "unnamed") + + print_status "VPC ID" "$vpc_id" "ok" + print_status "State" "$state" "ok" + print_status "CIDR Block" "$cidr" "info" + print_status "Name" "$name" "info" + else + ((RESOURCES_UNHEALTHY++)) || true + print_status "VPC" "$vpc_id (not accessible)" "error" + fi + else + print_status "VPC" "Not created by stack (existing-vpc template or analytics-only)" "info" + fi + + # Subnets + print_subheader "Subnets" + local subnet_ids=() + while IFS= read -r sid; do + [[ -n "$sid" ]] && subnet_ids+=("$sid") + done < <(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId | startswith("Subnet")) | .PhysicalResourceId' 2>/dev/null) + + if [[ ${#subnet_ids[@]} -gt 0 ]]; then + print_table_header "Subnet ID" "AZ" "CIDR" "Available IPs" + for subnet_id in "${subnet_ids[@]}"; do + ((RESOURCES_EXPECTED++)) || true + ((RESOURCES_FOUND++)) || true + local subnet_info + subnet_info=$(aws_cmd ec2 describe-subnets --subnet-ids "$subnet_id" --output json 2>/dev/null || echo "") + if [[ -n "$subnet_info" && $(echo "$subnet_info" | jq '.Subnets | length') -gt 0 ]]; then + ((RESOURCES_HEALTHY++)) || true + local az=$(echo "$subnet_info" | jq -r '.Subnets[0].AvailabilityZone') + local cidr=$(echo "$subnet_info" | jq -r '.Subnets[0].CidrBlock') + local avail_ips=$(echo "$subnet_info" | jq -r '.Subnets[0].AvailableIpAddressCount') + print_table_row "$subnet_id" "$az" "$cidr" "$avail_ips" + fi + done + else + print_status "Subnets" "Not created by this stack" "info" + fi + + # Security Groups + print_subheader "Security Groups" + local sg_ids=() + while IFS= read -r line; do + [[ -n "$line" ]] && sg_ids+=("$line") + done < <(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.ResourceType=="AWS::EC2::SecurityGroup") | "\(.LogicalResourceId)|\(.PhysicalResourceId)"' 2>/dev/null) + + if [[ ${#sg_ids[@]} -gt 0 ]]; then + print_table_header "Logical ID" "Group ID" "Ingress Rules" + for sg_entry in "${sg_ids[@]}"; do + IFS='|' read -r logical_id sg_id <<< "$sg_entry" + ((RESOURCES_EXPECTED++)) || true + ((RESOURCES_FOUND++)) || true + local sg_info + sg_info=$(aws_cmd ec2 describe-security-groups --group-ids "$sg_id" --output json 2>/dev/null || echo "") + if [[ -n "$sg_info" && $(echo "$sg_info" | jq '.SecurityGroups | length') -gt 0 ]]; then + ((RESOURCES_HEALTHY++)) || true + local rule_count=$(echo "$sg_info" | jq '.SecurityGroups[0].IpPermissions | length') + print_table_row "$logical_id" "$sg_id" "$rule_count rules" + fi + done + else + print_status "Security Groups" "None found in stack" "info" + fi + + # Internet Gateway + print_subheader "Internet & NAT Gateways" + local igw_id + igw_id=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="IGW") | .PhysicalResourceId' 2>/dev/null || echo "") + + if [[ -n "$igw_id" ]]; then + ((RESOURCES_EXPECTED++)) || true + ((RESOURCES_FOUND++)) || true + ((RESOURCES_HEALTHY++)) || true + print_status "Internet Gateway" "$igw_id" "ok" + else + print_status "Internet Gateway" "Not in stack" "info" + fi + + # NAT Gateway + local nat_id + nat_id=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="NATGW") | .PhysicalResourceId' 2>/dev/null || echo "") + + if [[ -n "$nat_id" ]]; then + ((RESOURCES_EXPECTED++)) || true + ((RESOURCES_FOUND++)) || true + local nat_info + nat_info=$(aws_cmd ec2 describe-nat-gateways --nat-gateway-ids "$nat_id" --output json 2>/dev/null || echo "") + if [[ -n "$nat_info" ]]; then + local nat_state=$(echo "$nat_info" | jq -r '.NatGateways[0].State') + if [[ "$nat_state" == "available" ]]; then + ((RESOURCES_HEALTHY++)) || true + print_status "NAT Gateway" "$nat_id ($nat_state)" "ok" + else + ((RESOURCES_UNHEALTHY++)) || true + print_status "NAT Gateway" "$nat_id ($nat_state)" "warn" + fi + print_status " Cost Note" "NAT Gateway: ~\$0.045/hr + data processing" "warn" + fi + else + print_status "NAT Gateway" "Not in stack" "info" + fi + + # Elastic IP + local eip_id + eip_id=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="ElasticIP1") | .PhysicalResourceId' 2>/dev/null || echo "") + + if [[ -n "$eip_id" ]]; then + ((RESOURCES_EXPECTED++)) || true + ((RESOURCES_FOUND++)) || true + ((RESOURCES_HEALTHY++)) || true + print_status "Elastic IP" "$eip_id" "ok" + print_status " Cost Note" "EIP: \$3.65/month if unattached" "info" + fi + + # VPC Endpoints + print_subheader "VPC Endpoints" + local vpce_ids=() + while IFS= read -r line; do + [[ -n "$line" ]] && vpce_ids+=("$line") + done < <(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.ResourceType | contains("VpcEndpoint")) | "\(.LogicalResourceId)|\(.PhysicalResourceId)"' 2>/dev/null) + + if [[ ${#vpce_ids[@]} -gt 0 ]]; then + for vpce_entry in "${vpce_ids[@]}"; do + IFS='|' read -r logical_id vpce_id <<< "$vpce_entry" + ((RESOURCES_EXPECTED++)) || true + ((RESOURCES_FOUND++)) || true + ((RESOURCES_HEALTHY++)) || true + print_status "$logical_id" "$vpce_id" "ok" + done + else + print_status "VPC Endpoints" "None in stack" "info" + fi +} + + +############################################################################### +# NEPTUNE DB AUDIT +############################################################################### + +audit_neptune_db() { + local cluster_id + cluster_id=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="NeptuneDBCluster") | .PhysicalResourceId' 2>/dev/null || echo "") + + if [[ -z "$cluster_id" ]]; then + return + fi + + print_header "NEPTUNE DB CLUSTER" + ((RESOURCES_EXPECTED++)) || true + + local cluster_info + cluster_info=$(aws_cmd neptune describe-db-clusters --db-cluster-identifier "$cluster_id" --output json 2>/dev/null || echo "") + + if [[ -z "$cluster_info" || $(echo "$cluster_info" | jq '.DBClusters | length') -eq 0 ]]; then + print_status "Neptune Cluster" "$cluster_id - NOT FOUND" "error" + ((RESOURCES_UNHEALTHY++)) || true + return + fi + + ((RESOURCES_FOUND++)) || true + local cluster=$(echo "$cluster_info" | jq '.DBClusters[0]') + local status=$(json_get "$cluster" '.Status') + local endpoint=$(json_get "$cluster" '.Endpoint') + local reader_endpoint=$(json_get "$cluster" '.ReaderEndpoint') + local engine_version=$(json_get "$cluster" '.EngineVersion') + local port=$(json_get "$cluster" '.Port') + local storage=$(json_get "$cluster" '.AllocatedStorage') + local serverless_config=$(echo "$cluster" | jq '.ServerlessV2ScalingConfiguration // empty' 2>/dev/null) + + if [[ "$status" == "available" ]]; then + ((RESOURCES_HEALTHY++)) || true + print_status "Cluster ID" "$cluster_id" "ok" + else + ((RESOURCES_UNHEALTHY++)) || true + print_status "Cluster ID" "$cluster_id" "warn" + fi + + print_status "Status" "$(echo -e "$(status_color "$status")")" "info" + print_status "Endpoint" "$endpoint" "info" + print_status "Reader Endpoint" "$reader_endpoint" "info" + print_status "Engine Version" "$engine_version" "info" + print_status "Port" "$port" "info" + + if [[ -n "$serverless_config" ]]; then + local min_ncu=$(json_get "$serverless_config" '.MinCapacity') + local max_ncu=$(json_get "$serverless_config" '.MaxCapacity') + print_status "Serverless Mode" "Min: ${min_ncu} NCU / Max: ${max_ncu} NCU" "info" + print_status " Cost Note" "Neptune Serverless: \$0.1028/NCU-hour" "warn" + fi + + # Neptune DB Instances + print_subheader "Neptune DB Instances" + local instance_id + instance_id=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="NeptuneDBInstance") | .PhysicalResourceId' 2>/dev/null || echo "") + + if [[ -n "$instance_id" ]]; then + ((RESOURCES_EXPECTED++)) || true + local inst_info + inst_info=$(aws_cmd neptune describe-db-instances --db-instance-identifier "$instance_id" --output json 2>/dev/null || echo "") + + if [[ -n "$inst_info" && $(echo "$inst_info" | jq '.DBInstances | length') -gt 0 ]]; then + ((RESOURCES_FOUND++)) || true + local inst=$(echo "$inst_info" | jq '.DBInstances[0]') + local inst_status=$(json_get "$inst" '.DBInstanceStatus') + local inst_class=$(json_get "$inst" '.DBInstanceClass') + local inst_az=$(json_get "$inst" '.AvailabilityZone') + + if [[ "$inst_status" == "available" ]]; then + ((RESOURCES_HEALTHY++)) || true + else + ((RESOURCES_UNHEALTHY++)) || true + fi + + print_status "Instance ID" "$instance_id" "ok" + print_status "Status" "$(echo -e "$(status_color "$inst_status")")" "info" + print_status "Instance Class" "$inst_class" "info" + print_status "AZ" "$inst_az" "info" + + if [[ "$inst_class" == "db.serverless" ]]; then + print_status " Cost Note" "Serverless - pay per NCU-hour" "info" + else + print_status " Cost Note" "Instance $inst_class running 24/7" "warn" + fi + fi + fi + + # Parameter Groups + local param_group + param_group=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="NeptuneDBClusterParameterGroup") | .PhysicalResourceId' 2>/dev/null || echo "") + if [[ -n "$param_group" ]]; then + ((RESOURCES_EXPECTED++)) || true + ((RESOURCES_FOUND++)) || true + ((RESOURCES_HEALTHY++)) || true + print_status "Cluster Param Group" "$param_group" "ok" + fi + + local db_param_group + db_param_group=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="NeptuneDBParameterGroup") | .PhysicalResourceId' 2>/dev/null || echo "") + if [[ -n "$db_param_group" ]]; then + ((RESOURCES_EXPECTED++)) || true + ((RESOURCES_FOUND++)) || true + ((RESOURCES_HEALTHY++)) || true + print_status "DB Param Group" "$db_param_group" "ok" + fi + + # Subnet Group + local subnet_group + subnet_group=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="NeptuneDBSubnetGroup") | .PhysicalResourceId' 2>/dev/null || echo "") + if [[ -n "$subnet_group" ]]; then + ((RESOURCES_EXPECTED++)) || true + ((RESOURCES_FOUND++)) || true + ((RESOURCES_HEALTHY++)) || true + print_status "Subnet Group" "$subnet_group" "ok" + fi +} + +############################################################################### +# NEPTUNE ANALYTICS AUDIT +############################################################################### + +audit_neptune_analytics() { + local graph_id + graph_id=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="Graph") | .PhysicalResourceId' 2>/dev/null || echo "") + + if [[ -z "$graph_id" ]]; then + return + fi + + print_header "NEPTUNE ANALYTICS GRAPH" + ((RESOURCES_EXPECTED++)) || true + + local graph_info + graph_info=$(aws_cmd neptune-graph get-graph --graph-identifier "$graph_id" --output json 2>/dev/null || echo "") + + if [[ -z "$graph_info" ]]; then + print_status "Neptune Graph" "$graph_id - NOT FOUND" "error" + ((RESOURCES_UNHEALTHY++)) || true + return + fi + + ((RESOURCES_FOUND++)) || true + local status=$(json_get "$graph_info" '.status') + local name=$(json_get "$graph_info" '.name') + local memory=$(json_get "$graph_info" '.provisionedMemory') + local endpoint=$(json_get "$graph_info" '.endpoint') + local vector_search=$(json_get "$graph_info" '.vectorSearchConfiguration.dimension') + local public_access=$(json_get "$graph_info" '.publicConnectivity') + + if [[ "$status" == "AVAILABLE" ]]; then + ((RESOURCES_HEALTHY++)) || true + print_status "Graph Name" "$name" "ok" + else + ((RESOURCES_UNHEALTHY++)) || true + print_status "Graph Name" "$name" "warn" + fi + + print_status "Graph ID" "$graph_id" "info" + print_status "Status" "$(echo -e "$(status_color "$status")")" "info" + print_status "Provisioned Memory" "${memory} GB" "info" + print_status "Endpoint" "$endpoint" "info" + [[ -n "$vector_search" ]] && print_status "Vector Dimensions" "$vector_search" "info" + print_status "Public Access" "$public_access" "info" + print_status " Cost Note" "Neptune Analytics: ~\$0.11/GB-hour (${memory}GB = ~\$$(echo "$memory * 0.11" | bc 2>/dev/null || echo "N/A")/hr)" "warn" +} + + +############################################################################### +# OPENSEARCH SERVERLESS AUDIT +############################################################################### + +audit_opensearch() { + local collection_id + collection_id=$(echo "$STACK_RESOURCES" | jq -r '[.StackResourceSummaries[]? | select(.LogicalResourceId=="OpenSearchServerless") | .PhysicalResourceId] | first // empty' 2>/dev/null || echo "") + + if [[ -z "$collection_id" ]]; then + return + fi + + print_header "OPENSEARCH SERVERLESS" + + # In manual mode, there may be multiple collections — audit each one + local all_collection_ids=() + while IFS= read -r cid; do + [[ -n "$cid" ]] && all_collection_ids+=("$cid") + done < <(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="OpenSearchServerless") | .PhysicalResourceId' 2>/dev/null) + + for collection_id in "${all_collection_ids[@]+"${all_collection_ids[@]}"}"; do + [[ -z "$collection_id" ]] && continue + ((RESOURCES_EXPECTED++)) || true + + # Get collection details + local collection_info + collection_info=$(aws_cmd opensearchserverless batch-get-collection --ids "$collection_id" --output json 2>/dev/null || echo "") + + if [[ -z "$collection_info" || $(echo "$collection_info" | jq '.collectionDetails | length') -eq 0 ]]; then + print_status "Collection" "$collection_id - NOT FOUND" "error" + ((RESOURCES_UNHEALTHY++)) || true + continue + fi + + ((RESOURCES_FOUND++)) || true + local collection=$(echo "$collection_info" | jq '.collectionDetails[0]') + local status=$(json_get "$collection" '.status') + local name=$(json_get "$collection" '.name') + local coll_type=$(json_get "$collection" '.type') + local endpoint=$(json_get "$collection" '.collectionEndpoint') + local dashboard=$(json_get "$collection" '.dashboardEndpoint') + local arn=$(json_get "$collection" '.arn') + + if [[ "$status" == "ACTIVE" ]]; then + ((RESOURCES_HEALTHY++)) || true + print_status "Collection Name" "$name" "ok" + else + ((RESOURCES_UNHEALTHY++)) || true + print_status "Collection Name" "$name" "warn" + fi + + print_status "Collection ID" "$collection_id" "info" + print_status "Status" "$(echo -e "$(status_color "$status")")" "info" + print_status "Type" "$coll_type" "info" + print_status "Endpoint" "$endpoint" "info" + print_status "Dashboard" "$dashboard" "info" + print_status " Cost Note" "OpenSearch Serverless: ~\$0.24/OCU-hour (min 2 OCUs indexing + 2 OCUs search)" "warn" + echo "" + done + + # Security Policies + print_subheader "Security & Access Policies" + local policy_ids=() + while IFS= read -r line; do + [[ -n "$line" ]] && policy_ids+=("$line") + done < <(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.ResourceType | contains("OpenSearchServerless")) | select(.LogicalResourceId != "OpenSearchServerless") | "\(.LogicalResourceId)|\(.PhysicalResourceId)"' 2>/dev/null) + + for policy_entry in "${policy_ids[@]+"${policy_ids[@]}"}"; do + IFS='|' read -r logical_id phys_id <<< "$policy_entry" + ((RESOURCES_EXPECTED++)) || true + ((RESOURCES_FOUND++)) || true + ((RESOURCES_HEALTHY++)) || true + print_status "$logical_id" "${phys_id:0:50}" "ok" + done + + # VPC Endpoint for OpenSearch + local vpce_id + vpce_id=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="OpenSearchServerlessVpcEndpoint") | .PhysicalResourceId' 2>/dev/null || echo "") + if [[ -n "$vpce_id" ]]; then + ((RESOURCES_EXPECTED++)) || true + ((RESOURCES_FOUND++)) || true + ((RESOURCES_HEALTHY++)) || true + print_status "VPC Endpoint" "$vpce_id" "ok" + fi +} + +############################################################################### +# AURORA POSTGRESQL AUDIT +############################################################################### + +audit_aurora_postgres() { + local cluster_id + cluster_id=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="PostgresCluster") | .PhysicalResourceId' 2>/dev/null || echo "") + + if [[ -z "$cluster_id" ]]; then + return + fi + + print_header "AURORA POSTGRESQL" + ((RESOURCES_EXPECTED++)) || true + + local cluster_info + cluster_info=$(aws_cmd rds describe-db-clusters --db-cluster-identifier "$cluster_id" --output json 2>/dev/null || echo "") + + if [[ -z "$cluster_info" || $(echo "$cluster_info" | jq '.DBClusters | length') -eq 0 ]]; then + print_status "Aurora Cluster" "$cluster_id - NOT FOUND" "error" + ((RESOURCES_UNHEALTHY++)) || true + return + fi + + ((RESOURCES_FOUND++)) || true + local cluster=$(echo "$cluster_info" | jq '.DBClusters[0]') + local status=$(json_get "$cluster" '.Status') + local endpoint=$(json_get "$cluster" '.Endpoint') + local reader_endpoint=$(json_get "$cluster" '.ReaderEndpoint') + local engine=$(json_get "$cluster" '.Engine') + local engine_version=$(json_get "$cluster" '.EngineVersion') + local port=$(json_get "$cluster" '.Port') + local serverless_config=$(echo "$cluster" | jq '.ServerlessV2ScalingConfiguration // empty' 2>/dev/null) + + if [[ "$status" == "available" ]]; then + ((RESOURCES_HEALTHY++)) || true + print_status "Cluster ID" "$cluster_id" "ok" + else + ((RESOURCES_UNHEALTHY++)) || true + print_status "Cluster ID" "$cluster_id" "warn" + fi + + print_status "Status" "$(echo -e "$(status_color "$status")")" "info" + print_status "Engine" "$engine $engine_version" "info" + print_status "Endpoint" "$endpoint" "info" + print_status "Reader Endpoint" "$reader_endpoint" "info" + print_status "Port" "$port" "info" + + if [[ -n "$serverless_config" ]]; then + local min_acu=$(json_get "$serverless_config" '.MinCapacity') + local max_acu=$(json_get "$serverless_config" '.MaxCapacity') + print_status "Serverless Mode" "Min: ${min_acu} ACU / Max: ${max_acu} ACU" "info" + print_status " Cost Note" "Aurora Serverless v2: \$0.12/ACU-hour" "warn" + fi + + # Aurora Instances + print_subheader "Aurora DB Instances" + local instance_id + instance_id=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="PostgresInstance") | .PhysicalResourceId' 2>/dev/null || echo "") + + if [[ -n "$instance_id" ]]; then + ((RESOURCES_EXPECTED++)) || true + local inst_info + inst_info=$(aws_cmd rds describe-db-instances --db-instance-identifier "$instance_id" --output json 2>/dev/null || echo "") + + if [[ -n "$inst_info" && $(echo "$inst_info" | jq '.DBInstances | length') -gt 0 ]]; then + ((RESOURCES_FOUND++)) || true + local inst=$(echo "$inst_info" | jq '.DBInstances[0]') + local inst_status=$(json_get "$inst" '.DBInstanceStatus') + local inst_class=$(json_get "$inst" '.DBInstanceClass') + local inst_az=$(json_get "$inst" '.AvailabilityZone') + local storage_type=$(json_get "$inst" '.StorageType') + + if [[ "$inst_status" == "available" ]]; then + ((RESOURCES_HEALTHY++)) || true + else + ((RESOURCES_UNHEALTHY++)) || true + fi + + print_status "Instance ID" "$instance_id" "ok" + print_status "Status" "$(echo -e "$(status_color "$inst_status")")" "info" + print_status "Instance Class" "$inst_class" "info" + print_status "AZ" "$inst_az" "info" + print_status "Storage Type" "$storage_type" "info" + fi + fi + + # Parameter Groups and Subnet Group + local pg_cluster + pg_cluster=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="PostgresClusterParameterGroup") | .PhysicalResourceId' 2>/dev/null || echo "") + [[ -n "$pg_cluster" ]] && { ((RESOURCES_EXPECTED++)); ((RESOURCES_FOUND++)); ((RESOURCES_HEALTHY++)); print_status "Cluster Param Group" "$pg_cluster" "ok"; } || true + + local pg_db + pg_db=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="PostgresDBParameterGroup") | .PhysicalResourceId' 2>/dev/null || echo "") + [[ -n "$pg_db" ]] && { ((RESOURCES_EXPECTED++)); ((RESOURCES_FOUND++)); ((RESOURCES_HEALTHY++)); print_status "DB Param Group" "$pg_db" "ok"; } || true + + local pg_subnet + pg_subnet=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="PostgresSubnetGroup") | .PhysicalResourceId' 2>/dev/null || echo "") + [[ -n "$pg_subnet" ]] && { ((RESOURCES_EXPECTED++)); ((RESOURCES_FOUND++)); ((RESOURCES_HEALTHY++)); print_status "Subnet Group" "$pg_subnet" "ok"; } || true +} + +############################################################################### +# S3 VECTORS AUDIT +############################################################################### + +audit_s3_vectors() { + local bucket_id + bucket_id=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="VectorBucket") | .PhysicalResourceId' 2>/dev/null || echo "") + + if [[ -z "$bucket_id" ]]; then + return + fi + + print_header "S3 VECTORS" + ((RESOURCES_EXPECTED++)) || true + + # S3 Vectors is a newer service; check via the resource status + local resource_status + resource_status=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="VectorBucket") | .ResourceStatus' 2>/dev/null || echo "") + + if [[ "$resource_status" == "CREATE_COMPLETE" || "$resource_status" == "UPDATE_COMPLETE" ]]; then + ((RESOURCES_FOUND++)) || true + ((RESOURCES_HEALTHY++)) || true + print_status "Vector Bucket" "$bucket_id" "ok" + print_status "Resource Status" "$resource_status" "ok" + else + ((RESOURCES_FOUND++)) || true + ((RESOURCES_UNHEALTHY++)) || true + print_status "Vector Bucket" "$bucket_id ($resource_status)" "warn" + fi + + # KMS Key + print_subheader "KMS Encryption" + local kms_key + kms_key=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="VectorBucketKMSKey") | .PhysicalResourceId' 2>/dev/null || echo "") + + if [[ -n "$kms_key" ]]; then + ((RESOURCES_EXPECTED++)) || true + ((RESOURCES_FOUND++)) || true + + local key_info + key_info=$(aws_cmd kms describe-key --key-id "$kms_key" --output json 2>/dev/null || echo "") + + if [[ -n "$key_info" ]]; then + local key_state=$(echo "$key_info" | jq -r '.KeyMetadata.KeyState') + local key_spec=$(echo "$key_info" | jq -r '.KeyMetadata.KeySpec') + + if [[ "$key_state" == "Enabled" ]]; then + ((RESOURCES_HEALTHY++)) || true + print_status "KMS Key" "$kms_key" "ok" + else + ((RESOURCES_UNHEALTHY++)) || true + print_status "KMS Key" "$kms_key ($key_state)" "warn" + fi + print_status "Key State" "$key_state" "info" + print_status "Key Spec" "$key_spec" "info" + print_status " Cost Note" "KMS: \$1/month/key + \$0.03/10K requests" "info" + fi + fi +} + + +############################################################################### +# SAGEMAKER NOTEBOOK AUDIT +############################################################################### + +audit_sagemaker() { + local notebook_id + notebook_id=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="NeptuneNotebookInstance") | .PhysicalResourceId' 2>/dev/null || echo "") + + if [[ -z "$notebook_id" ]]; then + return + fi + + print_header "SAGEMAKER NOTEBOOK" + ((RESOURCES_EXPECTED++)) || true + + local notebook_info + notebook_info=$(aws_cmd sagemaker describe-notebook-instance --notebook-instance-name "$notebook_id" --output json 2>/dev/null || echo "") + + if [[ -z "$notebook_info" ]]; then + print_status "Notebook" "$notebook_id - NOT FOUND" "error" + ((RESOURCES_UNHEALTHY++)) || true + return + fi + + ((RESOURCES_FOUND++)) || true + local status=$(json_get "$notebook_info" '.NotebookInstanceStatus') + local instance_type=$(json_get "$notebook_info" '.InstanceType') + local url=$(json_get "$notebook_info" '.Url') + local volume_size=$(json_get "$notebook_info" '.VolumeSizeInGB') + local direct_access=$(json_get "$notebook_info" '.DirectInternetAccess') + local role_arn=$(json_get "$notebook_info" '.RoleArn') + + if [[ "$status" == "InService" ]]; then + ((RESOURCES_HEALTHY++)) || true + print_status "Notebook Name" "$notebook_id" "ok" + print_status " Cost Note" "RUNNING - ${instance_type} incurring charges" "warn" + elif [[ "$status" == "Stopped" ]]; then + ((RESOURCES_HEALTHY++)) || true + print_status "Notebook Name" "$notebook_id" "ok" + print_status " Cost Note" "Stopped - only storage charges (\$${volume_size:-5}GB EBS)" "info" + else + ((RESOURCES_UNHEALTHY++)) || true + print_status "Notebook Name" "$notebook_id" "warn" + fi + + print_status "Status" "$(echo -e "$(status_color "$status")")" "info" + print_status "Instance Type" "$instance_type" "info" + print_status "Volume Size" "${volume_size}GB" "info" + print_status "URL" "$url" "info" + print_status "Direct Internet" "$direct_access" "info" + print_status "IAM Role" "$(echo "$role_arn" | awk -F'/' '{print $NF}')" "info" + + # Lifecycle Config + local lifecycle_id + lifecycle_id=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="NeptuneNotebookInstanceLifecycleConfig") | .PhysicalResourceId' 2>/dev/null || echo "") + if [[ -n "$lifecycle_id" ]]; then + ((RESOURCES_EXPECTED++)) || true + ((RESOURCES_FOUND++)) || true + ((RESOURCES_HEALTHY++)) || true + print_status "Lifecycle Config" "$lifecycle_id" "ok" + fi +} + +############################################################################### +# IAM ROLES & POLICIES AUDIT +############################################################################### + +audit_iam() { + print_header "IAM ROLES & POLICIES" + + # Roles + print_subheader "IAM Roles" + local roles=() + while IFS= read -r line; do + [[ -n "$line" ]] && roles+=("$line") + done < <(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.ResourceType=="AWS::IAM::Role") | "\(.LogicalResourceId)|\(.PhysicalResourceId)"' 2>/dev/null) + + if [[ ${#roles[@]} -gt 0 ]]; then + print_table_header "Logical ID" "Role Name" "Status" + for role_entry in "${roles[@]}"; do + IFS='|' read -r logical_id role_name <<< "$role_entry" + ((RESOURCES_EXPECTED++)) || true + + local role_info + role_info=$(aws_cmd iam get-role --role-name "$role_name" --output json 2>/dev/null || echo "") + + if [[ -n "$role_info" ]]; then + ((RESOURCES_FOUND++)) || true + ((RESOURCES_HEALTHY++)) || true + local create_date=$(echo "$role_info" | jq -r '.Role.CreateDate' 2>/dev/null) + print_table_row "$logical_id" "${role_name:0:40}" "${GREEN}EXISTS${RESET}" + else + ((RESOURCES_UNHEALTHY++)) || true + print_table_row "$logical_id" "${role_name:0:40}" "${RED}NOT FOUND${RESET}" + fi + done + else + print_status "IAM Roles" "None found in stack" "info" + fi + + # Managed Policies + print_subheader "IAM Managed Policies" + local policies=() + while IFS= read -r line; do + [[ -n "$line" ]] && policies+=("$line") + done < <(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.ResourceType=="AWS::IAM::ManagedPolicy") | "\(.LogicalResourceId)|\(.PhysicalResourceId)"' 2>/dev/null) + + if [[ ${#policies[@]} -gt 0 ]]; then + print_table_header "Policy Name" "ARN (truncated)" + for policy_entry in "${policies[@]}"; do + IFS='|' read -r logical_id policy_arn <<< "$policy_entry" + ((RESOURCES_EXPECTED++)) || true + ((RESOURCES_FOUND++)) || true + ((RESOURCES_HEALTHY++)) || true + + # Truncate ARN for display + local display_arn="${policy_arn}" + if [[ ${#display_arn} -gt 70 ]]; then + display_arn="...${display_arn: -67}" + fi + print_table_row "$logical_id" "$display_arn" + done + else + print_status "Managed Policies" "None found in stack" "info" + fi +} + +############################################################################### +# LAMBDA FUNCTIONS AUDIT +############################################################################### + +audit_lambda() { + local lambdas=() + while IFS= read -r line; do + [[ -n "$line" ]] && lambdas+=("$line") + done < <(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.ResourceType=="AWS::Lambda::Function") | "\(.LogicalResourceId)|\(.PhysicalResourceId)"' 2>/dev/null) + + if [[ ${#lambdas[@]} -eq 0 ]]; then + return + fi + + print_header "LAMBDA FUNCTIONS" + + for lambda_entry in "${lambdas[@]}"; do + IFS='|' read -r logical_id function_name <<< "$lambda_entry" + ((RESOURCES_EXPECTED++)) || true + + local func_info + func_info=$(aws_cmd lambda get-function --function-name "$function_name" --output json 2>/dev/null || echo "") + + if [[ -z "$func_info" ]]; then + ((RESOURCES_UNHEALTHY++)) || true + print_status "$logical_id" "$function_name - NOT FOUND" "error" + continue + fi + + ((RESOURCES_FOUND++)) || true + local config=$(echo "$func_info" | jq '.Configuration') + local state=$(json_get "$config" '.State') + local runtime=$(json_get "$config" '.Runtime') + local memory=$(json_get "$config" '.MemorySize') + local timeout=$(json_get "$config" '.Timeout') + local last_modified=$(json_get "$config" '.LastModified') + local code_size=$(json_get "$config" '.CodeSize') + + if [[ "$state" == "Active" ]]; then + ((RESOURCES_HEALTHY++)) || true + print_status "$logical_id" "$function_name" "ok" + else + ((RESOURCES_UNHEALTHY++)) || true + print_status "$logical_id" "$function_name ($state)" "warn" + fi + + print_status " Runtime" "$runtime" "info" + print_status " Memory" "${memory}MB / Timeout: ${timeout}s" "info" + print_status " Code Size" "$(echo "$code_size" | awk '{printf "%.1f KB", $1/1024}')" "info" + print_status " Last Modified" "$last_modified" "info" + done +} + + +############################################################################### +# ADDITIONAL RESOURCES AUDIT (Route Tables, Custom Resources) +############################################################################### + +audit_additional_resources() { + # Check for any resources we haven't explicitly covered + local other_resources=() + while IFS= read -r line; do + [[ -n "$line" ]] && other_resources+=("$line") + done < <(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select( + .ResourceType != "AWS::EC2::VPC" and + .ResourceType != "AWS::EC2::Subnet" and + .ResourceType != "AWS::EC2::InternetGateway" and + .ResourceType != "AWS::EC2::VPCGatewayAttachment" and + .ResourceType != "AWS::EC2::NatGateway" and + .ResourceType != "AWS::EC2::EIP" and + .ResourceType != "AWS::EC2::SecurityGroup" and + .ResourceType != "AWS::EC2::SecurityGroupIngress" and + .ResourceType != "AWS::EC2::SecurityGroupEgress" and + .ResourceType != "AWS::EC2::RouteTable" and + .ResourceType != "AWS::EC2::Route" and + .ResourceType != "AWS::EC2::SubnetRouteTableAssociation" and + .ResourceType != "AWS::Neptune::DBCluster" and + .ResourceType != "AWS::Neptune::DBInstance" and + .ResourceType != "AWS::Neptune::DBClusterParameterGroup" and + .ResourceType != "AWS::Neptune::DBParameterGroup" and + .ResourceType != "AWS::Neptune::DBSubnetGroup" and + .ResourceType != "AWS::NeptuneGraph::Graph" and + .ResourceType != "AWS::OpenSearchServerless::Collection" and + .ResourceType != "AWS::OpenSearchServerless::AccessPolicy" and + .ResourceType != "AWS::OpenSearchServerless::SecurityPolicy" and + .ResourceType != "AWS::OpenSearchServerless::VpcEndpoint" and + .ResourceType != "AWS::RDS::DBCluster" and + .ResourceType != "AWS::RDS::DBInstance" and + .ResourceType != "AWS::RDS::DBClusterParameterGroup" and + .ResourceType != "AWS::RDS::DBParameterGroup" and + .ResourceType != "AWS::RDS::DBSubnetGroup" and + .ResourceType != "AWS::S3Vectors::VectorBucket" and + .ResourceType != "AWS::KMS::Key" and + .ResourceType != "AWS::SageMaker::NotebookInstance" and + .ResourceType != "AWS::SageMaker::NotebookInstanceLifecycleConfig" and + .ResourceType != "AWS::IAM::Role" and + .ResourceType != "AWS::IAM::ManagedPolicy" and + .ResourceType != "AWS::Lambda::Function" and + .ResourceType != "Custom::CustomResource" + ) | "\(.ResourceType)|\(.LogicalResourceId)|\(.PhysicalResourceId)|\(.ResourceStatus)"' 2>/dev/null) + + if [[ ${#other_resources[@]} -gt 0 ]]; then + print_header "ADDITIONAL RESOURCES" + print_table_header "Type" "Logical ID" "Status" + for res_entry in "${other_resources[@]}"; do + IFS='|' read -r res_type logical_id phys_id res_status <<< "$res_entry" + ((RESOURCES_EXPECTED++)) || true + ((RESOURCES_FOUND++)) || true + if [[ "$res_status" == *"COMPLETE"* ]]; then + ((RESOURCES_HEALTHY++)) || true + else + ((RESOURCES_UNKNOWN++)) || true + fi + # Shorten type for display + local short_type="${res_type#AWS::}" + print_table_row "$short_type" "$logical_id" "$(echo -e "$(status_color "$res_status")")" + done + fi + + # Custom Resources + local custom_resources=() + while IFS= read -r line; do + [[ -n "$line" ]] && custom_resources+=("$line") + done < <(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.ResourceType=="Custom::CustomResource") | "\(.LogicalResourceId)|\(.PhysicalResourceId)|\(.ResourceStatus)"' 2>/dev/null) + + if [[ ${#custom_resources[@]} -gt 0 ]]; then + print_subheader "Custom Resources (Lambda-backed)" + for cr_entry in "${custom_resources[@]}"; do + IFS='|' read -r logical_id phys_id cr_status <<< "$cr_entry" + ((RESOURCES_EXPECTED++)) || true + ((RESOURCES_FOUND++)) || true + if [[ "$cr_status" == *"COMPLETE"* ]]; then + ((RESOURCES_HEALTHY++)) || true + fi + print_status "$logical_id" "$cr_status" "ok" + done + fi +} + +############################################################################### +# SUMMARY +############################################################################### + +print_summary() { + print_header "AUDIT SUMMARY" + + # Resource count from stack + local total_stack_resources + total_stack_resources=$(echo "$STACK_RESOURCES" | jq '.StackResourceSummaries | length' 2>/dev/null || echo "0") + + echo "" + echo -e " ${BOLD}Resource Overview${RESET}" + echo -e " ─────────────────────────────────────────" + printf " %-30s %s\n" "Stack Resources (total):" "$total_stack_resources" + printf " %-30s ${GREEN}%s${RESET}\n" "Resources Verified:" "$RESOURCES_FOUND" + printf " %-30s ${GREEN}%s${RESET}\n" "Healthy:" "$RESOURCES_HEALTHY" + printf " %-30s ${RED}%s${RESET}\n" "Unhealthy/Missing:" "$RESOURCES_UNHEALTHY" + printf " %-30s ${YELLOW}%s${RESET}\n" "Unknown:" "$RESOURCES_UNKNOWN" + echo "" + + # Health percentage + local health_pct=0 + if [[ $RESOURCES_FOUND -gt 0 ]]; then + health_pct=$(( (RESOURCES_HEALTHY * 100) / RESOURCES_FOUND )) + fi + + echo -e " ${BOLD}Health Score${RESET}" + echo -e " ─────────────────────────────────────────" + if [[ $health_pct -ge 90 ]]; then + echo -e " ${GREEN}██████████ ${health_pct}% - HEALTHY${RESET}" + elif [[ $health_pct -ge 70 ]]; then + echo -e " ${YELLOW}███████░░░ ${health_pct}% - DEGRADED${RESET}" + else + echo -e " ${RED}████░░░░░░ ${health_pct}% - UNHEALTHY${RESET}" + fi + echo "" + + # Cost indicators + echo -e " ${BOLD}Cost Indicators (estimated hourly when running)${RESET}" + echo -e " ─────────────────────────────────────────" + + local has_cost_items=false + + # Check what's running + local notebook_status + notebook_status=$(aws_cmd sagemaker describe-notebook-instance \ + --notebook-instance-name "$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="NeptuneNotebookInstance") | .PhysicalResourceId' 2>/dev/null)" \ + --query 'NotebookInstanceStatus' --output text 2>/dev/null || echo "") + + if [[ "$notebook_status" == "InService" ]]; then + printf " ${YELLOW}⚠${RESET} %-35s %s\n" "SageMaker Notebook:" "RUNNING (ml.m5.xlarge ~\$0.23/hr)" + has_cost_items=true + fi + + # Neptune Analytics + local graph_id + graph_id=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="Graph") | .PhysicalResourceId' 2>/dev/null || echo "") + if [[ -n "$graph_id" ]]; then + printf " ${YELLOW}⚠${RESET} %-35s %s\n" "Neptune Analytics Graph:" "Provisioned memory charges apply" + has_cost_items=true + fi + + # Neptune DB + local neptune_cluster + neptune_cluster=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="NeptuneDBCluster") | .PhysicalResourceId' 2>/dev/null || echo "") + if [[ -n "$neptune_cluster" ]]; then + printf " ${YELLOW}⚠${RESET} %-35s %s\n" "Neptune DB Cluster:" "Instance/serverless charges apply" + has_cost_items=true + fi + + # OpenSearch + local opensearch_col + opensearch_col=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="OpenSearchServerless") | .PhysicalResourceId' 2>/dev/null || echo "") + if [[ -n "$opensearch_col" ]]; then + printf " ${YELLOW}⚠${RESET} %-35s %s\n" "OpenSearch Serverless:" "Min 4 OCUs (~\$0.96/hr)" + has_cost_items=true + fi + + # Aurora + local aurora_cluster + aurora_cluster=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="PostgresCluster") | .PhysicalResourceId' 2>/dev/null || echo "") + if [[ -n "$aurora_cluster" ]]; then + printf " ${YELLOW}⚠${RESET} %-35s %s\n" "Aurora PostgreSQL:" "Serverless/instance charges apply" + has_cost_items=true + fi + + # NAT Gateway + local nat_gw + nat_gw=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="NATGW") | .PhysicalResourceId' 2>/dev/null || echo "") + if [[ -n "$nat_gw" ]]; then + printf " ${YELLOW}⚠${RESET} %-35s %s\n" "NAT Gateway:" "~\$0.045/hr + data" + has_cost_items=true + fi + + if [[ "$has_cost_items" == "false" ]]; then + printf " ${GREEN}✓${RESET} No active cost-incurring resources detected\n" + fi + + echo "" + echo -e " ${DIM}Audit completed at: $(date -u '+%Y-%m-%d %H:%M:%S UTC')${RESET}" + echo -e " ${DIM}Stack: ${STACK_NAME} | Region: ${REGION}${RESET}" + echo "" +} + +############################################################################### +# JSON OUTPUT +############################################################################### + +output_json() { + local total_stack_resources + total_stack_resources=$(echo "$STACK_RESOURCES" | jq '.StackResourceSummaries | length' 2>/dev/null || echo "0") + + local health_pct=0 + if [[ $RESOURCES_FOUND -gt 0 ]]; then + health_pct=$(( (RESOURCES_HEALTHY * 100) / RESOURCES_FOUND )) + fi + + # Build full JSON output + cat < /dev/null; then + echo -e "${RED}Error: AWS CLI not found. Please install it first.${RESET}" >&2 + exit 1 + fi + + # Verify jq is available + if ! command -v jq &> /dev/null; then + echo -e "${RED}Error: jq not found. Please install it (brew install jq / apt install jq).${RESET}" >&2 + exit 1 + fi + + # Verify AWS credentials + if ! aws_cmd sts get-caller-identity &> /dev/null; then + echo -e "${RED}Error: Unable to authenticate with AWS. Check your credentials/profile.${RESET}" >&2 + exit 1 + fi + + if [[ "$JSON_OUTPUT" != "true" ]]; then + echo "" + echo -e "${BOLD}${MAGENTA}╔══════════════════════════════════════════════════════════════════════════════╗${RESET}" + echo -e "${BOLD}${MAGENTA}║ GraphRAG Toolkit - Deployment Audit Script v${VERSION} ║${RESET}" + echo -e "${BOLD}${MAGENTA}╚══════════════════════════════════════════════════════════════════════════════╝${RESET}" + echo "" + local mode_label="Stack" + [[ "$MANUAL_MODE" == "true" ]] && mode_label="Manual Scan" + echo -e " ${DIM}Region: ${REGION} | Profile: ${PROFILE:-default} | Mode: ${mode_label}${RESET}" + fi + + if [[ "$MANUAL_MODE" == "true" ]]; then + # Manual mode: scan AWS directly without needing a CloudFormation stack + discover_resources_manual + + # Skip CloudFormation audit, go straight to resource audits + audit_networking + audit_neptune_db + audit_neptune_analytics + audit_opensearch + audit_aurora_postgres + audit_s3_vectors + audit_sagemaker + audit_iam + audit_lambda + audit_additional_resources + else + # Stack mode: detect or use provided stack name + detect_stack + + # Run all audit sections + audit_cloudformation + audit_networking + audit_neptune_db + audit_neptune_analytics + audit_opensearch + audit_aurora_postgres + audit_s3_vectors + audit_sagemaker + audit_iam + audit_lambda + audit_additional_resources + fi + + # Output results + if [[ "$JSON_OUTPUT" == "true" ]]; then + output_json + else + print_summary + fi +} + +# Run main with all arguments +main "$@" diff --git a/examples/lexical-graph/cloudformation-templates/audit-resources.json b/examples/lexical-graph/cloudformation-templates/audit-resources.json new file mode 100644 index 000000000..45cf212da --- /dev/null +++ b/examples/lexical-graph/cloudformation-templates/audit-resources.json @@ -0,0 +1,156 @@ +{ + "description": "Resource types to discover in manual scan mode. Remove or add entries to control what the audit finds.", + "resource_types": [ + { + "type": "neptune_clusters", + "label": "Neptune DB Clusters", + "enabled": true, + "service": "neptune", + "operation": "describe-db-clusters", + "jq_extract": ".DBClusters[].DBClusterIdentifier", + "logical_id": "NeptuneDBCluster", + "cfn_type": "AWS::Neptune::DBCluster" + }, + { + "type": "neptune_instances", + "label": "Neptune DB Instances", + "enabled": true, + "service": "neptune", + "operation": "describe-db-instances", + "jq_extract": ".DBInstances[].DBInstanceIdentifier", + "logical_id": "NeptuneDBInstance", + "cfn_type": "AWS::Neptune::DBInstance" + }, + { + "type": "neptune_analytics", + "label": "Neptune Analytics Graphs", + "enabled": true, + "service": "neptune-graph", + "operation": "list-graphs", + "jq_extract": ".graphs[].id", + "logical_id": "Graph", + "cfn_type": "AWS::NeptuneGraph::Graph" + }, + { + "type": "opensearch_collections", + "label": "OpenSearch Serverless Collections", + "enabled": true, + "service": "opensearchserverless", + "operation": "list-collections", + "jq_extract": ".collectionSummaries[].id", + "logical_id": "OpenSearchServerless", + "cfn_type": "AWS::OpenSearchServerless::Collection" + }, + { + "type": "opensearch_encryption_policies", + "label": "OpenSearch Encryption Policies", + "enabled": true, + "service": "opensearchserverless", + "operation": "list-security-policies --type encryption", + "jq_extract": ".securityPolicySummaries[].name", + "logical_id": "OSSEncryptionPolicy", + "cfn_type": "AWS::OpenSearchServerless::SecurityPolicy" + }, + { + "type": "opensearch_network_policies", + "label": "OpenSearch Network Policies", + "enabled": true, + "service": "opensearchserverless", + "operation": "list-security-policies --type network", + "jq_extract": ".securityPolicySummaries[].name", + "logical_id": "OSSNetworkPolicy", + "cfn_type": "AWS::OpenSearchServerless::SecurityPolicy" + }, + { + "type": "opensearch_access_policies", + "label": "OpenSearch Access Policies", + "enabled": true, + "service": "opensearchserverless", + "operation": "list-access-policies --type data", + "jq_extract": ".accessPolicySummaries[].name", + "logical_id": "OSSAccessPolicy", + "cfn_type": "AWS::OpenSearchServerless::AccessPolicy" + }, + { + "type": "sagemaker_notebooks", + "label": "SageMaker Notebook Instances", + "enabled": true, + "service": "sagemaker", + "operation": "list-notebook-instances", + "jq_extract": ".NotebookInstances[].NotebookInstanceName", + "logical_id": "NeptuneNotebookInstance", + "cfn_type": "AWS::SageMaker::NotebookInstance" + }, + { + "type": "s3_buckets", + "label": "S3 Buckets", + "enabled": true, + "service": "s3api", + "operation": "list-buckets", + "jq_extract": ".Buckets[].Name", + "logical_id": "S3Bucket", + "cfn_type": "AWS::S3::Bucket" + }, + { + "type": "aurora_clusters", + "label": "Aurora PostgreSQL Clusters", + "enabled": false, + "service": "rds", + "operation": "describe-db-clusters", + "jq_extract": ".DBClusters[] | select(.Engine==\"aurora-postgresql\") | .DBClusterIdentifier", + "logical_id": "PostgresCluster", + "cfn_type": "AWS::RDS::DBCluster" + }, + { + "type": "aurora_instances", + "label": "Aurora PostgreSQL Instances", + "enabled": false, + "service": "rds", + "operation": "describe-db-instances", + "jq_extract": ".DBInstances[] | select(.Engine==\"aurora-postgresql\") | .DBInstanceIdentifier", + "logical_id": "PostgresInstance", + "cfn_type": "AWS::RDS::DBInstance" + }, + { + "type": "lambda_functions", + "label": "Lambda Functions", + "enabled": false, + "service": "lambda", + "operation": "list-functions", + "jq_extract": ".Functions[].FunctionName", + "logical_id": "LambdaFunction", + "cfn_type": "AWS::Lambda::Function" + }, + { + "type": "iam_roles", + "label": "IAM Roles (non-service-linked)", + "enabled": false, + "service": "iam", + "operation": "list-roles", + "jq_extract": ".Roles[] | select(.Path != \"/aws-service-role/\") | .RoleName", + "logical_id": "IAMRole", + "cfn_type": "AWS::IAM::Role" + }, + { + "type": "iam_policies", + "label": "IAM Customer Managed Policies", + "enabled": false, + "service": "iam", + "operation": "list-policies --scope Local", + "jq_extract": ".Policies[].PolicyName", + "logical_id": "IAMManagedPolicy", + "cfn_type": "AWS::IAM::ManagedPolicy" + }, + { + "type": "vpc", + "label": "VPCs (non-default)", + "enabled": false, + "service": "ec2", + "operation": "describe-vpcs", + "jq_extract": ".Vpcs[] | select(.IsDefault == false) | .VpcId", + "logical_id": "VPC", + "cfn_type": "AWS::EC2::VPC", + "note": "VPC discovery also pulls subnets, security groups, NAT/IGW, and VPC endpoints" + } + ] +} diff --git a/examples/lexical-graph/cloudformation-templates/audit/README.md b/examples/lexical-graph/cloudformation-templates/audit/README.md new file mode 100644 index 000000000..ac024db76 --- /dev/null +++ b/examples/lexical-graph/cloudformation-templates/audit/README.md @@ -0,0 +1,189 @@ +# GraphRAG Toolkit — Deployment Audit + +A comprehensive audit script for discovering and inspecting AWS resources deployed by the GraphRAG Toolkit CloudFormation templates — or deployed manually without CloudFormation. + +## Quick Start + +```bash +# Stack-based audit (auto-detects graphrag CloudFormation stacks) +./audit-deployment.sh --profile --region us-east-1 + +# Manual audit (scans by resource type, no stack required) +./audit-deployment.sh --manual --profile --region us-east-1 +``` + +## Files + +| File | Description | +|------|-------------| +| `audit-deployment.sh` | Main audit script (bash) | +| `audit-resources.json` | Resource type configuration for manual scan mode | +| `README.md` | This file | + +## Modes + +### Stack Mode (default) + +Audits resources managed by a CloudFormation stack. The script will: +1. Auto-detect stacks with "graphrag" in the name, or use `--stack-name` +2. List all stack resources and check their status +3. Run drift detection +4. Inspect each resource type for detailed health info + +```bash +./audit-deployment.sh --stack-name my-graphrag-stack --region us-west-2 +``` + +### Manual Mode (`--manual`) + +Scans the AWS account directly by resource type — no CloudFormation stack required. Useful when resources were created manually, via CLI, or through other IaC tools. + +```bash +./audit-deployment.sh --manual --profile production --region us-east-1 +``` + +The scan is driven by `audit-resources.json` which defines what to look for. + +## Options + +| Flag | Description | Default | +|------|-------------|---------| +| `--stack-name NAME` | CloudFormation stack name | Auto-detect | +| `--region REGION` | AWS region | `us-east-1` | +| `--profile PROFILE` | AWS CLI profile name | Default profile | +| `--manual` | Scan by resource type (no stack required) | Off | +| `--resources FILE` | Custom resource config JSON (implies `--manual`) | `audit-resources.json` | +| `--json` | Machine-readable JSON output | Off | +| `--help` | Show usage | — | + +## Configuring Manual Scan (`audit-resources.json`) + +The config file controls which AWS resource types the manual scan discovers. Each entry has: + +```json +{ + "type": "neptune_clusters", + "label": "Neptune DB Clusters", + "enabled": true, + "service": "neptune", + "operation": "describe-db-clusters", + "jq_extract": ".DBClusters[].DBClusterIdentifier", + "logical_id": "NeptuneDBCluster", + "cfn_type": "AWS::Neptune::DBCluster" +} +``` + +| Field | Purpose | +|-------|---------| +| `enabled` | `true` to scan, `false` to skip | +| `service` | AWS CLI service name | +| `operation` | AWS CLI operation (with any flags like `--type encryption`) | +| `jq_extract` | jq expression to extract resource identifiers from the response | +| `logical_id` | Logical name used internally for audit mapping | +| `cfn_type` | CloudFormation resource type label | + +### Default Configuration + +**Enabled** (scanned by default): +- Neptune DB Clusters & Instances +- Neptune Analytics Graphs +- OpenSearch Serverless Collections +- OpenSearch Encryption, Network, and Access Policies +- SageMaker Notebook Instances +- S3 Buckets + +**Disabled** (available but off by default): +- Aurora PostgreSQL Clusters & Instances +- Lambda Functions +- IAM Roles & Customer Managed Policies +- VPCs (with subnets, security groups, NAT/IGW, endpoints) + +To enable a resource type, set `"enabled": true` in the JSON file. + +### Custom Config + +```bash +# Use a custom resource list +./audit-deployment.sh --resources my-scan-config.json --profile nw +``` + +## Resource Types Audited + +The script provides detailed inspection for these resource categories: + +| Category | Details Shown | +|----------|---------------| +| **Neptune DB** | Cluster status, endpoint, engine version, serverless NCU config, instances | +| **Neptune Analytics** | Graph ID, memory (GiB), vector search, endpoint | +| **OpenSearch Serverless** | Collection status, type, endpoints, all policies | +| **Aurora PostgreSQL** | Cluster status, serverless v2 config, instances | +| **S3 Buckets** | Bucket names, KMS keys | +| **SageMaker** | Notebook status, instance type, volume, running cost | +| **IAM** | Roles and customer-managed policies | +| **Lambda** | Function names, runtimes, memory | +| **VPC/Networking** | VPCs, subnets, security groups, NAT/IGW, VPC endpoints | +| **CloudFormation** | Stack status, outputs, parameters, drift detection | + +## Output + +### Terminal (default) + +Colorized output with: +- Section headers +- Status icons: ✓ (healthy), ⚠ (warning), ✗ (error), ℹ (info) +- Cost indicators for billable resources +- Health score bar + +### JSON (`--json`) + +```json +{ + "audit_metadata": { "timestamp": "...", "region": "...", "version": "1.1.0" }, + "summary": { + "total_stack_resources": 95, + "resources_verified": 95, + "resources_healthy": 27, + "health_percentage": 28 + }, + "stack_resources": [ ... ], + "stack_info": { ... } +} +``` + +## Prerequisites + +- **AWS CLI v2** — configured with valid credentials +- **jq** — JSON processor (`brew install jq` / `apt install jq`) +- **bash 3.2+** — compatible with macOS default bash + +## Examples + +```bash +# Audit everything enabled in the config +./audit-deployment.sh --manual --profile nw + +# Just Neptune and OpenSearch (create a minimal config) +./audit-deployment.sh --resources neptune-only.json --profile nw + +# Stack-based with JSON output for CI/CD +./audit-deployment.sh --stack-name graphrag-prod --json > audit-report.json + +# Pipe JSON to jq for specific info +./audit-deployment.sh --manual --json | jq '.stack_resources | group_by(.ResourceType) | map({type: .[0].ResourceType, count: length})' +``` + +## Cost Awareness + +The audit highlights running resources that incur charges: + +| Resource | Cost Note | +|----------|-----------| +| Neptune Serverless | ~$0.1028/NCU-hour | +| OpenSearch Serverless | ~$0.24/OCU-hour (min 4 OCUs) | +| SageMaker Notebooks | Varies by instance type (only when running) | +| NAT Gateways | ~$0.045/hour + data processing | +| Neptune DB (provisioned) | Varies by instance class | + +## License + +Internal tool — part of the GraphRAG Toolkit examples. diff --git a/examples/lexical-graph/cloudformation-templates/audit/audit-deployment.sh b/examples/lexical-graph/cloudformation-templates/audit/audit-deployment.sh new file mode 100755 index 000000000..a99b1c46a --- /dev/null +++ b/examples/lexical-graph/cloudformation-templates/audit/audit-deployment.sh @@ -0,0 +1,1618 @@ +#!/usr/bin/env bash +# +# audit-deployment.sh - Comprehensive audit script for GraphRAG Toolkit CloudFormation deployments +# +# This script audits all resources deployed by graphrag-toolkit CloudFormation templates. +# It checks resource existence, status, configuration, and cost-relevant details. +# +# Supported template variants: +# - neptune-analytics (with opensearch, s3-vectors, or aurora-postgres) +# - neptune-db (with opensearch, s3-vectors, or aurora-postgres) +# - neptune-db-aurora-postgres-existing-vpc +# +# Usage: +# ./audit-deployment.sh [--stack-name NAME] [--region REGION] [--profile PROFILE] [--json] +# +# Options: +# --stack-name CloudFormation stack name (auto-detects if not provided) +# --region AWS region (default: us-east-1) +# --profile AWS CLI profile (optional) +# --json Output in JSON format for machine-readable consumption +# --help Show this help message +# +# Author: Generated for graphrag-toolkit +# Date: 2024 +# + +set -euo pipefail + +############################################################################### +# CONFIGURATION & GLOBALS +############################################################################### + +VERSION="1.1.0" +STACK_NAME="" +REGION="us-east-1" +PROFILE="" +JSON_OUTPUT=false +MANUAL_MODE=false +RESOURCE_CONFIG="" +AWS_CMD="aws" + +# Counters for summary +RESOURCES_FOUND=0 +RESOURCES_EXPECTED=0 +RESOURCES_HEALTHY=0 +RESOURCES_UNHEALTHY=0 +RESOURCES_UNKNOWN=0 + +# JSON accumulator (stored as a simple variable for bash 3 compat) +JSON_SECTION_CLOUDFORMATION="{}" + +############################################################################### +# COLOR DEFINITIONS +############################################################################### + +if [[ -t 1 ]] && [[ "${NO_COLOR:-}" != "1" ]]; then + RED='\033[0;31m' + GREEN='\033[0;32m' + YELLOW='\033[1;33m' + BLUE='\033[0;34m' + CYAN='\033[0;36m' + MAGENTA='\033[0;35m' + BOLD='\033[1m' + DIM='\033[2m' + RESET='\033[0m' +else + RED='' GREEN='' YELLOW='' BLUE='' CYAN='' MAGENTA='' BOLD='' DIM='' RESET='' +fi + +############################################################################### +# UTILITY FUNCTIONS +############################################################################### + +print_header() { + local title="$1" + if [[ "$JSON_OUTPUT" == "true" ]]; then return; fi + echo "" + echo -e "${BOLD}${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" + echo -e "${BOLD}${BLUE} $title${RESET}" + echo -e "${BOLD}${BLUE}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" +} + +print_subheader() { + local title="$1" + if [[ "$JSON_OUTPUT" == "true" ]]; then return; fi + echo "" + echo -e " ${BOLD}${CYAN}▸ $title${RESET}" + echo -e " ${DIM}──────────────────────────────────────────────────────────${RESET}" +} + +print_status() { + local label="$1" + local value="$2" + local status="${3:-info}" # info, ok, warn, error + if [[ "$JSON_OUTPUT" == "true" ]]; then return; fi + + local color="" + local icon="" + case "$status" in + ok) color="$GREEN"; icon="✓" ;; + warn) color="$YELLOW"; icon="⚠" ;; + error) color="$RED"; icon="✗" ;; + info) color="$CYAN"; icon="ℹ" ;; + esac + printf " ${color}${icon}${RESET} %-35s %s\n" "$label:" "$value" +} + +print_table_row() { + local col1="$1" col2="$2" col3="${3:-}" col4="${4:-}" + if [[ "$JSON_OUTPUT" == "true" ]]; then return; fi + if [[ -n "$col4" ]]; then + printf " %-30s %-20s %-25s %s\n" "$col1" "$col2" "$col3" "$col4" + elif [[ -n "$col3" ]]; then + printf " %-30s %-25s %s\n" "$col1" "$col2" "$col3" + else + printf " %-35s %s\n" "$col1" "$col2" + fi +} + +print_table_header() { + if [[ "$JSON_OUTPUT" == "true" ]]; then return; fi + local cols=("$@") + local header="" + local separator="" + for col in "${cols[@]}"; do + header+=$(printf "%-30s " "$col") + separator+=$(printf "%-30s " "$(printf '%0.s─' $(seq 1 ${#col}))") + done + echo -e " ${BOLD}${header}${RESET}" + echo -e " ${DIM}${separator}${RESET}" +} + +status_color() { + local status="$1" + case "$status" in + *COMPLETE*|*AVAILABLE*|*ACTIVE*|*InService*|*running*|*available*) + echo -e "${GREEN}${status}${RESET}" ;; + *PROGRESS*|*UPDATING*|*CREATING*|*starting*|*pending*) + echo -e "${YELLOW}${status}${RESET}" ;; + *FAILED*|*DELETE*|*stopped*|*error*|*INACTIVE*) + echo -e "${RED}${status}${RESET}" ;; + *) + echo -e "${status}" ;; + esac +} + +# Execute AWS CLI command with proper profile/region +aws_cmd() { + local cmd="$AWS_CMD" + if [[ -n "$PROFILE" ]]; then + cmd+=" --profile $PROFILE" + fi + cmd+=" --region $REGION" + cmd+=" $*" + eval "$cmd" 2>/dev/null +} + +# Safe JSON extraction +json_get() { + local json="$1" + local query="$2" + echo "$json" | jq -r "$query // empty" 2>/dev/null || echo "" +} + +############################################################################### +# ARGUMENT PARSING +############################################################################### + +show_help() { + cat << EOF +${BOLD}GraphRAG Toolkit Deployment Audit Script v${VERSION}${RESET} + +Usage: $(basename "$0") [OPTIONS] + +Options: + --stack-name NAME CloudFormation stack name (auto-detects 'graphrag' stacks) + --region REGION AWS region (default: us-east-1) + --profile PROFILE AWS CLI profile name + --manual Scan for resources directly (no CloudFormation stack required) + --resources FILE JSON config file for manual mode (default: audit-resources.json) + --json Output in JSON format + --help Show this help message + +Examples: + $(basename "$0") + $(basename "$0") --stack-name my-graphrag-stack --region us-west-2 + $(basename "$0") --profile production --json + $(basename "$0") --manual --region us-east-1 + $(basename "$0") --manual --profile production + +EOF + exit 0 +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --stack-name) + STACK_NAME="$2" + shift 2 + ;; + --region) + REGION="$2" + shift 2 + ;; + --profile) + PROFILE="$2" + shift 2 + ;; + --manual) + MANUAL_MODE=true + shift + ;; + --resources) + RESOURCE_CONFIG="$2" + MANUAL_MODE=true + shift 2 + ;; + --json) + JSON_OUTPUT=true + shift + ;; + --help|-h) + show_help + ;; + *) + echo -e "${RED}Error: Unknown option: $1${RESET}" >&2 + echo "Use --help for usage information." >&2 + exit 1 + ;; + esac + done +} + + +############################################################################### +# STACK DETECTION +############################################################################### + +detect_stack() { + if [[ -n "$STACK_NAME" ]]; then + return 0 + fi + + if [[ "$JSON_OUTPUT" != "true" ]]; then + echo -e "${YELLOW}No --stack-name provided. Searching for GraphRAG stacks...${RESET}" + fi + + local stacks + stacks=$(aws_cmd cloudformation list-stacks \ + --stack-status-filter CREATE_COMPLETE UPDATE_COMPLETE UPDATE_ROLLBACK_COMPLETE \ + --query "StackSummaries[?contains(StackName, 'graphrag') || contains(StackName, 'GraphRAG')].{Name:StackName,Status:StackStatus,Created:CreationTime}" \ + --output json 2>/dev/null || echo "[]") + + local count + count=$(echo "$stacks" | jq 'length' 2>/dev/null || echo "0") + + if [[ "$count" -eq 0 ]]; then + echo -e "${RED}Error: No GraphRAG stacks found in ${REGION}.${RESET}" >&2 + echo "Use --stack-name to specify a stack explicitly." >&2 + exit 1 + elif [[ "$count" -eq 1 ]]; then + STACK_NAME=$(echo "$stacks" | jq -r '.[0].Name') + if [[ "$JSON_OUTPUT" != "true" ]]; then + echo -e "${GREEN}Auto-detected stack: ${BOLD}${STACK_NAME}${RESET}" + fi + else + if [[ "$JSON_OUTPUT" != "true" ]]; then + echo -e "${YELLOW}Multiple GraphRAG stacks found:${RESET}" + echo "$stacks" | jq -r '.[] | " - \(.Name) [\(.Status)] (created: \(.Created))"' + echo "" + echo -e "${YELLOW}Please specify one with --stack-name${RESET}" + fi + exit 1 + fi +} + +############################################################################### +# MANUAL RESOURCE DISCOVERY (no CloudFormation stack required) +############################################################################### + +discover_resources_manual() { + # In manual mode, we scan AWS directly by RESOURCE TYPE. + # Reads resource types from audit-resources.json config file. + + # Locate config file (same directory as script) + local script_dir + script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + local config_file="${RESOURCE_CONFIG:-${script_dir}/audit-resources.json}" + + if [[ ! -f "$config_file" ]]; then + echo -e "${RED}Error: Resource config not found: ${config_file}${RESET}" >&2 + echo "Create audit-resources.json or specify with --resources " >&2 + exit 1 + fi + + if [[ "$JSON_OUTPUT" != "true" ]]; then + print_header "MANUAL RESOURCE DISCOVERY" + echo -e " ${YELLOW}Config: ${config_file}${RESET}" + echo -e " ${YELLOW}Scanning region ${REGION} for enabled resource types...${RESET}" + echo "" + fi + + local resources="[]" + + # Read enabled resource types from config + local enabled_types + enabled_types=$(jq -c '.resource_types[] | select(.enabled == true)' "$config_file" 2>/dev/null) + + if [[ -z "$enabled_types" ]]; then + echo -e "${RED}Error: No enabled resource types in ${config_file}${RESET}" >&2 + exit 1 + fi + + # Iterate over each enabled resource type + while IFS= read -r entry; do + [[ -z "$entry" ]] && continue + + local label service operation jq_extract logical_id cfn_type rtype + label=$(echo "$entry" | jq -r '.label') + service=$(echo "$entry" | jq -r '.service') + operation=$(echo "$entry" | jq -r '.operation') + jq_extract=$(echo "$entry" | jq -r '.jq_extract') + logical_id=$(echo "$entry" | jq -r '.logical_id') + cfn_type=$(echo "$entry" | jq -r '.cfn_type') + rtype=$(echo "$entry" | jq -r '.type') + + if [[ "$JSON_OUTPUT" != "true" ]]; then + printf " Scanning %-40s\r" "${label}..." + fi + + # Execute the AWS CLI call + local result + local cmd="$AWS_CMD" + [[ -n "$PROFILE" ]] && cmd+=" --profile $PROFILE" + cmd+=" --region $REGION" + cmd+=" $service $operation --output json" + result=$(eval "$cmd" 2>/dev/null || echo "{}") + + # Extract resource IDs using the jq expression from config + while IFS= read -r rid; do + [[ -z "$rid" ]] && continue + resources=$(echo "$resources" | jq --arg id "$rid" --arg lid "$logical_id" --arg rt "$cfn_type" \ + '. += [{"LogicalResourceId":$lid,"PhysicalResourceId":$id,"ResourceType":$rt,"ResourceStatus":"DISCOVERED"}]') + done < <(echo "$result" | jq -r "$jq_extract" 2>/dev/null) + + # Special handling: VPC also discovers subnets, SGs, NAT/IGW, endpoints + if [[ "$rtype" == "vpc" ]]; then + # For each discovered VPC, get associated networking resources + while IFS= read -r vid; do + [[ -z "$vid" ]] && continue + + # Subnets + local subnets + subnets=$(aws_cmd ec2 describe-subnets --filters "Name=vpc-id,Values=$vid" --output json 2>/dev/null || echo '{"Subnets":[]}') + while IFS= read -r sid; do + [[ -z "$sid" ]] && continue + resources=$(echo "$resources" | jq --arg id "$sid" \ + '. += [{"LogicalResourceId":"Subnet","PhysicalResourceId":$id,"ResourceType":"AWS::EC2::Subnet","ResourceStatus":"DISCOVERED"}]') + done < <(echo "$subnets" | jq -r '.Subnets[].SubnetId' 2>/dev/null) + + # Security Groups (non-default) + local sgs + sgs=$(aws_cmd ec2 describe-security-groups --filters "Name=vpc-id,Values=$vid" --output json 2>/dev/null || echo '{"SecurityGroups":[]}') + while IFS= read -r sg_id; do + [[ -z "$sg_id" ]] && continue + resources=$(echo "$resources" | jq --arg id "$sg_id" \ + '. += [{"LogicalResourceId":"SecurityGroup","PhysicalResourceId":$id,"ResourceType":"AWS::EC2::SecurityGroup","ResourceStatus":"DISCOVERED"}]') + done < <(echo "$sgs" | jq -r '.SecurityGroups[] | select(.GroupName != "default") | .GroupId' 2>/dev/null) + + # NAT Gateways + local nats + nats=$(aws_cmd ec2 describe-nat-gateways --filter "Name=vpc-id,Values=$vid" "Name=state,Values=available" --output json 2>/dev/null || echo '{"NatGateways":[]}') + while IFS= read -r nid; do + [[ -z "$nid" ]] && continue + resources=$(echo "$resources" | jq --arg id "$nid" \ + '. += [{"LogicalResourceId":"NATGW","PhysicalResourceId":$id,"ResourceType":"AWS::EC2::NatGateway","ResourceStatus":"DISCOVERED"}]') + done < <(echo "$nats" | jq -r '.NatGateways[].NatGatewayId' 2>/dev/null) + + # Internet Gateways + local igws + igws=$(aws_cmd ec2 describe-internet-gateways --filters "Name=attachment.vpc-id,Values=$vid" --output json 2>/dev/null || echo '{"InternetGateways":[]}') + while IFS= read -r igid; do + [[ -z "$igid" ]] && continue + resources=$(echo "$resources" | jq --arg id "$igid" \ + '. += [{"LogicalResourceId":"IGW","PhysicalResourceId":$id,"ResourceType":"AWS::EC2::InternetGateway","ResourceStatus":"DISCOVERED"}]') + done < <(echo "$igws" | jq -r '.InternetGateways[].InternetGatewayId' 2>/dev/null) + + # VPC Endpoints + local vpces + vpces=$(aws_cmd ec2 describe-vpc-endpoints --filters "Name=vpc-id,Values=$vid" --output json 2>/dev/null || echo '{"VpcEndpoints":[]}') + while IFS= read -r eid; do + [[ -z "$eid" ]] && continue + resources=$(echo "$resources" | jq --arg id "$eid" \ + '. += [{"LogicalResourceId":"VpcEndpoint","PhysicalResourceId":$id,"ResourceType":"AWS::EC2::VPCEndpoint","ResourceStatus":"DISCOVERED"}]') + done < <(echo "$vpces" | jq -r '.VpcEndpoints[].VpcEndpointId' 2>/dev/null) + + done < <(echo "$result" | jq -r "$jq_extract" 2>/dev/null) + fi + + done <<< "$enabled_types" + + # ─── Build synthetic STACK_RESOURCES ─── + STACK_RESOURCES=$(echo "$resources" | jq '{StackResourceSummaries: .}') + + local total + total=$(echo "$resources" | jq 'length') + + if [[ "$JSON_OUTPUT" != "true" ]]; then + printf " %-50s\r" "" + echo "" + echo -e " ${GREEN}✓${RESET} Discovery complete: ${BOLD}${total} resources${RESET} found" + echo "" + if [[ "$total" -gt 0 ]]; then + print_subheader "Discovered Resources Summary" + echo "" + echo "$resources" | jq -r 'group_by(.ResourceType) | .[] | "\(.[0].ResourceType)|\(length)"' 2>/dev/null | sort | \ + while IFS='|' read -r rtype rcount; do + printf " %-55s %s\n" "$rtype" "${rcount} found" + done + echo "" + fi + fi + + # Set stack name for display purposes + STACK_NAME="[MANUAL SCAN]" +} + + +############################################################################### +# CLOUDFORMATION STACK AUDIT +############################################################################### + +audit_cloudformation() { + print_header "CLOUDFORMATION STACK" + + local stack_info + stack_info=$(aws_cmd cloudformation describe-stacks --stack-name "$STACK_NAME" --output json 2>/dev/null || echo "") + + if [[ -z "$stack_info" || "$stack_info" == "" ]]; then + print_status "Stack" "NOT FOUND" "error" + ((RESOURCES_UNHEALTHY++)) || true + return + fi + + local stack=$(echo "$stack_info" | jq '.Stacks[0]') + local status=$(json_get "$stack" '.StackStatus') + local created=$(json_get "$stack" '.CreationTime') + local updated=$(json_get "$stack" '.LastUpdatedTime') + local description=$(json_get "$stack" '.Description') + local stack_id=$(json_get "$stack" '.StackId') + + ((RESOURCES_FOUND++)) || true + ((RESOURCES_EXPECTED++)) || true + + if [[ "$status" == *"COMPLETE"* && "$status" != *"DELETE"* ]]; then + ((RESOURCES_HEALTHY++)) || true + print_status "Stack Name" "$STACK_NAME" "ok" + else + ((RESOURCES_UNHEALTHY++)) || true + print_status "Stack Name" "$STACK_NAME" "error" + fi + + print_status "Status" "$(echo -e "$(status_color "$status")")" "info" + print_status "Created" "$created" "info" + [[ -n "$updated" ]] && print_status "Last Updated" "$updated" "info" + [[ -n "$description" ]] && print_status "Description" "$description" "info" + + # Stack outputs + print_subheader "Stack Outputs" + local outputs + outputs=$(echo "$stack" | jq -r '.Outputs[]? | "\(.OutputKey)|\(.OutputValue)"' 2>/dev/null || echo "") + + if [[ -n "$outputs" ]]; then + print_table_header "Output Key" "Value" + while IFS='|' read -r key value; do + # Truncate long values + if [[ ${#value} -gt 60 ]]; then + value="${value:0:57}..." + fi + print_table_row "$key" "$value" + done <<< "$outputs" + else + print_status "Outputs" "None" "warn" + fi + + # Stack parameters + print_subheader "Stack Parameters" + local params + params=$(echo "$stack" | jq -r '.Parameters[]? | "\(.ParameterKey)|\(.ParameterValue)"' 2>/dev/null || echo "") + + if [[ -n "$params" ]]; then + print_table_header "Parameter" "Value" + while IFS='|' read -r key value; do + if [[ ${#value} -gt 60 ]]; then + value="${value:0:57}..." + fi + print_table_row "$key" "$value" + done <<< "$params" + fi + + # Detect stack drift + print_subheader "Drift Detection" + local drift_status + drift_status=$(aws_cmd cloudformation detect-stack-drift --stack-name "$STACK_NAME" --output json 2>/dev/null || echo "") + + if [[ -n "$drift_status" ]]; then + local drift_id=$(json_get "$drift_status" '.StackDriftDetectionId') + print_status "Drift Detection" "Initiated (ID: ${drift_id:0:20}...)" "info" + + # Wait briefly for drift results + sleep 2 + local drift_result + drift_result=$(aws_cmd cloudformation describe-stack-drift-detection-status \ + --stack-drift-detection-id "$drift_id" --output json 2>/dev/null || echo "") + + if [[ -n "$drift_result" ]]; then + local drift_stat=$(json_get "$drift_result" '.StackDriftStatus') + local detection_status=$(json_get "$drift_result" '.DetectionStatus') + + if [[ "$detection_status" == "DETECTION_COMPLETE" ]]; then + case "$drift_stat" in + IN_SYNC) print_status "Drift Status" "IN SYNC" "ok" ;; + DRIFTED) print_status "Drift Status" "DRIFTED" "warn" ;; + *) print_status "Drift Status" "$drift_stat" "info" ;; + esac + else + print_status "Drift Status" "Detection in progress ($detection_status)" "info" + fi + fi + else + print_status "Drift Detection" "Unable to initiate" "warn" + fi + + # Store for JSON + JSON_SECTION_CLOUDFORMATION=$(cat </dev/null || echo "{}") +} + + +############################################################################### +# VPC & NETWORKING AUDIT +############################################################################### + +audit_networking() { + print_header "VPC & NETWORKING" + + local vpc_id="" + local vpc_json="[]" + + # Try to get VPC from stack outputs or resources + vpc_id=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="VPC") | .PhysicalResourceId' 2>/dev/null || echo "") + + # VPC + print_subheader "VPC" + ((RESOURCES_EXPECTED++)) || true + if [[ -n "$vpc_id" ]]; then + ((RESOURCES_FOUND++)) || true + local vpc_info + vpc_info=$(aws_cmd ec2 describe-vpcs --vpc-ids "$vpc_id" --output json 2>/dev/null || echo "") + + if [[ -n "$vpc_info" && $(echo "$vpc_info" | jq '.Vpcs | length') -gt 0 ]]; then + ((RESOURCES_HEALTHY++)) || true + local cidr=$(echo "$vpc_info" | jq -r '.Vpcs[0].CidrBlock') + local state=$(echo "$vpc_info" | jq -r '.Vpcs[0].State') + local name=$(echo "$vpc_info" | jq -r '.Vpcs[0].Tags[]? | select(.Key=="Name") | .Value // "unnamed"' 2>/dev/null || echo "unnamed") + + print_status "VPC ID" "$vpc_id" "ok" + print_status "State" "$state" "ok" + print_status "CIDR Block" "$cidr" "info" + print_status "Name" "$name" "info" + else + ((RESOURCES_UNHEALTHY++)) || true + print_status "VPC" "$vpc_id (not accessible)" "error" + fi + else + print_status "VPC" "Not created by stack (existing-vpc template or analytics-only)" "info" + fi + + # Subnets + print_subheader "Subnets" + local subnet_ids=() + while IFS= read -r sid; do + [[ -n "$sid" ]] && subnet_ids+=("$sid") + done < <(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId | startswith("Subnet")) | .PhysicalResourceId' 2>/dev/null) + + if [[ ${#subnet_ids[@]} -gt 0 ]]; then + print_table_header "Subnet ID" "AZ" "CIDR" "Available IPs" + for subnet_id in "${subnet_ids[@]}"; do + ((RESOURCES_EXPECTED++)) || true + ((RESOURCES_FOUND++)) || true + local subnet_info + subnet_info=$(aws_cmd ec2 describe-subnets --subnet-ids "$subnet_id" --output json 2>/dev/null || echo "") + if [[ -n "$subnet_info" && $(echo "$subnet_info" | jq '.Subnets | length') -gt 0 ]]; then + ((RESOURCES_HEALTHY++)) || true + local az=$(echo "$subnet_info" | jq -r '.Subnets[0].AvailabilityZone') + local cidr=$(echo "$subnet_info" | jq -r '.Subnets[0].CidrBlock') + local avail_ips=$(echo "$subnet_info" | jq -r '.Subnets[0].AvailableIpAddressCount') + print_table_row "$subnet_id" "$az" "$cidr" "$avail_ips" + fi + done + else + print_status "Subnets" "Not created by this stack" "info" + fi + + # Security Groups + print_subheader "Security Groups" + local sg_ids=() + while IFS= read -r line; do + [[ -n "$line" ]] && sg_ids+=("$line") + done < <(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.ResourceType=="AWS::EC2::SecurityGroup") | "\(.LogicalResourceId)|\(.PhysicalResourceId)"' 2>/dev/null) + + if [[ ${#sg_ids[@]} -gt 0 ]]; then + print_table_header "Logical ID" "Group ID" "Ingress Rules" + for sg_entry in "${sg_ids[@]}"; do + IFS='|' read -r logical_id sg_id <<< "$sg_entry" + ((RESOURCES_EXPECTED++)) || true + ((RESOURCES_FOUND++)) || true + local sg_info + sg_info=$(aws_cmd ec2 describe-security-groups --group-ids "$sg_id" --output json 2>/dev/null || echo "") + if [[ -n "$sg_info" && $(echo "$sg_info" | jq '.SecurityGroups | length') -gt 0 ]]; then + ((RESOURCES_HEALTHY++)) || true + local rule_count=$(echo "$sg_info" | jq '.SecurityGroups[0].IpPermissions | length') + print_table_row "$logical_id" "$sg_id" "$rule_count rules" + fi + done + else + print_status "Security Groups" "None found in stack" "info" + fi + + # Internet Gateway + print_subheader "Internet & NAT Gateways" + local igw_id + igw_id=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="IGW") | .PhysicalResourceId' 2>/dev/null || echo "") + + if [[ -n "$igw_id" ]]; then + ((RESOURCES_EXPECTED++)) || true + ((RESOURCES_FOUND++)) || true + ((RESOURCES_HEALTHY++)) || true + print_status "Internet Gateway" "$igw_id" "ok" + else + print_status "Internet Gateway" "Not in stack" "info" + fi + + # NAT Gateway + local nat_id + nat_id=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="NATGW") | .PhysicalResourceId' 2>/dev/null || echo "") + + if [[ -n "$nat_id" ]]; then + ((RESOURCES_EXPECTED++)) || true + ((RESOURCES_FOUND++)) || true + local nat_info + nat_info=$(aws_cmd ec2 describe-nat-gateways --nat-gateway-ids "$nat_id" --output json 2>/dev/null || echo "") + if [[ -n "$nat_info" ]]; then + local nat_state=$(echo "$nat_info" | jq -r '.NatGateways[0].State') + if [[ "$nat_state" == "available" ]]; then + ((RESOURCES_HEALTHY++)) || true + print_status "NAT Gateway" "$nat_id ($nat_state)" "ok" + else + ((RESOURCES_UNHEALTHY++)) || true + print_status "NAT Gateway" "$nat_id ($nat_state)" "warn" + fi + print_status " Cost Note" "NAT Gateway: ~\$0.045/hr + data processing" "warn" + fi + else + print_status "NAT Gateway" "Not in stack" "info" + fi + + # Elastic IP + local eip_id + eip_id=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="ElasticIP1") | .PhysicalResourceId' 2>/dev/null || echo "") + + if [[ -n "$eip_id" ]]; then + ((RESOURCES_EXPECTED++)) || true + ((RESOURCES_FOUND++)) || true + ((RESOURCES_HEALTHY++)) || true + print_status "Elastic IP" "$eip_id" "ok" + print_status " Cost Note" "EIP: \$3.65/month if unattached" "info" + fi + + # VPC Endpoints + print_subheader "VPC Endpoints" + local vpce_ids=() + while IFS= read -r line; do + [[ -n "$line" ]] && vpce_ids+=("$line") + done < <(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.ResourceType | contains("VpcEndpoint")) | "\(.LogicalResourceId)|\(.PhysicalResourceId)"' 2>/dev/null) + + if [[ ${#vpce_ids[@]} -gt 0 ]]; then + for vpce_entry in "${vpce_ids[@]}"; do + IFS='|' read -r logical_id vpce_id <<< "$vpce_entry" + ((RESOURCES_EXPECTED++)) || true + ((RESOURCES_FOUND++)) || true + ((RESOURCES_HEALTHY++)) || true + print_status "$logical_id" "$vpce_id" "ok" + done + else + print_status "VPC Endpoints" "None in stack" "info" + fi +} + + +############################################################################### +# NEPTUNE DB AUDIT +############################################################################### + +audit_neptune_db() { + local cluster_id + cluster_id=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="NeptuneDBCluster") | .PhysicalResourceId' 2>/dev/null || echo "") + + if [[ -z "$cluster_id" ]]; then + return + fi + + print_header "NEPTUNE DB CLUSTER" + ((RESOURCES_EXPECTED++)) || true + + local cluster_info + cluster_info=$(aws_cmd neptune describe-db-clusters --db-cluster-identifier "$cluster_id" --output json 2>/dev/null || echo "") + + if [[ -z "$cluster_info" || $(echo "$cluster_info" | jq '.DBClusters | length') -eq 0 ]]; then + print_status "Neptune Cluster" "$cluster_id - NOT FOUND" "error" + ((RESOURCES_UNHEALTHY++)) || true + return + fi + + ((RESOURCES_FOUND++)) || true + local cluster=$(echo "$cluster_info" | jq '.DBClusters[0]') + local status=$(json_get "$cluster" '.Status') + local endpoint=$(json_get "$cluster" '.Endpoint') + local reader_endpoint=$(json_get "$cluster" '.ReaderEndpoint') + local engine_version=$(json_get "$cluster" '.EngineVersion') + local port=$(json_get "$cluster" '.Port') + local storage=$(json_get "$cluster" '.AllocatedStorage') + local serverless_config=$(echo "$cluster" | jq '.ServerlessV2ScalingConfiguration // empty' 2>/dev/null) + + if [[ "$status" == "available" ]]; then + ((RESOURCES_HEALTHY++)) || true + print_status "Cluster ID" "$cluster_id" "ok" + else + ((RESOURCES_UNHEALTHY++)) || true + print_status "Cluster ID" "$cluster_id" "warn" + fi + + print_status "Status" "$(echo -e "$(status_color "$status")")" "info" + print_status "Endpoint" "$endpoint" "info" + print_status "Reader Endpoint" "$reader_endpoint" "info" + print_status "Engine Version" "$engine_version" "info" + print_status "Port" "$port" "info" + + if [[ -n "$serverless_config" ]]; then + local min_ncu=$(json_get "$serverless_config" '.MinCapacity') + local max_ncu=$(json_get "$serverless_config" '.MaxCapacity') + print_status "Serverless Mode" "Min: ${min_ncu} NCU / Max: ${max_ncu} NCU" "info" + print_status " Cost Note" "Neptune Serverless: \$0.1028/NCU-hour" "warn" + fi + + # Neptune DB Instances + print_subheader "Neptune DB Instances" + local instance_id + instance_id=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="NeptuneDBInstance") | .PhysicalResourceId' 2>/dev/null || echo "") + + if [[ -n "$instance_id" ]]; then + ((RESOURCES_EXPECTED++)) || true + local inst_info + inst_info=$(aws_cmd neptune describe-db-instances --db-instance-identifier "$instance_id" --output json 2>/dev/null || echo "") + + if [[ -n "$inst_info" && $(echo "$inst_info" | jq '.DBInstances | length') -gt 0 ]]; then + ((RESOURCES_FOUND++)) || true + local inst=$(echo "$inst_info" | jq '.DBInstances[0]') + local inst_status=$(json_get "$inst" '.DBInstanceStatus') + local inst_class=$(json_get "$inst" '.DBInstanceClass') + local inst_az=$(json_get "$inst" '.AvailabilityZone') + + if [[ "$inst_status" == "available" ]]; then + ((RESOURCES_HEALTHY++)) || true + else + ((RESOURCES_UNHEALTHY++)) || true + fi + + print_status "Instance ID" "$instance_id" "ok" + print_status "Status" "$(echo -e "$(status_color "$inst_status")")" "info" + print_status "Instance Class" "$inst_class" "info" + print_status "AZ" "$inst_az" "info" + + if [[ "$inst_class" == "db.serverless" ]]; then + print_status " Cost Note" "Serverless - pay per NCU-hour" "info" + else + print_status " Cost Note" "Instance $inst_class running 24/7" "warn" + fi + fi + fi + + # Parameter Groups + local param_group + param_group=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="NeptuneDBClusterParameterGroup") | .PhysicalResourceId' 2>/dev/null || echo "") + if [[ -n "$param_group" ]]; then + ((RESOURCES_EXPECTED++)) || true + ((RESOURCES_FOUND++)) || true + ((RESOURCES_HEALTHY++)) || true + print_status "Cluster Param Group" "$param_group" "ok" + fi + + local db_param_group + db_param_group=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="NeptuneDBParameterGroup") | .PhysicalResourceId' 2>/dev/null || echo "") + if [[ -n "$db_param_group" ]]; then + ((RESOURCES_EXPECTED++)) || true + ((RESOURCES_FOUND++)) || true + ((RESOURCES_HEALTHY++)) || true + print_status "DB Param Group" "$db_param_group" "ok" + fi + + # Subnet Group + local subnet_group + subnet_group=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="NeptuneDBSubnetGroup") | .PhysicalResourceId' 2>/dev/null || echo "") + if [[ -n "$subnet_group" ]]; then + ((RESOURCES_EXPECTED++)) || true + ((RESOURCES_FOUND++)) || true + ((RESOURCES_HEALTHY++)) || true + print_status "Subnet Group" "$subnet_group" "ok" + fi +} + +############################################################################### +# NEPTUNE ANALYTICS AUDIT +############################################################################### + +audit_neptune_analytics() { + local graph_id + graph_id=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="Graph") | .PhysicalResourceId' 2>/dev/null || echo "") + + if [[ -z "$graph_id" ]]; then + return + fi + + print_header "NEPTUNE ANALYTICS GRAPH" + ((RESOURCES_EXPECTED++)) || true + + local graph_info + graph_info=$(aws_cmd neptune-graph get-graph --graph-identifier "$graph_id" --output json 2>/dev/null || echo "") + + if [[ -z "$graph_info" ]]; then + print_status "Neptune Graph" "$graph_id - NOT FOUND" "error" + ((RESOURCES_UNHEALTHY++)) || true + return + fi + + ((RESOURCES_FOUND++)) || true + local status=$(json_get "$graph_info" '.status') + local name=$(json_get "$graph_info" '.name') + local memory=$(json_get "$graph_info" '.provisionedMemory') + local endpoint=$(json_get "$graph_info" '.endpoint') + local vector_search=$(json_get "$graph_info" '.vectorSearchConfiguration.dimension') + local public_access=$(json_get "$graph_info" '.publicConnectivity') + + if [[ "$status" == "AVAILABLE" ]]; then + ((RESOURCES_HEALTHY++)) || true + print_status "Graph Name" "$name" "ok" + else + ((RESOURCES_UNHEALTHY++)) || true + print_status "Graph Name" "$name" "warn" + fi + + print_status "Graph ID" "$graph_id" "info" + print_status "Status" "$(echo -e "$(status_color "$status")")" "info" + print_status "Provisioned Memory" "${memory} GB" "info" + print_status "Endpoint" "$endpoint" "info" + [[ -n "$vector_search" ]] && print_status "Vector Dimensions" "$vector_search" "info" + print_status "Public Access" "$public_access" "info" + print_status " Cost Note" "Neptune Analytics: ~\$0.11/GB-hour (${memory}GB = ~\$$(echo "$memory * 0.11" | bc 2>/dev/null || echo "N/A")/hr)" "warn" +} + + +############################################################################### +# OPENSEARCH SERVERLESS AUDIT +############################################################################### + +audit_opensearch() { + local collection_id + collection_id=$(echo "$STACK_RESOURCES" | jq -r '[.StackResourceSummaries[]? | select(.LogicalResourceId=="OpenSearchServerless") | .PhysicalResourceId] | first // empty' 2>/dev/null || echo "") + + if [[ -z "$collection_id" ]]; then + return + fi + + print_header "OPENSEARCH SERVERLESS" + + # In manual mode, there may be multiple collections — audit each one + local all_collection_ids=() + while IFS= read -r cid; do + [[ -n "$cid" ]] && all_collection_ids+=("$cid") + done < <(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="OpenSearchServerless") | .PhysicalResourceId' 2>/dev/null) + + for collection_id in "${all_collection_ids[@]+"${all_collection_ids[@]}"}"; do + [[ -z "$collection_id" ]] && continue + ((RESOURCES_EXPECTED++)) || true + + # Get collection details + local collection_info + collection_info=$(aws_cmd opensearchserverless batch-get-collection --ids "$collection_id" --output json 2>/dev/null || echo "") + + if [[ -z "$collection_info" || $(echo "$collection_info" | jq '.collectionDetails | length') -eq 0 ]]; then + print_status "Collection" "$collection_id - NOT FOUND" "error" + ((RESOURCES_UNHEALTHY++)) || true + continue + fi + + ((RESOURCES_FOUND++)) || true + local collection=$(echo "$collection_info" | jq '.collectionDetails[0]') + local status=$(json_get "$collection" '.status') + local name=$(json_get "$collection" '.name') + local coll_type=$(json_get "$collection" '.type') + local endpoint=$(json_get "$collection" '.collectionEndpoint') + local dashboard=$(json_get "$collection" '.dashboardEndpoint') + local arn=$(json_get "$collection" '.arn') + + if [[ "$status" == "ACTIVE" ]]; then + ((RESOURCES_HEALTHY++)) || true + print_status "Collection Name" "$name" "ok" + else + ((RESOURCES_UNHEALTHY++)) || true + print_status "Collection Name" "$name" "warn" + fi + + print_status "Collection ID" "$collection_id" "info" + print_status "Status" "$(echo -e "$(status_color "$status")")" "info" + print_status "Type" "$coll_type" "info" + print_status "Endpoint" "$endpoint" "info" + print_status "Dashboard" "$dashboard" "info" + print_status " Cost Note" "OpenSearch Serverless: ~\$0.24/OCU-hour (min 2 OCUs indexing + 2 OCUs search)" "warn" + echo "" + done + + # Security Policies + print_subheader "Security & Access Policies" + local policy_ids=() + while IFS= read -r line; do + [[ -n "$line" ]] && policy_ids+=("$line") + done < <(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.ResourceType | contains("OpenSearchServerless")) | select(.LogicalResourceId != "OpenSearchServerless") | "\(.LogicalResourceId)|\(.PhysicalResourceId)"' 2>/dev/null) + + for policy_entry in "${policy_ids[@]+"${policy_ids[@]}"}"; do + IFS='|' read -r logical_id phys_id <<< "$policy_entry" + ((RESOURCES_EXPECTED++)) || true + ((RESOURCES_FOUND++)) || true + ((RESOURCES_HEALTHY++)) || true + print_status "$logical_id" "${phys_id:0:50}" "ok" + done + + # VPC Endpoint for OpenSearch + local vpce_id + vpce_id=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="OpenSearchServerlessVpcEndpoint") | .PhysicalResourceId' 2>/dev/null || echo "") + if [[ -n "$vpce_id" ]]; then + ((RESOURCES_EXPECTED++)) || true + ((RESOURCES_FOUND++)) || true + ((RESOURCES_HEALTHY++)) || true + print_status "VPC Endpoint" "$vpce_id" "ok" + fi +} + +############################################################################### +# AURORA POSTGRESQL AUDIT +############################################################################### + +audit_aurora_postgres() { + local cluster_id + cluster_id=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="PostgresCluster") | .PhysicalResourceId' 2>/dev/null || echo "") + + if [[ -z "$cluster_id" ]]; then + return + fi + + print_header "AURORA POSTGRESQL" + ((RESOURCES_EXPECTED++)) || true + + local cluster_info + cluster_info=$(aws_cmd rds describe-db-clusters --db-cluster-identifier "$cluster_id" --output json 2>/dev/null || echo "") + + if [[ -z "$cluster_info" || $(echo "$cluster_info" | jq '.DBClusters | length') -eq 0 ]]; then + print_status "Aurora Cluster" "$cluster_id - NOT FOUND" "error" + ((RESOURCES_UNHEALTHY++)) || true + return + fi + + ((RESOURCES_FOUND++)) || true + local cluster=$(echo "$cluster_info" | jq '.DBClusters[0]') + local status=$(json_get "$cluster" '.Status') + local endpoint=$(json_get "$cluster" '.Endpoint') + local reader_endpoint=$(json_get "$cluster" '.ReaderEndpoint') + local engine=$(json_get "$cluster" '.Engine') + local engine_version=$(json_get "$cluster" '.EngineVersion') + local port=$(json_get "$cluster" '.Port') + local serverless_config=$(echo "$cluster" | jq '.ServerlessV2ScalingConfiguration // empty' 2>/dev/null) + + if [[ "$status" == "available" ]]; then + ((RESOURCES_HEALTHY++)) || true + print_status "Cluster ID" "$cluster_id" "ok" + else + ((RESOURCES_UNHEALTHY++)) || true + print_status "Cluster ID" "$cluster_id" "warn" + fi + + print_status "Status" "$(echo -e "$(status_color "$status")")" "info" + print_status "Engine" "$engine $engine_version" "info" + print_status "Endpoint" "$endpoint" "info" + print_status "Reader Endpoint" "$reader_endpoint" "info" + print_status "Port" "$port" "info" + + if [[ -n "$serverless_config" ]]; then + local min_acu=$(json_get "$serverless_config" '.MinCapacity') + local max_acu=$(json_get "$serverless_config" '.MaxCapacity') + print_status "Serverless Mode" "Min: ${min_acu} ACU / Max: ${max_acu} ACU" "info" + print_status " Cost Note" "Aurora Serverless v2: \$0.12/ACU-hour" "warn" + fi + + # Aurora Instances + print_subheader "Aurora DB Instances" + local instance_id + instance_id=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="PostgresInstance") | .PhysicalResourceId' 2>/dev/null || echo "") + + if [[ -n "$instance_id" ]]; then + ((RESOURCES_EXPECTED++)) || true + local inst_info + inst_info=$(aws_cmd rds describe-db-instances --db-instance-identifier "$instance_id" --output json 2>/dev/null || echo "") + + if [[ -n "$inst_info" && $(echo "$inst_info" | jq '.DBInstances | length') -gt 0 ]]; then + ((RESOURCES_FOUND++)) || true + local inst=$(echo "$inst_info" | jq '.DBInstances[0]') + local inst_status=$(json_get "$inst" '.DBInstanceStatus') + local inst_class=$(json_get "$inst" '.DBInstanceClass') + local inst_az=$(json_get "$inst" '.AvailabilityZone') + local storage_type=$(json_get "$inst" '.StorageType') + + if [[ "$inst_status" == "available" ]]; then + ((RESOURCES_HEALTHY++)) || true + else + ((RESOURCES_UNHEALTHY++)) || true + fi + + print_status "Instance ID" "$instance_id" "ok" + print_status "Status" "$(echo -e "$(status_color "$inst_status")")" "info" + print_status "Instance Class" "$inst_class" "info" + print_status "AZ" "$inst_az" "info" + print_status "Storage Type" "$storage_type" "info" + fi + fi + + # Parameter Groups and Subnet Group + local pg_cluster + pg_cluster=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="PostgresClusterParameterGroup") | .PhysicalResourceId' 2>/dev/null || echo "") + [[ -n "$pg_cluster" ]] && { ((RESOURCES_EXPECTED++)); ((RESOURCES_FOUND++)); ((RESOURCES_HEALTHY++)); print_status "Cluster Param Group" "$pg_cluster" "ok"; } || true + + local pg_db + pg_db=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="PostgresDBParameterGroup") | .PhysicalResourceId' 2>/dev/null || echo "") + [[ -n "$pg_db" ]] && { ((RESOURCES_EXPECTED++)); ((RESOURCES_FOUND++)); ((RESOURCES_HEALTHY++)); print_status "DB Param Group" "$pg_db" "ok"; } || true + + local pg_subnet + pg_subnet=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="PostgresSubnetGroup") | .PhysicalResourceId' 2>/dev/null || echo "") + [[ -n "$pg_subnet" ]] && { ((RESOURCES_EXPECTED++)); ((RESOURCES_FOUND++)); ((RESOURCES_HEALTHY++)); print_status "Subnet Group" "$pg_subnet" "ok"; } || true +} + +############################################################################### +# S3 VECTORS AUDIT +############################################################################### + +audit_s3_vectors() { + local bucket_id + bucket_id=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="VectorBucket") | .PhysicalResourceId' 2>/dev/null || echo "") + + if [[ -z "$bucket_id" ]]; then + return + fi + + print_header "S3 VECTORS" + ((RESOURCES_EXPECTED++)) || true + + # S3 Vectors is a newer service; check via the resource status + local resource_status + resource_status=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="VectorBucket") | .ResourceStatus' 2>/dev/null || echo "") + + if [[ "$resource_status" == "CREATE_COMPLETE" || "$resource_status" == "UPDATE_COMPLETE" ]]; then + ((RESOURCES_FOUND++)) || true + ((RESOURCES_HEALTHY++)) || true + print_status "Vector Bucket" "$bucket_id" "ok" + print_status "Resource Status" "$resource_status" "ok" + else + ((RESOURCES_FOUND++)) || true + ((RESOURCES_UNHEALTHY++)) || true + print_status "Vector Bucket" "$bucket_id ($resource_status)" "warn" + fi + + # KMS Key + print_subheader "KMS Encryption" + local kms_key + kms_key=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="VectorBucketKMSKey") | .PhysicalResourceId' 2>/dev/null || echo "") + + if [[ -n "$kms_key" ]]; then + ((RESOURCES_EXPECTED++)) || true + ((RESOURCES_FOUND++)) || true + + local key_info + key_info=$(aws_cmd kms describe-key --key-id "$kms_key" --output json 2>/dev/null || echo "") + + if [[ -n "$key_info" ]]; then + local key_state=$(echo "$key_info" | jq -r '.KeyMetadata.KeyState') + local key_spec=$(echo "$key_info" | jq -r '.KeyMetadata.KeySpec') + + if [[ "$key_state" == "Enabled" ]]; then + ((RESOURCES_HEALTHY++)) || true + print_status "KMS Key" "$kms_key" "ok" + else + ((RESOURCES_UNHEALTHY++)) || true + print_status "KMS Key" "$kms_key ($key_state)" "warn" + fi + print_status "Key State" "$key_state" "info" + print_status "Key Spec" "$key_spec" "info" + print_status " Cost Note" "KMS: \$1/month/key + \$0.03/10K requests" "info" + fi + fi +} + + +############################################################################### +# SAGEMAKER NOTEBOOK AUDIT +############################################################################### + +audit_sagemaker() { + local notebook_id + notebook_id=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="NeptuneNotebookInstance") | .PhysicalResourceId' 2>/dev/null || echo "") + + if [[ -z "$notebook_id" ]]; then + return + fi + + print_header "SAGEMAKER NOTEBOOK" + ((RESOURCES_EXPECTED++)) || true + + local notebook_info + notebook_info=$(aws_cmd sagemaker describe-notebook-instance --notebook-instance-name "$notebook_id" --output json 2>/dev/null || echo "") + + if [[ -z "$notebook_info" ]]; then + print_status "Notebook" "$notebook_id - NOT FOUND" "error" + ((RESOURCES_UNHEALTHY++)) || true + return + fi + + ((RESOURCES_FOUND++)) || true + local status=$(json_get "$notebook_info" '.NotebookInstanceStatus') + local instance_type=$(json_get "$notebook_info" '.InstanceType') + local url=$(json_get "$notebook_info" '.Url') + local volume_size=$(json_get "$notebook_info" '.VolumeSizeInGB') + local direct_access=$(json_get "$notebook_info" '.DirectInternetAccess') + local role_arn=$(json_get "$notebook_info" '.RoleArn') + + if [[ "$status" == "InService" ]]; then + ((RESOURCES_HEALTHY++)) || true + print_status "Notebook Name" "$notebook_id" "ok" + print_status " Cost Note" "RUNNING - ${instance_type} incurring charges" "warn" + elif [[ "$status" == "Stopped" ]]; then + ((RESOURCES_HEALTHY++)) || true + print_status "Notebook Name" "$notebook_id" "ok" + print_status " Cost Note" "Stopped - only storage charges (\$${volume_size:-5}GB EBS)" "info" + else + ((RESOURCES_UNHEALTHY++)) || true + print_status "Notebook Name" "$notebook_id" "warn" + fi + + print_status "Status" "$(echo -e "$(status_color "$status")")" "info" + print_status "Instance Type" "$instance_type" "info" + print_status "Volume Size" "${volume_size}GB" "info" + print_status "URL" "$url" "info" + print_status "Direct Internet" "$direct_access" "info" + print_status "IAM Role" "$(echo "$role_arn" | awk -F'/' '{print $NF}')" "info" + + # Lifecycle Config + local lifecycle_id + lifecycle_id=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="NeptuneNotebookInstanceLifecycleConfig") | .PhysicalResourceId' 2>/dev/null || echo "") + if [[ -n "$lifecycle_id" ]]; then + ((RESOURCES_EXPECTED++)) || true + ((RESOURCES_FOUND++)) || true + ((RESOURCES_HEALTHY++)) || true + print_status "Lifecycle Config" "$lifecycle_id" "ok" + fi +} + +############################################################################### +# IAM ROLES & POLICIES AUDIT +############################################################################### + +audit_iam() { + print_header "IAM ROLES & POLICIES" + + # Roles + print_subheader "IAM Roles" + local roles=() + while IFS= read -r line; do + [[ -n "$line" ]] && roles+=("$line") + done < <(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.ResourceType=="AWS::IAM::Role") | "\(.LogicalResourceId)|\(.PhysicalResourceId)"' 2>/dev/null) + + if [[ ${#roles[@]} -gt 0 ]]; then + print_table_header "Logical ID" "Role Name" "Status" + for role_entry in "${roles[@]}"; do + IFS='|' read -r logical_id role_name <<< "$role_entry" + ((RESOURCES_EXPECTED++)) || true + + local role_info + role_info=$(aws_cmd iam get-role --role-name "$role_name" --output json 2>/dev/null || echo "") + + if [[ -n "$role_info" ]]; then + ((RESOURCES_FOUND++)) || true + ((RESOURCES_HEALTHY++)) || true + local create_date=$(echo "$role_info" | jq -r '.Role.CreateDate' 2>/dev/null) + print_table_row "$logical_id" "${role_name:0:40}" "${GREEN}EXISTS${RESET}" + else + ((RESOURCES_UNHEALTHY++)) || true + print_table_row "$logical_id" "${role_name:0:40}" "${RED}NOT FOUND${RESET}" + fi + done + else + print_status "IAM Roles" "None found in stack" "info" + fi + + # Managed Policies + print_subheader "IAM Managed Policies" + local policies=() + while IFS= read -r line; do + [[ -n "$line" ]] && policies+=("$line") + done < <(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.ResourceType=="AWS::IAM::ManagedPolicy") | "\(.LogicalResourceId)|\(.PhysicalResourceId)"' 2>/dev/null) + + if [[ ${#policies[@]} -gt 0 ]]; then + print_table_header "Policy Name" "ARN (truncated)" + for policy_entry in "${policies[@]}"; do + IFS='|' read -r logical_id policy_arn <<< "$policy_entry" + ((RESOURCES_EXPECTED++)) || true + ((RESOURCES_FOUND++)) || true + ((RESOURCES_HEALTHY++)) || true + + # Truncate ARN for display + local display_arn="${policy_arn}" + if [[ ${#display_arn} -gt 70 ]]; then + display_arn="...${display_arn: -67}" + fi + print_table_row "$logical_id" "$display_arn" + done + else + print_status "Managed Policies" "None found in stack" "info" + fi +} + +############################################################################### +# LAMBDA FUNCTIONS AUDIT +############################################################################### + +audit_lambda() { + local lambdas=() + while IFS= read -r line; do + [[ -n "$line" ]] && lambdas+=("$line") + done < <(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.ResourceType=="AWS::Lambda::Function") | "\(.LogicalResourceId)|\(.PhysicalResourceId)"' 2>/dev/null) + + if [[ ${#lambdas[@]} -eq 0 ]]; then + return + fi + + print_header "LAMBDA FUNCTIONS" + + for lambda_entry in "${lambdas[@]}"; do + IFS='|' read -r logical_id function_name <<< "$lambda_entry" + ((RESOURCES_EXPECTED++)) || true + + local func_info + func_info=$(aws_cmd lambda get-function --function-name "$function_name" --output json 2>/dev/null || echo "") + + if [[ -z "$func_info" ]]; then + ((RESOURCES_UNHEALTHY++)) || true + print_status "$logical_id" "$function_name - NOT FOUND" "error" + continue + fi + + ((RESOURCES_FOUND++)) || true + local config=$(echo "$func_info" | jq '.Configuration') + local state=$(json_get "$config" '.State') + local runtime=$(json_get "$config" '.Runtime') + local memory=$(json_get "$config" '.MemorySize') + local timeout=$(json_get "$config" '.Timeout') + local last_modified=$(json_get "$config" '.LastModified') + local code_size=$(json_get "$config" '.CodeSize') + + if [[ "$state" == "Active" ]]; then + ((RESOURCES_HEALTHY++)) || true + print_status "$logical_id" "$function_name" "ok" + else + ((RESOURCES_UNHEALTHY++)) || true + print_status "$logical_id" "$function_name ($state)" "warn" + fi + + print_status " Runtime" "$runtime" "info" + print_status " Memory" "${memory}MB / Timeout: ${timeout}s" "info" + print_status " Code Size" "$(echo "$code_size" | awk '{printf "%.1f KB", $1/1024}')" "info" + print_status " Last Modified" "$last_modified" "info" + done +} + + +############################################################################### +# ADDITIONAL RESOURCES AUDIT (Route Tables, Custom Resources) +############################################################################### + +audit_additional_resources() { + # Check for any resources we haven't explicitly covered + local other_resources=() + while IFS= read -r line; do + [[ -n "$line" ]] && other_resources+=("$line") + done < <(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select( + .ResourceType != "AWS::EC2::VPC" and + .ResourceType != "AWS::EC2::Subnet" and + .ResourceType != "AWS::EC2::InternetGateway" and + .ResourceType != "AWS::EC2::VPCGatewayAttachment" and + .ResourceType != "AWS::EC2::NatGateway" and + .ResourceType != "AWS::EC2::EIP" and + .ResourceType != "AWS::EC2::SecurityGroup" and + .ResourceType != "AWS::EC2::SecurityGroupIngress" and + .ResourceType != "AWS::EC2::SecurityGroupEgress" and + .ResourceType != "AWS::EC2::RouteTable" and + .ResourceType != "AWS::EC2::Route" and + .ResourceType != "AWS::EC2::SubnetRouteTableAssociation" and + .ResourceType != "AWS::Neptune::DBCluster" and + .ResourceType != "AWS::Neptune::DBInstance" and + .ResourceType != "AWS::Neptune::DBClusterParameterGroup" and + .ResourceType != "AWS::Neptune::DBParameterGroup" and + .ResourceType != "AWS::Neptune::DBSubnetGroup" and + .ResourceType != "AWS::NeptuneGraph::Graph" and + .ResourceType != "AWS::OpenSearchServerless::Collection" and + .ResourceType != "AWS::OpenSearchServerless::AccessPolicy" and + .ResourceType != "AWS::OpenSearchServerless::SecurityPolicy" and + .ResourceType != "AWS::OpenSearchServerless::VpcEndpoint" and + .ResourceType != "AWS::RDS::DBCluster" and + .ResourceType != "AWS::RDS::DBInstance" and + .ResourceType != "AWS::RDS::DBClusterParameterGroup" and + .ResourceType != "AWS::RDS::DBParameterGroup" and + .ResourceType != "AWS::RDS::DBSubnetGroup" and + .ResourceType != "AWS::S3Vectors::VectorBucket" and + .ResourceType != "AWS::KMS::Key" and + .ResourceType != "AWS::SageMaker::NotebookInstance" and + .ResourceType != "AWS::SageMaker::NotebookInstanceLifecycleConfig" and + .ResourceType != "AWS::IAM::Role" and + .ResourceType != "AWS::IAM::ManagedPolicy" and + .ResourceType != "AWS::Lambda::Function" and + .ResourceType != "Custom::CustomResource" + ) | "\(.ResourceType)|\(.LogicalResourceId)|\(.PhysicalResourceId)|\(.ResourceStatus)"' 2>/dev/null) + + if [[ ${#other_resources[@]} -gt 0 ]]; then + print_header "ADDITIONAL RESOURCES" + print_table_header "Type" "Logical ID" "Status" + for res_entry in "${other_resources[@]}"; do + IFS='|' read -r res_type logical_id phys_id res_status <<< "$res_entry" + ((RESOURCES_EXPECTED++)) || true + ((RESOURCES_FOUND++)) || true + if [[ "$res_status" == *"COMPLETE"* ]]; then + ((RESOURCES_HEALTHY++)) || true + else + ((RESOURCES_UNKNOWN++)) || true + fi + # Shorten type for display + local short_type="${res_type#AWS::}" + print_table_row "$short_type" "$logical_id" "$(echo -e "$(status_color "$res_status")")" + done + fi + + # Custom Resources + local custom_resources=() + while IFS= read -r line; do + [[ -n "$line" ]] && custom_resources+=("$line") + done < <(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.ResourceType=="Custom::CustomResource") | "\(.LogicalResourceId)|\(.PhysicalResourceId)|\(.ResourceStatus)"' 2>/dev/null) + + if [[ ${#custom_resources[@]} -gt 0 ]]; then + print_subheader "Custom Resources (Lambda-backed)" + for cr_entry in "${custom_resources[@]}"; do + IFS='|' read -r logical_id phys_id cr_status <<< "$cr_entry" + ((RESOURCES_EXPECTED++)) || true + ((RESOURCES_FOUND++)) || true + if [[ "$cr_status" == *"COMPLETE"* ]]; then + ((RESOURCES_HEALTHY++)) || true + fi + print_status "$logical_id" "$cr_status" "ok" + done + fi +} + +############################################################################### +# SUMMARY +############################################################################### + +print_summary() { + print_header "AUDIT SUMMARY" + + # Resource count from stack + local total_stack_resources + total_stack_resources=$(echo "$STACK_RESOURCES" | jq '.StackResourceSummaries | length' 2>/dev/null || echo "0") + + echo "" + echo -e " ${BOLD}Resource Overview${RESET}" + echo -e " ─────────────────────────────────────────" + printf " %-30s %s\n" "Stack Resources (total):" "$total_stack_resources" + printf " %-30s ${GREEN}%s${RESET}\n" "Resources Verified:" "$RESOURCES_FOUND" + printf " %-30s ${GREEN}%s${RESET}\n" "Healthy:" "$RESOURCES_HEALTHY" + printf " %-30s ${RED}%s${RESET}\n" "Unhealthy/Missing:" "$RESOURCES_UNHEALTHY" + printf " %-30s ${YELLOW}%s${RESET}\n" "Unknown:" "$RESOURCES_UNKNOWN" + echo "" + + # Health percentage + local health_pct=0 + if [[ $RESOURCES_FOUND -gt 0 ]]; then + health_pct=$(( (RESOURCES_HEALTHY * 100) / RESOURCES_FOUND )) + fi + + echo -e " ${BOLD}Health Score${RESET}" + echo -e " ─────────────────────────────────────────" + if [[ $health_pct -ge 90 ]]; then + echo -e " ${GREEN}██████████ ${health_pct}% - HEALTHY${RESET}" + elif [[ $health_pct -ge 70 ]]; then + echo -e " ${YELLOW}███████░░░ ${health_pct}% - DEGRADED${RESET}" + else + echo -e " ${RED}████░░░░░░ ${health_pct}% - UNHEALTHY${RESET}" + fi + echo "" + + # Cost indicators + echo -e " ${BOLD}Cost Indicators (estimated hourly when running)${RESET}" + echo -e " ─────────────────────────────────────────" + + local has_cost_items=false + + # Check what's running + local notebook_status + notebook_status=$(aws_cmd sagemaker describe-notebook-instance \ + --notebook-instance-name "$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="NeptuneNotebookInstance") | .PhysicalResourceId' 2>/dev/null)" \ + --query 'NotebookInstanceStatus' --output text 2>/dev/null || echo "") + + if [[ "$notebook_status" == "InService" ]]; then + printf " ${YELLOW}⚠${RESET} %-35s %s\n" "SageMaker Notebook:" "RUNNING (ml.m5.xlarge ~\$0.23/hr)" + has_cost_items=true + fi + + # Neptune Analytics + local graph_id + graph_id=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="Graph") | .PhysicalResourceId' 2>/dev/null || echo "") + if [[ -n "$graph_id" ]]; then + printf " ${YELLOW}⚠${RESET} %-35s %s\n" "Neptune Analytics Graph:" "Provisioned memory charges apply" + has_cost_items=true + fi + + # Neptune DB + local neptune_cluster + neptune_cluster=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="NeptuneDBCluster") | .PhysicalResourceId' 2>/dev/null || echo "") + if [[ -n "$neptune_cluster" ]]; then + printf " ${YELLOW}⚠${RESET} %-35s %s\n" "Neptune DB Cluster:" "Instance/serverless charges apply" + has_cost_items=true + fi + + # OpenSearch + local opensearch_col + opensearch_col=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="OpenSearchServerless") | .PhysicalResourceId' 2>/dev/null || echo "") + if [[ -n "$opensearch_col" ]]; then + printf " ${YELLOW}⚠${RESET} %-35s %s\n" "OpenSearch Serverless:" "Min 4 OCUs (~\$0.96/hr)" + has_cost_items=true + fi + + # Aurora + local aurora_cluster + aurora_cluster=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="PostgresCluster") | .PhysicalResourceId' 2>/dev/null || echo "") + if [[ -n "$aurora_cluster" ]]; then + printf " ${YELLOW}⚠${RESET} %-35s %s\n" "Aurora PostgreSQL:" "Serverless/instance charges apply" + has_cost_items=true + fi + + # NAT Gateway + local nat_gw + nat_gw=$(echo "$STACK_RESOURCES" | jq -r '.StackResourceSummaries[]? | select(.LogicalResourceId=="NATGW") | .PhysicalResourceId' 2>/dev/null || echo "") + if [[ -n "$nat_gw" ]]; then + printf " ${YELLOW}⚠${RESET} %-35s %s\n" "NAT Gateway:" "~\$0.045/hr + data" + has_cost_items=true + fi + + if [[ "$has_cost_items" == "false" ]]; then + printf " ${GREEN}✓${RESET} No active cost-incurring resources detected\n" + fi + + echo "" + echo -e " ${DIM}Audit completed at: $(date -u '+%Y-%m-%d %H:%M:%S UTC')${RESET}" + echo -e " ${DIM}Stack: ${STACK_NAME} | Region: ${REGION}${RESET}" + echo "" +} + +############################################################################### +# JSON OUTPUT +############################################################################### + +output_json() { + local total_stack_resources + total_stack_resources=$(echo "$STACK_RESOURCES" | jq '.StackResourceSummaries | length' 2>/dev/null || echo "0") + + local health_pct=0 + if [[ $RESOURCES_FOUND -gt 0 ]]; then + health_pct=$(( (RESOURCES_HEALTHY * 100) / RESOURCES_FOUND )) + fi + + # Build full JSON output + cat < /dev/null; then + echo -e "${RED}Error: AWS CLI not found. Please install it first.${RESET}" >&2 + exit 1 + fi + + # Verify jq is available + if ! command -v jq &> /dev/null; then + echo -e "${RED}Error: jq not found. Please install it (brew install jq / apt install jq).${RESET}" >&2 + exit 1 + fi + + # Verify AWS credentials + if ! aws_cmd sts get-caller-identity &> /dev/null; then + echo -e "${RED}Error: Unable to authenticate with AWS. Check your credentials/profile.${RESET}" >&2 + exit 1 + fi + + if [[ "$JSON_OUTPUT" != "true" ]]; then + echo "" + echo -e "${BOLD}${MAGENTA}╔══════════════════════════════════════════════════════════════════════════════╗${RESET}" + echo -e "${BOLD}${MAGENTA}║ GraphRAG Toolkit - Deployment Audit Script v${VERSION} ║${RESET}" + echo -e "${BOLD}${MAGENTA}╚══════════════════════════════════════════════════════════════════════════════╝${RESET}" + echo "" + local mode_label="Stack" + [[ "$MANUAL_MODE" == "true" ]] && mode_label="Manual Scan" + echo -e " ${DIM}Region: ${REGION} | Profile: ${PROFILE:-default} | Mode: ${mode_label}${RESET}" + fi + + if [[ "$MANUAL_MODE" == "true" ]]; then + # Manual mode: scan AWS directly without needing a CloudFormation stack + discover_resources_manual + + # Skip CloudFormation audit, go straight to resource audits + audit_networking + audit_neptune_db + audit_neptune_analytics + audit_opensearch + audit_aurora_postgres + audit_s3_vectors + audit_sagemaker + audit_iam + audit_lambda + audit_additional_resources + else + # Stack mode: detect or use provided stack name + detect_stack + + # Run all audit sections + audit_cloudformation + audit_networking + audit_neptune_db + audit_neptune_analytics + audit_opensearch + audit_aurora_postgres + audit_s3_vectors + audit_sagemaker + audit_iam + audit_lambda + audit_additional_resources + fi + + # Output results + if [[ "$JSON_OUTPUT" == "true" ]]; then + output_json + else + print_summary + fi +} + +# Run main with all arguments +main "$@" diff --git a/examples/lexical-graph/cloudformation-templates/audit/audit-resources-recommended.json b/examples/lexical-graph/cloudformation-templates/audit/audit-resources-recommended.json new file mode 100644 index 000000000..d81048505 --- /dev/null +++ b/examples/lexical-graph/cloudformation-templates/audit/audit-resources-recommended.json @@ -0,0 +1,291 @@ +{ + "description": "Recommended resource scan for graphrag-toolkit CloudFormation templates. Covers ALL resource types defined across the 8 template variants (neptune-analytics, neptune-db, opensearch, aurora-postgres, s3-vectors).", + "source_templates": [ + "graphrag-toolkit-neptune-analytics.json", + "graphrag-toolkit-neptune-analytics-opensearch-serverless.json", + "graphrag-toolkit-neptune-analytics-s3-vectors.json", + "graphrag-toolkit-neptune-analytics-aurora-postgres.json", + "graphrag-toolkit-neptune-db-opensearch-serverless.json", + "graphrag-toolkit-neptune-db-s3-vectors.json", + "graphrag-toolkit-neptune-db-aurora-postgres.json", + "graphrag-toolkit-neptune-db-aurora-postgres-existing-vpc.json" + ], + "resource_types": [ + { + "type": "neptune_clusters", + "label": "Neptune DB Clusters", + "enabled": true, + "service": "neptune", + "operation": "describe-db-clusters", + "jq_extract": ".DBClusters[].DBClusterIdentifier", + "logical_id": "NeptuneDBCluster", + "cfn_type": "AWS::Neptune::DBCluster", + "templates": ["neptune-db-*"] + }, + { + "type": "neptune_instances", + "label": "Neptune DB Instances", + "enabled": true, + "service": "neptune", + "operation": "describe-db-instances", + "jq_extract": ".DBInstances[].DBInstanceIdentifier", + "logical_id": "NeptuneDBInstance", + "cfn_type": "AWS::Neptune::DBInstance", + "templates": ["neptune-db-*"] + }, + { + "type": "neptune_cluster_parameter_groups", + "label": "Neptune Cluster Parameter Groups", + "enabled": true, + "service": "neptune", + "operation": "describe-db-cluster-parameter-groups", + "jq_extract": ".DBClusterParameterGroups[].DBClusterParameterGroupName", + "logical_id": "NeptuneDBClusterParameterGroup", + "cfn_type": "AWS::Neptune::DBClusterParameterGroup", + "templates": ["neptune-db-*"] + }, + { + "type": "neptune_parameter_groups", + "label": "Neptune DB Parameter Groups", + "enabled": true, + "service": "neptune", + "operation": "describe-db-parameter-groups", + "jq_extract": ".DBParameterGroups[].DBParameterGroupName", + "logical_id": "NeptuneDBParameterGroup", + "cfn_type": "AWS::Neptune::DBParameterGroup", + "templates": ["neptune-db-*"] + }, + { + "type": "neptune_subnet_groups", + "label": "Neptune DB Subnet Groups", + "enabled": true, + "service": "neptune", + "operation": "describe-db-subnet-groups", + "jq_extract": ".DBSubnetGroups[].DBSubnetGroupName", + "logical_id": "NeptuneDBSubnetGroup", + "cfn_type": "AWS::Neptune::DBSubnetGroup", + "templates": ["neptune-db-*"] + }, + { + "type": "neptune_analytics_graphs", + "label": "Neptune Analytics Graphs", + "enabled": true, + "service": "neptune-graph", + "operation": "list-graphs", + "jq_extract": ".graphs[].id", + "logical_id": "Graph", + "cfn_type": "AWS::NeptuneGraph::Graph", + "templates": ["neptune-analytics-*"] + }, + { + "type": "opensearch_collections", + "label": "OpenSearch Serverless Collections", + "enabled": true, + "service": "opensearchserverless", + "operation": "list-collections", + "jq_extract": ".collectionSummaries[].id", + "logical_id": "OpenSearchServerless", + "cfn_type": "AWS::OpenSearchServerless::Collection", + "templates": ["*-opensearch-serverless"] + }, + { + "type": "opensearch_encryption_policies", + "label": "OpenSearch Encryption Policies", + "enabled": true, + "service": "opensearchserverless", + "operation": "list-security-policies --type encryption", + "jq_extract": ".securityPolicySummaries[].name", + "logical_id": "OSSEncryptionPolicy", + "cfn_type": "AWS::OpenSearchServerless::SecurityPolicy", + "templates": ["*-opensearch-serverless"] + }, + { + "type": "opensearch_network_policies", + "label": "OpenSearch Network Policies", + "enabled": true, + "service": "opensearchserverless", + "operation": "list-security-policies --type network", + "jq_extract": ".securityPolicySummaries[].name", + "logical_id": "OSSNetworkPolicy", + "cfn_type": "AWS::OpenSearchServerless::SecurityPolicy", + "templates": ["*-opensearch-serverless"] + }, + { + "type": "opensearch_access_policies", + "label": "OpenSearch Access Policies", + "enabled": true, + "service": "opensearchserverless", + "operation": "list-access-policies --type data", + "jq_extract": ".accessPolicySummaries[].name", + "logical_id": "OSSAccessPolicy", + "cfn_type": "AWS::OpenSearchServerless::AccessPolicy", + "templates": ["*-opensearch-serverless"] + }, + { + "type": "opensearch_vpc_endpoints", + "label": "OpenSearch Serverless VPC Endpoints", + "enabled": true, + "service": "opensearchserverless", + "operation": "list-vpc-endpoints", + "jq_extract": ".vpcEndpointSummaries[].id", + "logical_id": "OpenSearchServerlessVpcEndpoint", + "cfn_type": "AWS::OpenSearchServerless::VpcEndpoint", + "templates": ["*-opensearch-serverless"] + }, + { + "type": "aurora_clusters", + "label": "Aurora PostgreSQL Clusters", + "enabled": true, + "service": "rds", + "operation": "describe-db-clusters", + "jq_extract": ".DBClusters[] | select(.Engine==\"aurora-postgresql\") | .DBClusterIdentifier", + "logical_id": "PostgresCluster", + "cfn_type": "AWS::RDS::DBCluster", + "templates": ["*-aurora-postgres*"] + }, + { + "type": "aurora_instances", + "label": "Aurora PostgreSQL Instances", + "enabled": true, + "service": "rds", + "operation": "describe-db-instances", + "jq_extract": ".DBInstances[] | select(.Engine==\"aurora-postgresql\") | .DBInstanceIdentifier", + "logical_id": "PostgresInstance", + "cfn_type": "AWS::RDS::DBInstance", + "templates": ["*-aurora-postgres*"] + }, + { + "type": "aurora_cluster_parameter_groups", + "label": "Aurora Cluster Parameter Groups", + "enabled": true, + "service": "rds", + "operation": "describe-db-cluster-parameter-groups", + "jq_extract": ".DBClusterParameterGroups[] | select(.DBParameterGroupFamily | startswith(\"aurora-postgresql\")) | .DBClusterParameterGroupName", + "logical_id": "PostgresClusterParameterGroup", + "cfn_type": "AWS::RDS::DBClusterParameterGroup", + "templates": ["*-aurora-postgres*"] + }, + { + "type": "aurora_parameter_groups", + "label": "Aurora DB Parameter Groups", + "enabled": true, + "service": "rds", + "operation": "describe-db-parameter-groups", + "jq_extract": ".DBParameterGroups[] | select(.DBParameterGroupFamily | startswith(\"aurora-postgresql\")) | .DBParameterGroupName", + "logical_id": "PostgresDBParameterGroup", + "cfn_type": "AWS::RDS::DBParameterGroup", + "templates": ["*-aurora-postgres*"] + }, + { + "type": "aurora_subnet_groups", + "label": "Aurora DB Subnet Groups", + "enabled": true, + "service": "rds", + "operation": "describe-db-subnet-groups", + "jq_extract": ".DBSubnetGroups[].DBSubnetGroupName", + "logical_id": "PostgresSubnetGroup", + "cfn_type": "AWS::RDS::DBSubnetGroup", + "templates": ["*-aurora-postgres*"] + }, + { + "type": "s3_buckets", + "label": "S3 Buckets", + "enabled": true, + "service": "s3api", + "operation": "list-buckets", + "jq_extract": ".Buckets[].Name", + "logical_id": "S3Bucket", + "cfn_type": "AWS::S3::Bucket", + "templates": ["*-s3-vectors"] + }, + { + "type": "sagemaker_notebooks", + "label": "SageMaker Notebook Instances", + "enabled": true, + "service": "sagemaker", + "operation": "list-notebook-instances", + "jq_extract": ".NotebookInstances[].NotebookInstanceName", + "logical_id": "NeptuneNotebookInstance", + "cfn_type": "AWS::SageMaker::NotebookInstance", + "templates": ["all"] + }, + { + "type": "sagemaker_lifecycle_configs", + "label": "SageMaker Lifecycle Configs", + "enabled": true, + "service": "sagemaker", + "operation": "list-notebook-instance-lifecycle-configs", + "jq_extract": ".NotebookInstanceLifecycleConfigs[].NotebookInstanceLifecycleConfigName", + "logical_id": "NeptuneNotebookInstanceLifecycleConfig", + "cfn_type": "AWS::SageMaker::NotebookInstanceLifecycleConfig", + "templates": ["all"] + }, + { + "type": "lambda_functions", + "label": "Lambda Functions", + "enabled": true, + "service": "lambda", + "operation": "list-functions", + "jq_extract": ".Functions[].FunctionName", + "logical_id": "LambdaFunction", + "cfn_type": "AWS::Lambda::Function", + "templates": ["all (Custom::CustomResource backing)"] + }, + { + "type": "kms_keys", + "label": "KMS Keys", + "enabled": true, + "service": "kms", + "operation": "list-keys", + "jq_extract": ".Keys[].KeyId", + "logical_id": "KMSKey", + "cfn_type": "AWS::KMS::Key", + "templates": ["*-s3-vectors"] + }, + { + "type": "iam_roles", + "label": "IAM Roles (non-service-linked)", + "enabled": true, + "service": "iam", + "operation": "list-roles", + "jq_extract": ".Roles[] | select(.Path != \"/aws-service-role/\") | .RoleName", + "logical_id": "IAMRole", + "cfn_type": "AWS::IAM::Role", + "templates": ["all"] + }, + { + "type": "iam_policies", + "label": "IAM Customer Managed Policies", + "enabled": true, + "service": "iam", + "operation": "list-policies --scope Local", + "jq_extract": ".Policies[].PolicyName", + "logical_id": "IAMManagedPolicy", + "cfn_type": "AWS::IAM::ManagedPolicy", + "templates": ["all"] + }, + { + "type": "vpc", + "label": "VPCs (non-default)", + "enabled": true, + "service": "ec2", + "operation": "describe-vpcs", + "jq_extract": ".Vpcs[] | select(.IsDefault == false) | .VpcId", + "logical_id": "VPC", + "cfn_type": "AWS::EC2::VPC", + "note": "Also discovers subnets, security groups, NAT/IGW, VPC endpoints for each VPC", + "templates": ["neptune-db-*, neptune-analytics-aurora-postgres"] + }, + { + "type": "elastic_ips", + "label": "Elastic IPs", + "enabled": true, + "service": "ec2", + "operation": "describe-addresses", + "jq_extract": ".Addresses[].AllocationId", + "logical_id": "ElasticIP", + "cfn_type": "AWS::EC2::EIP", + "templates": ["neptune-db-*"] + } + ] +} diff --git a/examples/lexical-graph/cloudformation-templates/audit/audit-resources.json b/examples/lexical-graph/cloudformation-templates/audit/audit-resources.json new file mode 100644 index 000000000..45cf212da --- /dev/null +++ b/examples/lexical-graph/cloudformation-templates/audit/audit-resources.json @@ -0,0 +1,156 @@ +{ + "description": "Resource types to discover in manual scan mode. Remove or add entries to control what the audit finds.", + "resource_types": [ + { + "type": "neptune_clusters", + "label": "Neptune DB Clusters", + "enabled": true, + "service": "neptune", + "operation": "describe-db-clusters", + "jq_extract": ".DBClusters[].DBClusterIdentifier", + "logical_id": "NeptuneDBCluster", + "cfn_type": "AWS::Neptune::DBCluster" + }, + { + "type": "neptune_instances", + "label": "Neptune DB Instances", + "enabled": true, + "service": "neptune", + "operation": "describe-db-instances", + "jq_extract": ".DBInstances[].DBInstanceIdentifier", + "logical_id": "NeptuneDBInstance", + "cfn_type": "AWS::Neptune::DBInstance" + }, + { + "type": "neptune_analytics", + "label": "Neptune Analytics Graphs", + "enabled": true, + "service": "neptune-graph", + "operation": "list-graphs", + "jq_extract": ".graphs[].id", + "logical_id": "Graph", + "cfn_type": "AWS::NeptuneGraph::Graph" + }, + { + "type": "opensearch_collections", + "label": "OpenSearch Serverless Collections", + "enabled": true, + "service": "opensearchserverless", + "operation": "list-collections", + "jq_extract": ".collectionSummaries[].id", + "logical_id": "OpenSearchServerless", + "cfn_type": "AWS::OpenSearchServerless::Collection" + }, + { + "type": "opensearch_encryption_policies", + "label": "OpenSearch Encryption Policies", + "enabled": true, + "service": "opensearchserverless", + "operation": "list-security-policies --type encryption", + "jq_extract": ".securityPolicySummaries[].name", + "logical_id": "OSSEncryptionPolicy", + "cfn_type": "AWS::OpenSearchServerless::SecurityPolicy" + }, + { + "type": "opensearch_network_policies", + "label": "OpenSearch Network Policies", + "enabled": true, + "service": "opensearchserverless", + "operation": "list-security-policies --type network", + "jq_extract": ".securityPolicySummaries[].name", + "logical_id": "OSSNetworkPolicy", + "cfn_type": "AWS::OpenSearchServerless::SecurityPolicy" + }, + { + "type": "opensearch_access_policies", + "label": "OpenSearch Access Policies", + "enabled": true, + "service": "opensearchserverless", + "operation": "list-access-policies --type data", + "jq_extract": ".accessPolicySummaries[].name", + "logical_id": "OSSAccessPolicy", + "cfn_type": "AWS::OpenSearchServerless::AccessPolicy" + }, + { + "type": "sagemaker_notebooks", + "label": "SageMaker Notebook Instances", + "enabled": true, + "service": "sagemaker", + "operation": "list-notebook-instances", + "jq_extract": ".NotebookInstances[].NotebookInstanceName", + "logical_id": "NeptuneNotebookInstance", + "cfn_type": "AWS::SageMaker::NotebookInstance" + }, + { + "type": "s3_buckets", + "label": "S3 Buckets", + "enabled": true, + "service": "s3api", + "operation": "list-buckets", + "jq_extract": ".Buckets[].Name", + "logical_id": "S3Bucket", + "cfn_type": "AWS::S3::Bucket" + }, + { + "type": "aurora_clusters", + "label": "Aurora PostgreSQL Clusters", + "enabled": false, + "service": "rds", + "operation": "describe-db-clusters", + "jq_extract": ".DBClusters[] | select(.Engine==\"aurora-postgresql\") | .DBClusterIdentifier", + "logical_id": "PostgresCluster", + "cfn_type": "AWS::RDS::DBCluster" + }, + { + "type": "aurora_instances", + "label": "Aurora PostgreSQL Instances", + "enabled": false, + "service": "rds", + "operation": "describe-db-instances", + "jq_extract": ".DBInstances[] | select(.Engine==\"aurora-postgresql\") | .DBInstanceIdentifier", + "logical_id": "PostgresInstance", + "cfn_type": "AWS::RDS::DBInstance" + }, + { + "type": "lambda_functions", + "label": "Lambda Functions", + "enabled": false, + "service": "lambda", + "operation": "list-functions", + "jq_extract": ".Functions[].FunctionName", + "logical_id": "LambdaFunction", + "cfn_type": "AWS::Lambda::Function" + }, + { + "type": "iam_roles", + "label": "IAM Roles (non-service-linked)", + "enabled": false, + "service": "iam", + "operation": "list-roles", + "jq_extract": ".Roles[] | select(.Path != \"/aws-service-role/\") | .RoleName", + "logical_id": "IAMRole", + "cfn_type": "AWS::IAM::Role" + }, + { + "type": "iam_policies", + "label": "IAM Customer Managed Policies", + "enabled": false, + "service": "iam", + "operation": "list-policies --scope Local", + "jq_extract": ".Policies[].PolicyName", + "logical_id": "IAMManagedPolicy", + "cfn_type": "AWS::IAM::ManagedPolicy" + }, + { + "type": "vpc", + "label": "VPCs (non-default)", + "enabled": false, + "service": "ec2", + "operation": "describe-vpcs", + "jq_extract": ".Vpcs[] | select(.IsDefault == false) | .VpcId", + "logical_id": "VPC", + "cfn_type": "AWS::EC2::VPC", + "note": "VPC discovery also pulls subnets, security groups, NAT/IGW, and VPC endpoints" + } + ] +} diff --git a/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/load/readers/providers/llama_index_plugin_reader_provider.py b/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/load/readers/providers/llama_index_plugin_reader_provider.py index d28bfb434..b18848187 100644 --- a/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/load/readers/providers/llama_index_plugin_reader_provider.py +++ b/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/load/readers/providers/llama_index_plugin_reader_provider.py @@ -25,6 +25,8 @@ """ import importlib +import os +import re import signal import time from typing import Any, List, Optional @@ -38,6 +40,12 @@ logger = logging.getLogger(__name__) +# Item #4: Pattern for environment variable references in config values +_ENV_VAR_PATTERN = re.compile(r'^\$([A-Z_][A-Z0-9_]*)$') + +# Item #7: Pattern for valid LlamaIndex package names (prevents typosquat suggestions) +_VALID_PACKAGE_PATTERN = re.compile(r'^llama-index-readers?-[a-z0-9-]+$') + class ReaderImportError(ImportError): """Raised when the required LlamaIndex reader package is not installed.""" @@ -59,15 +67,25 @@ class LlamaIndexPluginReaderProvider: Features: - Dynamic import of any LlamaIndex reader package + - Namespace allowlist (prevents arbitrary module loading) + - Interface validation before instantiation + - Environment variable resolution in credentials ($VAR_NAME) - Configurable timeout (prevents hangs) - Retry with exponential backoff on transient failures - Graceful degradation (fail_on_error=False returns [] instead of raising) - - Structured logging at every decision point + - Structured logging at every decision point (never logs credential values) - Auth failure detection (recognizes 401/403 patterns) - Empty result warnings - Metadata enrichment support """ + # Item #1: Namespace allowlist — only these module prefixes can be loaded. + # Subclass and override to allow custom reader namespaces. + ALLOWED_MODULE_PREFIXES = ("llama_index.readers.",) + + # Item #3: Only these methods may be called on the reader instance. + ALLOWED_LOAD_METHODS = ("load_data", "lazy_load", "aload_data") + # Known auth-related error patterns (case-insensitive matching) _AUTH_ERROR_PATTERNS = ( "401", "403", "unauthorized", "forbidden", @@ -112,6 +130,51 @@ def _validate_config(self) -> None: "(e.g. 'llama-index-readers-confluence')" ) + def _validate_module_path(self, module_path: str) -> None: + """Item #1: Validate module path against namespace allowlist before import. + + Prevents arbitrary code execution via malicious module_path values. + Validation applies to the RESOLVED path (after _resolve_module_path). + + Raises: + ReaderImportError: If module_path is not in allowed namespaces. + """ + if not any(module_path.startswith(prefix) for prefix in self.ALLOWED_MODULE_PREFIXES): + raise ReaderImportError( + f"Module '{module_path}' is not in the allowed namespace. " + f"Allowed prefixes: {self.ALLOWED_MODULE_PREFIXES}. " + f"To allow custom namespaces, subclass and override ALLOWED_MODULE_PREFIXES." + ) + + def _resolve_env_vars(self, args: dict) -> dict: + """Item #4: Resolve $VAR_NAME references in dict values from environment. + + Only resolves top-level string values matching $UPPER_CASE_NAME. + Nested dicts, non-string values, lowercase vars, and mid-string + dollar signs are passed through unchanged. + + Raises: + ValueError: If a referenced environment variable is not set. + """ + if not args: + return args + resolved = {} + for key, value in args.items(): + if isinstance(value, str): + match = _ENV_VAR_PATTERN.match(value) + if match: + env_name = match.group(1) + env_value = os.environ.get(env_name) + if env_value is None: + raise ValueError( + f"Config field '{key}' references ${env_name} " + f"but it is not set in the environment" + ) + resolved[key] = env_value + continue + resolved[key] = value + return resolved + def _resolve_module_path(self) -> str: """Resolve the Python module path from config. @@ -128,10 +191,20 @@ def _resolve_module_path(self) -> str: return self._config.package.replace("-", "_") def _import_and_init_reader(self) -> None: - """Dynamically import the reader module and instantiate the reader class.""" + """Dynamically import the reader module and instantiate the reader class. + + Security measures: + - Item #1: Namespace allowlist validation before import + - Item #2: Interface validation before instantiation + - Item #4: Environment variable resolution for credentials + - Item #7: Safe package name validation in error messages + """ module_path = self._resolve_module_path() class_name = self._config.reader_class + # Item #1: Validate namespace before importing + self._validate_module_path(module_path) + logger.info( f"LlamaIndexPlugin: importing {class_name} from {module_path} " f"(package: {self._config.package or 'not specified'})" @@ -140,11 +213,17 @@ def _import_and_init_reader(self) -> None: try: module = importlib.import_module(module_path) except ImportError as e: - install_hint = self._config.package or module_path - msg = ( - f"Failed to import '{module_path}'. " - f"Install the reader package: pip install {install_hint}" - ) + # Item #7: Only suggest pip install for validated package names + if self._config.package and _VALID_PACKAGE_PATTERN.match(self._config.package): + msg = ( + f"Failed to import '{module_path}'. " + f"Install the reader package: pip install {self._config.package}" + ) + else: + msg = ( + f"Failed to import '{module_path}'. " + f"Ensure the reader package is installed." + ) logger.error(msg) raise ReaderImportError(msg) from e @@ -159,10 +238,23 @@ def _import_and_init_reader(self) -> None: logger.error(msg) raise ReaderImportError(msg) from e - # Instantiate the reader with init_args - init_args = self._config.init_args or {} + # Item #2: Interface validation BEFORE instantiation + if not callable(reader_cls): + raise ReaderImportError(f"'{class_name}' in '{module_path}' is not callable") + + load_method_name = self._config.load_method or "load_data" + if not hasattr(reader_cls, load_method_name) and not hasattr(reader_cls, "lazy_load"): + raise ReaderImportError( + f"'{class_name}' does not implement '{load_method_name}()' or 'lazy_load()'. " + f"It may not be a LlamaIndex reader." + ) + + # Item #4: Resolve environment variables in init_args + init_args = self._resolve_env_vars(self._config.init_args or {}) + try: self._reader = reader_cls(**init_args) + # Item #5: Log keys only, NEVER values logger.info( f"LlamaIndexPlugin: {class_name} initialized successfully " f"(init_args keys: {list(init_args.keys())})" @@ -221,6 +313,8 @@ def read(self, input_source: Any = None) -> List[Document]: def _resolve_load_function(self) -> Optional[Any]: """Resolve the callable load method from the reader instance. + Item #3: Validates load_method against ALLOWED_LOAD_METHODS. + Returns: The load function, or None if resolution fails (with appropriate error/log). """ @@ -231,6 +325,16 @@ def _resolve_load_function(self) -> Optional[Any]: return None load_method_name = self._config.load_method or "load_data" + + # Item #3: Restrict to safe method names only + if load_method_name not in self.ALLOWED_LOAD_METHODS: + msg = ( + f"load_method '{load_method_name}' is not allowed. " + f"Permitted: {self.ALLOWED_LOAD_METHODS}" + ) + logger.error(msg) + raise ValueError(msg) + load_fn = getattr(self._reader, load_method_name, None) if load_fn is None: @@ -248,9 +352,10 @@ def _resolve_load_function(self) -> Optional[Any]: def _build_load_args(self, input_source: Any = None) -> dict: """Assemble the keyword arguments for the load function. + Item #4: Resolves $VAR_NAME environment variable references in load_args. Merges configured load_args with an optional input_source. """ - load_args = dict(self._config.load_args or {}) + load_args = self._resolve_env_vars(dict(self._config.load_args or {})) if input_source is not None: load_args["input_source"] = input_source return load_args diff --git a/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/load/readers/reader_provider_config.py b/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/load/readers/reader_provider_config.py index fa5069972..efb86294e 100644 --- a/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/load/readers/reader_provider_config.py +++ b/lexical-graph/src/graphrag_toolkit/lexical_graph/indexing/load/readers/reader_provider_config.py @@ -221,5 +221,5 @@ class LlamaIndexPluginReaderConfig(ReaderProviderConfig): timeout_seconds: int = 120 max_retries: int = 2 retry_backoff_seconds: float = 2.0 - fail_on_error: bool = False + fail_on_error: bool = True metadata_fn: Optional[Callable[[str], Dict[str, Any]]] = None diff --git a/lexical-graph/tests/unit/indexing/load/readers/providers/test_llama_index_plugin_reader_provider.py b/lexical-graph/tests/unit/indexing/load/readers/providers/test_llama_index_plugin_reader_provider.py index d92076520..adb055ef4 100644 --- a/lexical-graph/tests/unit/indexing/load/readers/providers/test_llama_index_plugin_reader_provider.py +++ b/lexical-graph/tests/unit/indexing/load/readers/providers/test_llama_index_plugin_reader_provider.py @@ -524,7 +524,7 @@ def test_module_path_takes_priority(self): """Explicit module_path is used over package derivation.""" mock_module, _, _ = _mock_reader_module() config = _make_config( - module_path="my.custom.module", + module_path="llama_index.readers.custom", package="something-else", ) @@ -534,7 +534,7 @@ def test_module_path_takes_priority(self): ) LlamaIndexPluginReaderProvider(config) - mock_import.assert_called_once_with("my.custom.module") + mock_import.assert_called_once_with("llama_index.readers.custom") def test_derives_module_from_package_name(self): """Package name llama-index-readers-X → llama_index.readers.X.""" @@ -564,10 +564,10 @@ class TestLoadMethod: def test_uses_custom_load_method(self): """load_method config routes to the correct reader method.""" mock_module, _, mock_reader = _mock_reader_module() - mock_reader.lazy_load_data = Mock(return_value=[ + mock_reader.lazy_load = Mock(return_value=[ Document(text="lazy doc", metadata={}) ]) - config = _make_config(load_method="lazy_load_data") + config = _make_config(load_method="lazy_load") with patch("importlib.import_module", return_value=mock_module): from graphrag_toolkit.lexical_graph.indexing.load.readers.providers.llama_index_plugin_reader_provider import ( @@ -577,20 +577,198 @@ def test_uses_custom_load_method(self): docs = provider.read() assert len(docs) == 1 - mock_reader.lazy_load_data.assert_called_once() + mock_reader.lazy_load.assert_called_once() def test_missing_load_method_fails_gracefully(self): - """Non-existent load method returns [] when fail_on_error=False.""" + """Disallowed load_method raises ValueError (security: only allowed methods).""" mock_module, _, mock_reader = _mock_reader_module() - # Remove the method - del mock_reader.custom_method - config = _make_config(load_method="custom_method", fail_on_error=False) + config = _make_config(load_method="custom_method", fail_on_error=True) with patch("importlib.import_module", return_value=mock_module): from graphrag_toolkit.lexical_graph.indexing.load.readers.providers.llama_index_plugin_reader_provider import ( LlamaIndexPluginReaderProvider, ) provider = LlamaIndexPluginReaderProvider(config) - docs = provider.read() + with pytest.raises(ValueError, match="not allowed"): + provider.read() - assert docs == [] + +# ─── Security Hardening Tests (Items #1-7 from review) ──────────────────────── + +from graphrag_toolkit.lexical_graph.indexing.load.readers.providers.llama_index_plugin_reader_provider import ( + LlamaIndexPluginReaderProvider, + ReaderImportError, + ReaderAuthError, +) + + +class TestNamespaceAllowlist: + """Item #1: Verify namespace restriction prevents arbitrary module loading.""" + + def test_disallowed_module_raises(self): + """Module not in ALLOWED_MODULE_PREFIXES is rejected.""" + config = _make_config(module_path="os", reader_class="system") + with pytest.raises(ReaderImportError, match="not in the allowed namespace"): + LlamaIndexPluginReaderProvider(config) + + def test_shutil_blocked(self): + """Filesystem destruction via shutil.rmtree is prevented.""" + config = _make_config(module_path="shutil", reader_class="rmtree") + with pytest.raises(ReaderImportError, match="not in the allowed namespace"): + LlamaIndexPluginReaderProvider(config) + + def test_allowed_prefix_passes(self): + """Module in allowed namespace proceeds to import.""" + mock_module, _, _ = _mock_reader_module() + config = _make_config(module_path="llama_index.readers.confluence") + with patch("importlib.import_module", return_value=mock_module): + provider = LlamaIndexPluginReaderProvider(config) + assert provider._reader is not None + + def test_custom_subclass_extends_allowlist(self): + """Subclass can extend ALLOWED_MODULE_PREFIXES for custom namespaces.""" + class MyProvider(LlamaIndexPluginReaderProvider): + ALLOWED_MODULE_PREFIXES = ("llama_index.readers.", "mycompany.readers.") + + mock_module, _, _ = _mock_reader_module() + config = _make_config(module_path="mycompany.readers.internal") + with patch("importlib.import_module", return_value=mock_module): + provider = MyProvider(config) + assert provider._reader is not None + + +class TestInterfaceValidation: + """Item #2: Verify interface check before instantiation.""" + + def test_non_callable_rejected(self): + """Non-callable attribute is rejected before constructor runs.""" + mock_module = Mock() + mock_module.NotAClass = "just a string" + config = _make_config(module_path="llama_index.readers.test", reader_class="NotAClass") + with patch("importlib.import_module", return_value=mock_module): + with pytest.raises(ReaderImportError, match="is not callable"): + LlamaIndexPluginReaderProvider(config) + + def test_missing_load_data_rejected(self): + """Class without load_data or lazy_load is rejected before instantiation.""" + mock_module = Mock() + mock_cls = Mock() + mock_cls.load_data = None + del mock_cls.load_data + del mock_cls.lazy_load + mock_module.BadReader = mock_cls + config = _make_config(module_path="llama_index.readers.test", reader_class="BadReader") + with patch("importlib.import_module", return_value=mock_module): + with pytest.raises(ReaderImportError, match="does not implement"): + LlamaIndexPluginReaderProvider(config) + + +class TestLoadMethodRestriction: + """Item #3: Only allowed methods can be called.""" + + def test_dunder_method_blocked(self): + """Dunder methods like __delattr__ cannot be called.""" + mock_module, _, _ = _mock_reader_module() + config = _make_config(load_method="__delattr__") + with patch("importlib.import_module", return_value=mock_module): + provider = LlamaIndexPluginReaderProvider(config) + with pytest.raises(ValueError, match="not allowed"): + provider.read() + + def test_arbitrary_method_blocked(self): + """Arbitrary method names are blocked.""" + mock_module, _, _ = _mock_reader_module() + config = _make_config(load_method="execute_shell") + with patch("importlib.import_module", return_value=mock_module): + provider = LlamaIndexPluginReaderProvider(config) + with pytest.raises(ValueError, match="not allowed"): + provider.read() + + +class TestEnvVarResolution: + """Item #4: $VAR_NAME references resolved from environment.""" + + def test_resolves_env_var(self, monkeypatch): + """$VAR_NAME is replaced with environment value.""" + monkeypatch.setenv("MY_TOKEN", "secret123") + mock_module, mock_cls, _ = _mock_reader_module() + config = _make_config(init_args={"token": "$MY_TOKEN", "url": "https://example.com"}) + with patch("importlib.import_module", return_value=mock_module): + LlamaIndexPluginReaderProvider(config) + # Verify the constructor received the resolved value + mock_cls.assert_called_once_with(token="secret123", url="https://example.com") + + def test_missing_env_var_raises(self): + """Reference to unset env var raises ValueError.""" + config = _make_config(init_args={"token": "$NONEXISTENT_VAR_XYZ"}) + mock_module, _, _ = _mock_reader_module() + with patch("importlib.import_module", return_value=mock_module): + with pytest.raises(ValueError, match="not set in the environment"): + LlamaIndexPluginReaderProvider(config) + + def test_non_env_string_unchanged(self): + """Strings without $ prefix are passed through unchanged.""" + mock_module, mock_cls, _ = _mock_reader_module() + config = _make_config(init_args={"url": "https://example.com", "count": "5"}) + with patch("importlib.import_module", return_value=mock_module): + LlamaIndexPluginReaderProvider(config) + mock_cls.assert_called_once_with(url="https://example.com", count="5") + + def test_lowercase_dollar_not_resolved(self): + """$lowercase is not treated as env var (only $UPPER_CASE).""" + mock_module, mock_cls, _ = _mock_reader_module() + config = _make_config(init_args={"note": "$not_an_env_var"}) + with patch("importlib.import_module", return_value=mock_module): + LlamaIndexPluginReaderProvider(config) + mock_cls.assert_called_once_with(note="$not_an_env_var") + + +class TestCredentialLogging: + """Item #5: Credential values must never appear in logs.""" + + def test_credentials_not_logged(self, caplog): + """Secret values in init_args never appear in log output.""" + import logging as stdlib_logging + secret = "super_secret_token_value_12345" + mock_module, _, _ = _mock_reader_module() + config = _make_config(init_args={"github_token": secret}) + with caplog.at_level(stdlib_logging.DEBUG): + with patch("importlib.import_module", return_value=mock_module): + try: + LlamaIndexPluginReaderProvider(config) + except Exception: + pass + assert secret not in caplog.text, "Credential value leaked into logs" + + +class TestPackageNameValidation: + """Item #7: Only suggest pip install for validated package names.""" + + def test_valid_package_gets_install_hint(self): + """Valid llama-index-readers-* package gets pip install suggestion.""" + config = _make_config( + package="llama-index-readers-confluence", + module_path="llama_index.readers.confluence", + ) + with pytest.raises(ReaderImportError, match="pip install llama-index-readers-confluence"): + LlamaIndexPluginReaderProvider(config) + + def test_invalid_package_no_install_hint(self): + """Non-llama-index package does NOT get pip install suggestion.""" + config = _make_config( + package="some-random-package", + module_path="llama_index.readers.random", + ) + with pytest.raises(ReaderImportError) as exc_info: + LlamaIndexPluginReaderProvider(config) + assert "pip install" not in str(exc_info.value) + + def test_arbitrary_package_no_install_hint(self): + """Completely arbitrary package name does NOT get pip install suggestion.""" + config = _make_config( + package="evil-package", + module_path="llama_index.readers.evil", + ) + with pytest.raises(ReaderImportError) as exc_info: + LlamaIndexPluginReaderProvider(config) + assert "pip install" not in str(exc_info.value) diff --git a/working/ADR-0009-dependency-discipline.md b/working/ADR-0009-dependency-discipline.md new file mode 100644 index 000000000..0ece446ba --- /dev/null +++ b/working/ADR-0009-dependency-discipline.md @@ -0,0 +1,43 @@ +# ADR-0009: Dependency Discipline — Insulating the Bespoke and Proprietary Layers from Toolkit Churn + +- **Status:** Accepted principle (engineering-conformance — ours to hold, not a client decision; ADR-0001 §1) +- **Date:** 2026-06-29 +- **Scope:** Cross-cutting design principle governing how CPG-RAG (bespoke) and the Master + Evidence Graph (proprietary, Kanjani AI Research) depend on the graphrag-toolkit. + +## 1. Context +The CPG-RAG layer is bespoke and forever-maintained; the Master Evidence Graph sits above it +as proprietary IP. Both depend on the graphrag-toolkit, which evolves independently. +Uncontrolled coupling produces two failures: unbounded maintenance cost as the toolkit +changes, and erosion of the IP boundary between the toolkit (AWS/open), the bespoke CPG-RAG, +and the proprietary MEG. + +## 2. Principle +The bespoke and proprietary layers MUST be insulated from toolkit churn. Dependency direction +is strictly one-way: **MEG → CPG-RAG → graphrag-toolkit.** Never the reverse. + +## 3. Rules +1. **Depend only on stable/public toolkit contracts** — `GraphStore`/`VectorStore`, the + document-graph public API (`PipelineExecutor`, constructors), `codeproperty-graph` + `DeltaIngestor`, the `BaseEmbedding`/`LLM` seam, `TenantId`. Do not reach into private or + internal modules. +2. **Pin exact toolkit versions** (`==`, never `>=`). Upgrade deliberately, never transitively. +3. **Wrap every toolkit touch-point behind an adapter** (anti-corruption layer). A toolkit + change lands in one adapter, not across CPG-RAG/MEG. +4. **Maintain a contract/compatibility test suite** that runs against each candidate toolkit + version before upgrade — schema-validation at the boundary, so breakage is caught early. +5. **Push genuinely-generic capability upstream** (ADR-0001 D6) to shrink the bespoke surface; + keep only domain (CPG) and proprietary (MEG) logic downstream. +6. **Enforce the IP boundary via the dependency direction** — proprietary MEG logic never + couples into the toolkit or into the CPG-RAG package's public surface. + +## 4. Consequences +- **Cost:** upfront adapter, version-pinning, and contract-test discipline; upgrades are + deliberate work, not automatic. +- **Benefit:** maintenance against toolkit change is bounded and predictable; the IP boundary + is structurally protected; generic improvements accrue upstream to everyone while + proprietary value stays isolated. + +## 5. Relationship to other ADRs +Operationalises ADR-0001 (D1 packaging/dependency direction, D6 upstream-generic). Applies to +every bespoke component named in ADR-0002 through ADR-0008 and to the MEG layer above them. \ No newline at end of file diff --git a/working/Defining the Master Evidence Graph: A Formal Model for Stateful Software Verdicts.md b/working/Defining the Master Evidence Graph: A Formal Model for Stateful Software Verdicts.md new file mode 100644 index 000000000..2e46e9fce --- /dev/null +++ b/working/Defining the Master Evidence Graph: A Formal Model for Stateful Software Verdicts.md @@ -0,0 +1,366 @@ +# Defining the Master Evidence Graph: A Formal Model for Stateful Software Verdicts + +> PRIVATE AND PROPRIETARY. Owned by Kanjani AI Research. See [NOTICE.md](NOTICE.md). + +## 1. Purpose + +The first paper introduced the need for a master evidence graph: a governed structure that converts software artifacts into stateful, verdict-demanding evidence. This paper defines the formal structure of that graph. + +The objective is to define a graph model that can represent: + +1. Evidence artifacts +2. Evidence elements +3. Composite observation models +4. Common vector variables +5. Variable states +6. State gaps +7. Verdict demands +8. Seven verdict-domain projections +9. Time-bound and context-sensitive relationships + +The central design rule is: + +> The master graph must use one common evidence vector. The seven verdicts do not use different evidence universes. They use the same variables with different relationships, weights, and standards of judgment. + +## 2. Core Graph Principle + +The master evidence graph is not simply a storage graph. It is a mediated graph of meaning. + +The graph does not ask only: + +> What evidence exists? + +It asks: + +> What does this evidence support, which state does it affect, what gap does it expose, and which verdict does it demand? + +The minimal chain is: + +`Evidence Artifact → Evidence Element → Composite Observation Model → Evidence Variable → Variable State → State Gap → Verdict Demand → Verdict Domain → Verdict` + +This chain separates raw artifact existence from organizational judgment. + +## 3. Node Types + +The master graph contains the following minimum node classes. + +| Node Type | Purpose | +| --------------------------- | ------------------------------------------------------------------------------ | +| `EvidenceArtifact` | Raw artifact such as SBOM, CVE list, CPG, design document, assessment | +| `EvidenceElement` | Extracted fact or observation from an artifact | +| `CompositeObservationModel` | Mediated object combining related evidence elements | +| `EvidenceVariable` | Named quantitative or categorical variable derived from evidence | +| `VariableState` | Desired, actual, evidenced, or perceived state of a variable | +| `StateGap` | Difference between two states | +| `VerdictDemand` | Trigger that requires a verdict question | +| `VerdictDomain` | One of the seven domains | +| `VerdictProjection` | Domain-specific weighting over the common vector | +| `Verdict` | Resulting mediated judgment | +| `ConcernRelationship` | Relationship between concerns at time `t` and context `c` | +| `Context` | Scope, system, environment, time, tenant, deployment, or operational condition | +| `Authority` | Policy, owner, reviewer, or governance authority | +| `EvidenceSource` | Tool, observer, system, or person that produced evidence | + +## 4. Edge Types + +The minimum graph requires the following relationship types. + +| Edge Type | Meaning | +| ------------------------ | ----------------------------------------------------------------- | +| `PRODUCES` | Artifact holder produces an evidence artifact or element | +| `EXTRACTS` | Observer extracts evidence from an artifact | +| `DECLARES_COMPONENT` | SBOM declares a software component | +| `DECLARES_VULNERABILITY` | Vulnerability source declares vulnerability fact | +| `MATCHES_AFFECTED_RANGE` | Component version matches vulnerability affected range | +| `EVIDENCES_STATIC_USE` | CPG evidence shows static component or function usage | +| `EVIDENCES_REACHABILITY` | CPG evidence supports reachability | +| `CLAIMS_DESIGN_STATE` | Design document claims intended state | +| `SCORES_PERCEPTION` | Likert or assessment score produces perceived state | +| `DERIVES_VARIABLE` | Observation model derives an evidence variable | +| `POPULATES_STATE` | Evidence populates desired, actual, evidenced, or perceived state | +| `EXPOSES_GAP` | State comparison exposes a gap | +| `DEMANDS_VERDICT` | Gap or variable demands a verdict | +| `SUPPORTS_VERDICT` | Variable or gap supports verdict-domain calculation | +| `AMPLIFIES` | One variable increases concern in another | +| `WEAKENS` | One variable reduces concern or confidence | +| `CORROBORATES` | Evidence independently supports another observation | +| `CONTRADICTS` | Evidence conflicts with another observation | +| `RELATES_CONCERN` | Concern-to-concern relationship with type and strength | +| `HAS_CONTEXT` | Node or edge applies within context | +| `HAS_AUTHORITY` | Verdict or state is linked to authority | + +## 5. Composite Observation Model: SCOM + +The first composite model is the Software Component Observation Model, or SCOM. + +SCOM is produced by relating: + +* SBOM component evidence +* Vulnerability-list evidence +* CPG reachability evidence +* Design-document evidence +* Likert or assessment evidence + +SCOM is not the SBOM. SBOM declares component facts. SCOM relates component facts to vulnerability, code, design, and perception evidence. + +Formal expression: + +`SCOM = f(SBOM, VulnerabilityList, CPG, DesignEvidence, AssessmentEvidence)` + +SCOM then derives the minimal evidence vector. + +## 6. Minimal Common Evidence Vector + +The initial SCOM vector contains the following variables. + +| Variable | Type | Meaning | +| ------------------------------- | -------------------------- | --------------------------------------------------------- | +| `component_present` | Binary | Component exists in software | +| `component_identity_confidence` | Continuous | Confidence that component identity/version is correct | +| `vulnerability_match` | Binary | Component/version matches known vulnerability | +| `vulnerability_applicability` | Continuous | Confidence that vulnerability applies | +| `vulnerability_severity` | Continuous | Normalized severity | +| `static_reachability` | Continuous | CPG suggests vulnerable or relevant function is reachable | +| `static_usage_confidence` | Continuous | Confidence in CPG usage signal | +| `design_currency` | Ordinal-derived continuous | Design document is current | +| `design_alignment` | Derived continuous | Design aligns with observed implementation | +| `perceived_component_risk` | Ordinal-derived continuous | Human or organizational perception of concern | + +The common vector is: + +`X = [component_present, component_identity_confidence, vulnerability_match, vulnerability_applicability, vulnerability_severity, static_reachability, static_usage_confidence, design_currency, design_alignment, perceived_component_risk]` + +Every verdict domain uses this same vector. + +## 7. Variable State Model + +Each variable has four states. + +| State | Meaning | +| --------------- | -------------------------------------------------------- | +| Desired state | What should be true | +| Actual state | What is empirically measured | +| Evidenced state | What available evidence currently supports | +| Perceived state | What humans, documents, or organizational claims believe | + +Formal form: + +`variable_i = {desired_i, actual_i, evidenced_i, perceived_i}` + +Actual state must only be assigned when empirically measured. Evidenced state is the best currently supportable value from available artifacts. Perceived state comes from claims, documents, Likert assessments, or organizational belief. + +## 8. State Gap Functions + +The graph must calculate gaps between states. + +| Gap | Formula | Meaning | +| ----------------- | --------------------------------------------------------- | ----------------------------------------------------- | +| `target_gap` | `abs(desired - evidenced)` | Difference between required state and evidenced state | +| `perception_gap` | `abs(perceived - evidenced)` | Difference between belief and evidence | +| `measurement_gap` | `abs(actual - evidenced)` | Difference between measured and evidenced state | +| `governance_gap` | `abs(desired - perceived)` | Difference between intent and belief | +| `design_gap` | `1 - design_alignment` | Difference between design and implementation | +| `awareness_gap` | `abs(perceived_component_risk - component_concern_score)` | Difference between perceived and evidenced concern | + +These gaps are the primary mechanisms by which evidence becomes verdict-demanding. + +## 9. Concern Vector + +The common evidence vector contains both concern-increasing and concern-reducing variables. Therefore, the graph computes a transformed concern vector. + +Concern-increasing examples: + +* `vulnerability_match` +* `vulnerability_applicability` +* `vulnerability_severity` +* `static_reachability` +* `perceived_component_risk` + +Concern-reducing examples: + +* `component_identity_confidence` +* `static_usage_confidence` +* `design_currency` +* `design_alignment` + +The transformed concern vector is: + +`C = [component_present, identity_uncertainty, vulnerability_match, vulnerability_applicability, vulnerability_severity, static_reachability, usage_uncertainty, design_currency_concern, design_alignment_concern, perceived_component_risk]` + +Where: + +`identity_uncertainty = 1 - component_identity_confidence` + +`usage_uncertainty = 1 - static_usage_confidence` + +`design_currency_concern = 1 - design_currency` + +`design_alignment_concern = 1 - design_alignment` + +## 10. Seven Verdict Domains + +The graph supports seven verdict domains. + +| Verdict Domain | Central Question | +| --------------------------- | -------------------------------------------------------------------------- | +| Software Engineering | Does the implementation create engineering concern? | +| Cybersecurity | Does the software contain a meaningful security concern? | +| Architecture and Dependency | Does implementation match intended architecture and dependency design? | +| Operational Capacity | Does the component or code path create operational capacity concern? | +| Organizational Capability | Can the organization understand, sustain, and remediate the concern? | +| Governance and Legitimacy | Is the state aligned with policy, authority, evidence, and accountability? | +| Business and Mission Impact | Does the concern matter materially to business or mission outcomes? | + +Each verdict is a projection over the same concern vector. + +General form: + +`Concern_d(t,c) = W_d · C(t,c)` + +Where: + +* `d` is the verdict domain +* `W_d` is the domain-specific weight vector +* `C(t,c)` is the concern vector at time `t` and context `c` + +## 11. Verdict-Domain Projections + +The seven verdict projections are: + +`Concern_SE = W_SE · C` + +`Concern_SEC = W_SEC · C` + +`Concern_ARCH = W_ARCH · C` + +`Concern_CAP = W_CAP · C` + +`Concern_ORG = W_ORG · C` + +`Concern_GOV = W_GOV · C` + +`Concern_BUS = W_BUS · C` + +The variables remain the same. The weight vector changes. + +This allows one master graph to produce a spider or radar profile of software concern across seven verdict domains. + +## 12. Time-Bound Concern Relationships + +The master graph must treat relationships as time-bound and context-sensitive. + +The fundamental relationship is: + +> At time `t`, under context `c`, concern `A` has relationship `r` to concern `B` with strength `s`. + +Formal form: + +`R(A, B, t, c) = (r, s)` + +Where: + +* `A` is the source concern +* `B` is the target concern +* `t` is time or validity window +* `c` is context +* `r` is relationship type +* `s` is relationship strength + +Relationship type: + +`r ∈ {complementary, opposing, neutral, conditional}` + +Strength: + +`s ∈ [-1, +1]` + +This permits relationships such as: + +* Security may oppose performance in one context. +* Security may complement governance in another context. +* Observability may complement reliability. +* Design currency may weaken over time. +* Vulnerability severity may become more material when exploit activity increases. + +Time is not metadata. Time is an operand in the verdict function. + +## 13. Verdict Demand + +Evidence does not equal verdict. Evidence demands a verdict when it exposes a meaningful gap or crosses a concern threshold. + +Formal form: + +`VerdictDemand = demand(variable, state_gap, threshold, context, domain)` + +A verdict demand is triggered when: + +* Desired state conflicts with evidenced state +* Perceived state conflicts with evidenced state +* Actual state conflicts with evidenced state +* Evidence exceeds threshold +* Evidence contradicts design or policy +* Evidence is insufficient for a required verdict + +Example: + +If: + +`vulnerability_match_desired = 0` + +and: + +`vulnerability_match_evidenced = 1` + +then: + +`target_gap = 1` + +This demands at least a cybersecurity verdict and likely a governance verdict. + +## 14. Master Graph Definition + +The master evidence graph can now be defined as: + +`G_master = (N, E, X, S, Δ, R, W, V)` + +Where: + +* `N` is the set of graph nodes +* `E` is the set of graph edges +* `X` is the common evidence vector +* `S` is the variable state model +* `Δ` is the set of state-gap functions +* `R` is the set of time-bound concern relationships +* `W` is the verdict-domain weight matrix +* `V` is the set of verdict outputs + +The master graph is therefore not one graph of facts. It is a graph of evidence, state, relationship, gap, concern, and verdict. + +## 15. First Implementation Boundary + +The first implementation should be limited to SCOM. + +Inputs: + +* SBOM +* Vulnerability list +* CPG +* Design document +* Likert assessment + +Outputs: + +* SCOM +* Minimal evidence vector +* Four-state variable model +* State gaps +* Seven preliminary verdict concern scores +* Missing-evidence map + +The purpose of the first implementation is not to produce final enterprise verdicts. The purpose is to prove that a common evidence vector can support multiple verdict projections and reveal where evidence is missing. + +## 16. Working Conclusion + +The master evidence graph provides a disciplined way to move from isolated software artifacts to mediated organizational judgment. It does this by preserving one common evidence vector, assigning each variable state, calculating gaps, applying verdict-domain relationships, and producing concern scores across seven verdict domains. + +The result is not merely a graph of software. It is a graph of what the organization knows, believes, intends, measures, lacks, and must decide. diff --git a/working/Domain Graph Migration Architecture_ Portable JSON Boundary and AWS Neptune Runtime.docx.md b/working/Domain Graph Migration Architecture_ Portable JSON Boundary and AWS Neptune Runtime.docx.md new file mode 100644 index 000000000..8a5c88133 --- /dev/null +++ b/working/Domain Graph Migration Architecture_ Portable JSON Boundary and AWS Neptune Runtime.docx.md @@ -0,0 +1,329 @@ +# Domain Graph Migration Architecture: Portable JSON Boundary and AWS Neptune Runtime + +## 1\. Purpose + +This document describes a practical transition architecture for moving existing client graph workloads into an AWS-native GraphRAG runtime using Amazon Neptune, Neptune Analytics, and/or OpenSearch. + +The schema of the JSON files is illustrative and must be updated by the client. This document, once factual and accurate, will serve as the development requirements for the Code Property Graph AWS Solution that will be developed to augment the existing solution. + +The new solution will be based on the existing solution accelerator from AWS Labs: [https://github.com/awslabs/graphrag-toolkit](https://github.com/awslabs/graphrag-toolkit) + +*Note: The AWS solution assumes that the required artifacts are available and in the correct format.* + +The client currently operates separate graph processes, including: + +1. A **lexical graph** based on Confluence/document content. +2. A **domain graph of type CPG** based on Code Property Graph output, currently persisted into Neo4j with vector-in-graph enrichment. + +The target architecture does not require immediate ownership of the client’s extraction processes. Instead, it introduces a **portable artifact boundary** between client-owned extraction/enrichment and AWS-owned build/runtime. + +## 2\. Core Architectural Principle + +**Caution: Extraction can remain portable. Build and runtime become target-specific.** + +When the target graph runtime becomes AWS Neptune, the graph build and query runtime must execute in an AWS-reachable environment. However, extraction does not always need to move into AWS if the extracted artifacts can be persisted and shipped. + +| Concern | Architectural Position | +| :---- | :---- | +| Extraction | Can remain outside AWS | +| Enrichment | Can remain outside AWS if outputs are portable | +| Embedding generation | Can remain outside AWS if vectors are included in the artifact | +| Build/load | Must run in AWS or from an AWS-reachable environment | +| Query/runtime | Must run in AWS or from an AWS-reachable environment | +| Model access | Required wherever build/query depends on the model | + +## 3\. Graph Domains + +The client has two distinct graph domains. Each follows a different migration strategy: + +| Graph Domain | Source | Current Pattern | Target Pattern | +| :---- | :---- | :---- | :---- | +| Lexical graph | Confluence, documents, pages, chunks | Existing document graph pipeline | **Runs entirely in AWS EKS** using graphrag-toolkit native document-graph pipeline (extract + build + query). No portable artifact needed. | +| CPG domain graph | Joern / Code Property Graph / Neo4j | CPG \+ vector-in-graph in Neo4j | Portable JSON artifact shipped to S3; loaded by custom Domain Graph Module in AWS EKS | + +**Key architectural distinction:** +- **Lexical**: The graphrag-toolkit already has a production lexical/document-graph pipeline. It makes no sense to customize the toolkit for a proprietary extraction process. Extract, build, and query all run natively in AWS EKS. +- **CPG**: The client's Joern extraction process is proprietary. The toolkit does not own CPG extraction. The client ships a portable JSON artifact; Deloitte builds a custom Domain Graph Module to load it. + +Both domains share the same AWS EKS cluster, same Neptune instance, same OpenSearch collection, and same query layer. The graphrag-toolkit is deployed as **one container image with three runtime personalities** (Extract, Build, Query) — see architecture.md section 10. + +These graph domains are handled side-by-side, not collapsed into a single extraction process. + +## 4\. CPG Domain Graph: Target Boundary + +For the CPG graph, the client should update the current process so that all outputs currently written directly into Neo4j are instead written to a **portable JSON domain graph artifact**. + +### Current State + +Repo checkout + + ↓ + +Joern / CPG extraction + + ↓ + +Code slicing + + ↓ + +Optional summaries + + ↓ + +Embedding generation + + ↓ + +Neo4j graph \+ vector-in-graph + +### Target State + +Repo checkout + + ↓ + +Joern / CPG extraction + + ↓ + +Code slicing + + ↓ + +Optional summaries + + ↓ + +Embedding generation + + ↓ + +Portable JSON Domain Graph Artifact + + ↓ + +AWS Domain Graph Module + + ↓ + +Neptune Analytics + +or + +Neptune Database \+ OpenSearch + +## 5\. Transformation Required from the Client + +The client does not need to redesign the entire CPG process. The main change is the persistence boundary. + +| Current Process | Target Process | +| :---- | :---- | +| Write CPG nodes to Neo4j | Write CPG nodes to `nodes.jsonl` | +| Write CPG edges to Neo4j | Write CPG edges to `edges.jsonl` | +| Store vectors as Neo4j node properties | Write vectors to `vectors.jsonl` | +| Store summaries on Neo4j nodes | Write summaries to `summaries.jsonl` | +| Store code snippets in Neo4j | Write code slices to `code_slices.jsonl` | +| Use Neo4j as the handoff boundary | Use portable JSON as the handoff boundary | + +The orchestration can remain largely the same. The database-specific writer changes. + +## 6\. CPG Portable JSON Artifact + +The CPG artifact must include enough information for AWS to understand and reconstruct the graph and vector enrichment without participating in extraction. + +| File | Purpose | +| :---- | :---- | +| `manifest.json` | Describes artifact type, schema version, repo, commit, embedding model, dimensions, and vector strategy | +| `nodes.jsonl` | Contains CPG nodes with stable identifiers | +| `edges.jsonl` | Contains CPG relationships between stable node IDs | +| `vectors.jsonl` | Contains precomputed vectors and embedding metadata | +| `summaries.jsonl` | Contains method/file/slice summaries, if available | +| `code_slices.jsonl` | Contains code snippets, line ranges, and evidence slices | +| `findings.jsonl` | Contains scanner findings or security findings, if available | +| `lineage.jsonl` | Contains analysis run, repo, commit, and source artifact metadata | + +## 7\. Required CPG Artifact Metadata + +The artifact must avoid Neo4j internal IDs and use stable portable identifiers. + +| Required Field | Purpose | +| :---- | :---- | +| `repo_id` | Identifies the repository | +| `commit_sha` | Anchors graph to a specific code version | +| `file_path` | Connects nodes to source files | +| `node_type` | Identifies CPG node type, such as `Method`, `File`, `Call`, `TypeDecl` | +| `cpg_node_id` | Stable cross-system node identifier | +| `line_start` | Source-code evidence start line | +| `line_end` | Source-code evidence end line | +| `code_hash` | Detects code changes | +| `analysis_run_id` | Identifies the extraction/enrichment run | +| `embedding_model` | Identifies vector model used | +| `embedding_dimensions` | Required for vector index compatibility | +| `similarity_function` | Cosine, dot product, or other similarity function | +| `embedding_target` | Describes what was embedded: raw code, summary, slice, finding, etc. | + +## 8\. Example CPG Artifact Records + +### Example Node Record + +| { "cpg\_node\_id": "payments-api:abc123:Method:validateToken:sha256001", "node\_type": "Method", "labels": \["Method"\], "repo\_id": "payments-api", "commit\_sha": "abc123", "file\_path": "src/auth/token.py", "fully\_qualified\_name": "auth.TokenService.validateToken", "name": "validateToken", "line\_start": 42, "line\_end": 91, "code\_hash": "sha256001", "properties": { "language": "python", "visibility": "public" } } | +| :---- | + +### Example Edge Record + +| { "edge\_id": "edge:001", "source\_cpg\_node\_id": "payments-api:abc123:File:src/auth/token.py", "target\_cpg\_node\_id": "payments-api:abc123:Method:validateToken:sha256001", "edge\_type": "CONTAINS", "properties": { "analysis\_run\_id": "run-20260704-001" } } | +| :---- | + +### Example Vector Record + +| { "cpg\_node\_id": "payments-api:abc123:Method:validateToken:sha256001", "embedding\_target": "method\_summary", "embedding\_model": "client-embedding-model", "embedding\_dimensions": 1024, "similarity\_function": "cosine", "embedding\_text\_hash": "sha256:abc123", "vector": \[0.012, \-0.044, 0.087\] } | +| :---- | + +## 9\. AWS Responsibility for the CPG Domain Graph + +The AWS-side responsibility begins at the portable JSON boundary. + +| AWS Component | Responsibility | +| :---- | :---- | +| S3 landing zone | Receives portable JSON artifact | +| Domain Graph Module | Validates, normalizes, and loads graph artifacts | +| Neptune Analytics | Optional native graph \+ vector target | +| Neptune Database | Relationship graph target | +| OpenSearch | Vector sidecar when using Neptune Database | +| Query service | Executes graph/vector retrieval | +| Evidence assembler | Builds grounded evidence packages | +| Response LLM | Generates final answer from evidence | + +## 10\. Vector Strategy Depends on Neptune Target + +The handling of precomputed vectors depends on the selected AWS target. + +| Target | Graph Location | Vector Location | Join Pattern | +| :---- | :---- | :---- | :---- | +| Neptune Analytics | Neptune Analytics | Neptune Analytics | Native graph \+ vector | +| Neptune Database | Neptune Database | OpenSearch | Join by stable `cpg_node_id` | +| Hybrid future mode | Neptune / OpenSearch / S3 | Depends on workload | Join by artifact identity | + +For CPG, vectors should be provided in the portable JSON artifact. AWS should not need to regenerate vectors during build unless the artifact is incomplete or incompatible. + +## 11\. Build-Time and Query-Time Model Dependency + +If the portable JSON artifact contains precomputed vectors, the AWS build stage does not need access to the client’s embedding model. + +However, query-time semantic retrieval still needs a compatible embedding model to vectorize the user question. + +| Stage | Requires Model? | Explanation | +| :---- | ----: | :---- | +| CPG JSON validation | No | Schema validation only | +| CPG graph load | No | Deterministic graph load | +| CPG vector load | No | Uses precomputed vectors | +| CPG graph traversal | No | Graph engine handles traversal | +| CPG semantic query | Yes | User question must be embedded | +| CPG final answer | Yes | Response LLM generates answer | +| CPG summarization/enrichment in AWS | Yes | Only if AWS generates new summaries | + +## 12\. Lexical Graph — Runs Entirely in AWS (Toolkit Native) + +The lexical graph uses the graphrag-toolkit's **native document-graph pipeline**. There is no portable JSON artifact for the lexical domain — the toolkit handles extraction, chunking, embedding, build, and query natively. + +It makes no architectural sense to customize the AWS graphrag-toolkit for a proprietary lexical extraction process when the toolkit already handles Confluence/document content out of the box. + +### Lexical Graph in AWS EKS + +| Stage | Runs in | Personality | Notes | +| :---- | :---- | :---- | :---- | +| Confluence content access | AWS EKS | Extract | API access or content export | +| Lexical extraction + chunking | AWS EKS | Extract | Toolkit native pipeline | +| Embedding generation | AWS EKS | Extract | Uses toolkit embedding seam; model in AWS | +| Summarization | AWS EKS | Extract | Uses toolkit generation seam; model in AWS | +| Build into Neptune + OpenSearch | AWS EKS | Build | Toolkit native build | +| Query / retrieval | AWS EKS | Query | Toolkit native query | + +### Client responsibility for lexical domain +- Make Confluence content accessible to AWS EKS (API credentials, network access, or content export to S3) +- Deploy embedding and generation models in AWS EKS +- No extraction code to write — the toolkit does this + +## 13\. Lexical Graph Target Pattern + +Entirely in AWS EKS (graphrag-toolkit native): + +``` +Confluence content (API or export) + ↓ +graphrag-toolkit Extract personality (chunking, summarization, embedding) + ↓ +graphrag-toolkit Build personality (graph + vectors → Neptune + OpenSearch) + ↓ +graphrag-toolkit Query personality (retrieval, evidence assembly, answer generation) +``` + +No cross-cloud boundary. No portable artifact. The toolkit handles this end-to-end. + +## 14\. Practical Transition Architecture + +This architecture is feasible but operationally more complex than a fully cloud-native implementation. + +| Benefit | Cost | +| :---- | :---- | +| Client keeps existing extraction/enrichment logic | Requires portable artifact contract | +| AWS does not need to own Joern extraction | Requires stable ID design | +| Build can be deterministic if vectors are precomputed | Requires vector compatibility management | +| Supports phased migration | Requires artifact shipping and validation | +| Preserves existing client investment | Requires target-specific AWS loaders | +| Avoids immediate CPG-RAG overclaiming | Requires later retrieval/evidence layer work | + +## 15\. Recommended Phase Breakdown + +| Phase | Name | Scope | +| :---- | :---- | :---- | +| Phase 1 | Portable Artifact Boundary | Client emits portable JSON for CPG and/or lexical extracted artifacts | +| Phase 2 | AWS Domain Graph Loader | AWS validates and loads graph/vector artifacts | +| Phase 3 | AWS Graph Runtime | Neptune/Neptune Analytics/OpenSearch become runtime targets | +| Phase 4 | Retrieval Adapter | Query services retrieve graph/vector evidence | +| Phase 5 | CPG-RAG / Lexical-RAG Semantics | Add graph-aware retrieval, evidence assembly, and LLM answer generation | + +## 16\. Scope Clarification + +This should not initially be positioned as a full CPG-RAG rebuild. + +A better Phase 1 position is: + +We are introducing a Domain Graph Module that can ingest portable graph artifacts. The first supported domain graph type is CPG. The client remains responsible for extraction and enrichment. AWS becomes responsible for loading, indexing, querying, and operationalizing the graph. + +## 17\. Key Architectural Statement + +The migration is not from Joern to AWS. + +The migration is from: + +Neo4j as the integration boundary + +to: + +Portable JSON as the integration boundary + +Then AWS becomes the target-specific build and runtime environment. + +## 18\. Final Position + +This architecture is technically viable but not necessarily optimal. It is a pragmatic transition pattern. + +The client must update existing Neo4j-oriented processes to produce portable JSON artifacts containing graph structure, stable identities, lineage, and precomputed vectors. AWS then consumes those artifacts through a Domain Graph Module and loads them into Neptune, Neptune Analytics, and/or OpenSearch. + +The main dependency created by moving to Neptune is that build and runtime become AWS-specific. Model reachability is only required where models are used: extraction, embedding generation, query embedding, planning, summarization, or final answer generation. + +In short: + +| Principle | Statement | +| :---- | :---- | +| Extraction | Portable | +| Artifact | Portable JSON | +| Build | AWS-specific | +| Runtime | AWS-specific | +| Vectors | Precomputed where possible | +| Query | Model-dependent | +| Traversal | Graph-dependent | +| Optimization | Future phase | + diff --git a/working/README.md b/working/README.md new file mode 100644 index 000000000..96e1dd79b --- /dev/null +++ b/working/README.md @@ -0,0 +1,5 @@ +# working/ + +Local scratch area. This entire folder is git-ignored (see `/working/` in `.gitignore`) +and is never committed or pushed. Use it for experiments, converted examples, notes, +throwaway data, and anything you don't want tracked. diff --git a/working/Toward a Master Evidence Graph for Software Verdicts.md b/working/Toward a Master Evidence Graph for Software Verdicts.md new file mode 100644 index 000000000..a17e30b76 --- /dev/null +++ b/working/Toward a Master Evidence Graph for Software Verdicts.md @@ -0,0 +1,145 @@ +# Toward a Master Evidence Graph for Software Verdicts + +> PRIVATE AND PROPRIETARY. Owned by Kanjani AI Research. See [NOTICE.md](NOTICE.md). + +## 1. Purpose + +This paper proposes a method for transforming software evidence artifacts into a governed, stateful, and verdict-supporting master graph. + +Modern software organizations possess many evidence artifacts: SBOMs, vulnerability lists, code property graphs, design documents, runtime telemetry, control records, ownership data, and human assessments. These artifacts are usually evaluated in isolation. A vulnerability scanner reports a match. A design document claims an architecture. A code graph shows reachability. A team believes a component is low risk. Each artifact provides evidence, but none alone provides a complete verdict. + +The purpose of the master evidence graph is to make these artifacts mathematically comparable, semantically related, and verdict-demanding. + +The central claim is: + +> Evidence is not the verdict, but evidence demands a verdict. + +Evidence becomes organizationally meaningful only when it is related to a concern, a desired state, an evidenced state, a perceived state, and a verdict domain. + +## 2. Core Thesis + +The proposed model is built on five principles. + +First, there must be one common evidence vector. The seven verdict domains must not invent separate evidence universes. They must consume the same variables. + +Second, each variable must have state. A variable is not merely a value. It has a desired state, an actual state, an evidenced state, and a perceived state. + +Third, relationships are verdict-relative. The same evidence may support a cybersecurity verdict, weaken an architecture verdict, expose an organizational capability gap, or demand a governance review. + +Fourth, relationships are time-bound and context-sensitive. At time `t`, under context `c`, concern `A` has relationship `r` to concern `B` with strength `s`. + +Fifth, the master graph exists to expose gaps. The most important organizational signals are often not the raw values, but the difference between what was intended, what was evidenced, what was measured, and what was believed. + +## 3. The Minimal Evidence Vector + +The first version of the model begins with a Software Component Observation Model, or SCOM. + +SCOM is not the SBOM. The SBOM is a component declaration artifact. SCOM is a mediated component observation model produced by relating SBOM evidence to vulnerability intelligence, code property graph observations, design claims, and human or document-based assessments. + +The minimal SCOM vector contains the following variables: + +| Variable | Type | Meaning | +| ------------------------------- | -------------------------- | ---------------------------------------------------------- | +| `component_present` | Binary | Component exists in the software | +| `component_identity_confidence` | Continuous | Confidence that component identity and version are correct | +| `vulnerability_match` | Binary | Component/version matches a known vulnerability | +| `vulnerability_applicability` | Continuous | Confidence that the vulnerability applies | +| `vulnerability_severity` | Continuous | Normalized vulnerability severity | +| `static_reachability` | Continuous | CPG evidence suggests reachable use | +| `static_usage_confidence` | Continuous | Confidence in the CPG usage signal | +| `design_currency` | Ordinal-derived continuous | Design documentation is current | +| `design_alignment` | Derived continuous | Design claims align with implementation evidence | +| `perceived_component_risk` | Ordinal-derived continuous | Human or organizational perception of component concern | + +These variables form the first common evidence vector. + +## 4. Four-State Variable Model + +Each variable has four possible states: + +| State | Meaning | +| --------------- | -------------------------------------------------------- | +| Desired state | What should be true | +| Actual state | What is empirically measured | +| Evidenced state | What available evidence currently supports | +| Perceived state | What humans, documents, or organizational claims believe | + +A variable is therefore represented as: + +`variable = {desired_state, actual_state, evidenced_state, perceived_state}` + +Actual state may only be assigned when empirically measured. Evidenced state is the best currently supportable value from available artifacts. Perceived state represents claims, assumptions, Likert assessments, design beliefs, or team understanding. + +The graph must calculate gaps between these states: + +| Gap | Formula | Meaning | +| --------------- | ---------------------------------------- | ----------------------------------------------- | +| Target gap | `abs(desired_state - evidenced_state)` | Difference between required and evidenced truth | +| Perception gap | `abs(perceived_state - evidenced_state)` | Difference between belief and evidence | +| Measurement gap | `abs(actual_state - evidenced_state)` | Difference between measured and evidenced truth | +| Governance gap | `abs(desired_state - perceived_state)` | Difference between intent and belief | + +These gaps are the primary mechanisms by which evidence becomes verdict-demanding. + +## 5. Seven Verdict Domains + +The model defines seven verdict domains: + +1. Software Engineering +2. Cybersecurity +3. Architecture and Dependency +4. Operational Capacity +5. Organizational Capability +6. Governance and Legitimacy +7. Business and Mission Impact + +Each verdict domain consumes the same evidence vector, but applies different relationships and weights. + +The general form is: + +`VerdictConcern_d(t,c) = W_d · C(t,c)` + +Where: + +* `d` is the verdict domain. +* `W_d` is the verdict-specific weight vector. +* `C(t,c)` is the transformed common concern vector at time `t` under context `c`. + +The variables remain the same. The verdict interpretation changes because the relationship and weighting change. + +## 6. Master Graph Objective + +The master graph is not merely an evidence store. It is a mediated relationship graph that connects artifacts, variables, states, concerns, gaps, verdict demands, and verdicts. + +The graph must answer questions such as: + +* What evidence exists? +* What variable does the evidence support? +* Which state does the evidence populate? +* What gap does the evidence expose? +* Which verdict domain does the gap demand? +* What additional evidence is missing? +* How strong is the concern? +* How does the concern change over time and context? +* Which verdicts are currently underdetermined? + +The master graph therefore acts as the structure through which software evidence becomes organizational intelligence. + +## 7. Initial Master Graph Pattern + +The minimal graph pattern is: + +`Evidence Artifact → Evidence Element → SCOM → Evidence Variable → Variable State → State Gap → Verdict Demand → Verdict Domain → Verdict` + +The first implementation begins with: + +* SBOM evidence +* Vulnerability-list evidence +* CPG evidence +* Design and Likert assessment evidence + +These artifacts create the first SCOM vector. The SCOM vector supports the seven verdict domains. The gaps reveal what is known, unknown, misaligned, contradicted, or merely perceived. + +## 8. Working Definition + +A master evidence graph is a time-bound, context-sensitive graph that represents how evidence artifacts produce quantitative variables, how those variables express desired, actual, evidenced, and perceived states, how state gaps demand verdicts, and how seven verdict domains project meaning from the same common evidence vector. diff --git a/working/adr/ADR-0001-cpg-rag-delta.md b/working/adr/ADR-0001-cpg-rag-delta.md new file mode 100644 index 000000000..0e2e8739d --- /dev/null +++ b/working/adr/ADR-0001-cpg-rag-delta.md @@ -0,0 +1,128 @@ +# ADR-0001: CPG-RAG on the GraphRAG Toolkit — Delta Analysis and Architectural Approach + +- **Status:** Proposed (for client approval) +- **Date:** 2026-06-29 +- **Owners:** Solution architecture +- **Related:** `working/graphrag_com_redrawn_architecture.md`, `working/federated_semantics_graph_architecture.md`, `working/neptune_vs_neptune_analytics_cpg_rag_architecture.md`, `READMORE.md` + +--- + +## 1. Framing (read this first) + +The AWS GraphRAG Toolkit (`lexical-graph`, plus the `document-graph` and `codeproperty-graph` packages layered on it) is a **general-purpose, modular, SOLID platform**. It was **not built for this client**. This record is therefore a **delta analysis**: it separates what the toolkit already provides and does correctly from the **net-new components the client's CPG-RAG use case requires**. + +Two consequences follow directly and constrain every decision below: + +1. **We do not fork or specialize the toolkit for the client.** The core stays general-purpose. Client-specific composition lives above it. +2. **We reuse the toolkit's existing abstractions rather than inventing parallel ones.** Where the toolkit already defines a seam (model providers, graph/vector stores, transformer providers), the delta consumes that seam. Anything that bypasses an existing seam is treated as non-conformant and is part of the delta to correct — not a shortcut to add. + +### Decision stance (applies to all ADRs in this set) + +We advise; the client decides. Every item recorded as a "decision" is one of: + +- **(a) an engineering conformance decision** about how *we* build on the toolkit (SOLID conformance, dependency direction, no vendor coupling) — ours to own and hold the line on; or +- **(b) a proposed decision the client must approve**, always presented with options, a recommendation, and consequences. + +An ADR that records options without a final selection is still a decision record: it fixes *the decision that must be made* and its trade-offs, pending client approval. Nothing in these ADRs commits the client to a course of action. Where we lack information to choose, we say so and list the open question rather than assume. + +This is an architecture record, not an implementation plan. No code changes are proposed here. Implementation is gated on the design document that follows this ADR. + +--- + +## 2. Toolkit baseline (what already exists and is conformant) + +Verified against the source, not assumed: + +| Capability | Where | SOLID property it already satisfies | +|---|---|---| +| Model abstraction for generation | `GraphRAGConfig.extraction_llm` / `response_llm` typed as LlamaIndex `LLM`; `LLMType = Union[LLM, str]`; resolved via `to_llm` | DIP — callers depend on the `LLM` abstraction; instances are injectable | +| Model abstraction for embeddings | `GraphRAGConfig.embed_model` typed as `BaseEmbedding`; `EmbeddingType = Union[BaseEmbedding, str]`; resolved via `to_embedding_model` | DIP — callers depend on `BaseEmbedding`; instances are injectable | +| Store separation | distinct `GraphStore` and `VectorStore`; Neptune = graph truth, OpenSearch = vectors, S3 = artifacts | SRP — relationship truth, similarity, and raw artifacts are separate concerns | +| Provider extensibility | factory + registry patterns: `graph_store_factory`, `vector_store_factory`, reader/transformer/schema/extract provider factories and registries | OCP — new providers register without modifying consumers | +| Structured ETL to graph | `document-graph` pipeline: extract → ingest → transform → build (constructors → Cypher) → load; operates on generic record dicts | SRP / strategy pattern per stage | +| CPG domain layer | `codeproperty-graph`: `CPGNode/CPGEdge.from_joern`, `Manifest`, `GraphDiff`, `DeltaIngestor` (skip-or-replace, pluggable `write_fn`, tenant purge), `ManifestManager`, `tenant_ops` | SRP — CPG semantics isolated from graph infrastructure | +| Layering | `codeproperty-graph` → `document-graph` → `lexical-graph` (foundation) | Clean one-directional dependencies | + +**Conclusion of the baseline:** the platform already embodies the model-provider abstraction and the enrich-before-load ETL the client needs. The client's local Mistral 7B is, architecturally, a LlamaIndex `LLM`/`BaseEmbedding` implementation injected through the seam that already exists. It is not a new capability of the platform. + +--- + +## 3. Decision drivers + +- Preserve the toolkit's SOLID, modular character; no client-specific logic in the core. +- Depend on abstractions, never on vendor SDKs, at every layer (DIP). +- One-directional dependencies: client/domain packages depend on the foundation, never the reverse. +- No in-place mutation of a live graph; no read-modify-write; enrichment happens on data, before the graph store. +- Embeddings are entry points; the graph is proof (per the storage architecture doc). +- Correctness of the delta/idempotency contract over convenience. + +--- + +## 4. Decisions + +### D1 — CPG-RAG is a separate package that depends on the toolkit +Client CPG-RAG composition (overlay building, identity resolution, cross-graph linking, multi-graph planning) lives in its own package that **depends on** `lexical-graph` / `document-graph` / `codeproperty-graph`. The toolkit core remains general-purpose. Dependency direction is strictly domain → foundation. This mirrors the existing layering (`codeproperty-graph` already sits on `document-graph` on `lexical-graph`) and keeps release cadence and client specifics isolated. + +### D2 — All inference and embedding flow through the toolkit's existing model seam +Every model call — CPG summarisation (generation) and vectorisation (embedding) — depends on the toolkit's `LLM` / `BaseEmbedding` abstraction resolved by `GraphRAGConfig`. **Local Mistral 7B is injected as a LlamaIndex provider through that seam**, not reached by a vendor-specific code path. The current `document-graph` enricher plugins (`LLMEnricherPlugin` binding the OpenAI SDK, `BedrockEnricherPlugin` binding `boto3` bedrock-runtime, each duplicating the transform loop) **embed the inference backend inside a transformer**, which couples a transform concern to a vendor and violates DIP. They are recorded as **non-conformant** and are re-expressed as a single model-agnostic enrichment transformer that consumes an injected `LLM`. This is the conformant design, not an add-on. + +### D3 — Enrich Joern CPG JSON before Neptune; retire the Neo4j read-modify-write +The client's current flow (Joern JSON → load Neo4j → read → enrich → write back to Neo4j) mutates a live graph on every run. The conformant flow enriches the **records** (`document-graph` transformers driven by the injected model) and then delta-loads the finished graph to Neptune via `codeproperty-graph.DeltaIngestor`. No graph-database round-trip for enrichment. This is already the platform's intended shape (`document-graph` enriches records; `DeltaIngestor` does skip-or-replace) — the delta is only wiring it for the CPG domain. + +### D4 — The delta/idempotency contract keys on (code signature + enrichment recipe version) +`DeltaIngestor` today keys the skip/replace decision on METHOD `full_name:hash` only. That is correct for code change but unsound once enrichment is part of the output: improving the summariser (model/prompt/version) while code is unchanged would incorrectly SKIP and leave stale overlay. The manifest signature must therefore incorporate the **enrichment recipe version** so any change to enrichment inputs forces re-ingest. This is a correctness property of the contract, not a patch. + +### D5 — The net-new delta is a set of SRP-bounded components +The following do not exist in any package today and constitute the client delta. Each is a component with a single responsibility and an explicit boundary: +- **Semantic Code Overlay** — `SemanticCodeUnit` (method / endpoint / source-to-sink summaries) as a first-class graph shape, built by an overlay-builder; only summaries are embedded, never raw AST/edges. +- **Canonical Identity Spine** — the bounded context that application/service/repo/API/control/owner identities resolve into. +- **Cross-Graph Linker** — writes explicit canonical-identity edges at ingest, replacing fuzzy entity-extraction correlation. +- **Multi-Graph Query Planner + Provenance** — federated retrieval across lexical, domain, CPG, and evidence graphs with source/graph-path provenance. + +### D6 — Generic seams are broadened upstream; client specifics stay downstream +Where a gap is genuinely general-purpose (for example, broadening `to_llm` / `to_embedding_model` string resolution beyond the Bedrock default so any registered provider resolves), the enhancement is contributed **upstream to the toolkit via extension (OCP)** — additive, behaviour-preserving. Anything client- or CPG-specific stays in the downstream CPG-RAG package. The test for placement: *would a non-client user of the toolkit want this?* If yes, upstream; if no, downstream. + +--- + +## 5. What we explicitly are NOT doing + +- Not forking or specialising the toolkit for the client. +- Not introducing a parallel model abstraction — the `LLM`/`BaseEmbedding` seam is the abstraction. +- Not leaving vendor SDK bindings inside transformers. +- Not mutating a live graph in place or reading-modifying-writing for enrichment. +- Not embedding raw AST nodes, identifiers, literals, or edges. +- Not putting client orchestration state (Cosmos DB / MongoDB API) on the synchronous retrieval path (tracked separately). + +--- + +## 6. Delta summary (Have / Non-conformant / Missing) + +| Component | Status | Action | +|---|---|---| +| Model seam (`LLM` / `BaseEmbedding` via `GraphRAGConfig`) | Have, conformant | Reuse; inject local Mistral 7B provider | +| `document-graph` ETL + transformer registry | Have, conformant | Reuse | +| `document-graph` enrichers (OpenAI/Bedrock-coupled) | Have, **non-conformant** | Re-express as model-agnostic enricher on the injected `LLM` (D2) | +| `codeproperty-graph` `from_joern` + `DeltaIngestor` | Have, conformant | Reuse; extend manifest signature (D4) | +| Semantic Code Overlay (`SemanticCodeUnit`) | Missing | Build in CPG-RAG package (D5) | +| Canonical identity spine + resolver | Missing | Build (D5) | +| Cross-graph linker | Missing | Build (D5) | +| Multi-graph query planner + provenance | Missing | Build (D5) | +| Provider resolution beyond Bedrock default | Enhancement | Upstream via OCP (D6) | + +--- + +## 7. Consequences + +**Positive:** the toolkit stays general-purpose and SOLID; the client sees a precise, defensible delta rather than a rewrite; local Mistral 7B, Bedrock, and any future backend are configuration choices behind one seam; the enrich-before-Neptune flow removes the Neo4j round-trip; dependencies stay one-directional. + +**Negative / cost:** the non-conformant enrichers must be reworked before they carry production load (D2); the delta components (D5) are real engineering, not configuration; the manifest-signature change (D4) requires a migration of existing manifests. + +**Neutral:** client orchestration (MongoDB API / A2A agents) is out of scope for this ADR and is addressed as a separate integration decision. + +--- + +## 8. Next steps (gated) + +1. **SPEC (requirements)** — acceptance criteria for D2–D5, phased. +2. **Design document (for approval)** — deep on the buildable first phase (model-agnostic enricher on the injected `LLM` + Semantic Code Overlay + manifest-signature contract), with the identity spine, linker, and planner at outline depth. +3. **Implementation** — only after the design is approved. diff --git a/working/adr/ADR-0002-tenant-strategy-cpg-separation.md b/working/adr/ADR-0002-tenant-strategy-cpg-separation.md new file mode 100644 index 000000000..e50f10b59 --- /dev/null +++ b/working/adr/ADR-0002-tenant-strategy-cpg-separation.md @@ -0,0 +1,69 @@ +# ADR-0002: Tenant Strategy and Logical Separation for CPG + +- **Status:** Proposed — client decision required +- **Date:** 2026-06-29 +- **Stance:** We propose and inform; the client decides (see ADR-0001 §1 Decision stance). +- **Relates to:** ADR-0001 (D1, D5), `requirements.md` R6/R9, `lexical-graph/.../tenant_id.py`, `docs-site/.../lexical-graph/multi-tenancy.mdx` + +--- + +## 1. Context + +The client currently runs **two Neo4j databases per Project**, where a **Project = one or many repositories**. We are proposing to replace physical database separation with **logical separation inside a shared Neptune (and/or Neptune Analytics) store**, using the toolkit's existing `TenantId` mechanism. + +This ADR records the decision to be made about **tenant granularity** and how Project / repo / the client's two-database split map onto `TenantId`. We do not select for the client; we present options, constraints, a recommendation, and open questions. + +> Open question carried forward: **what do the client's two Neo4j databases per Project actually represent** (e.g. raw vs enriched CPG, two domains, two environments)? The mapping below changes with the answer, so this must be confirmed. We do not assume. + +## 2. Constraints (tooling-imposed, non-negotiable) + +From `tenant_id.py` (verified in source): + +- A `TenantId` value is **1–25 characters, lowercase letters/numbers/periods, periods not at start or end**. No uppercase, hyphens, slashes, or `@`. +- The **default tenant** is `None` (renders as `default_`). +- **Label encoding:** `format_label` produces `` `{label}{value}__` `` — the tenant value is concatenated onto the node label with a trailing `__` (e.g. label `Method`, tenant `proj1` → `` `Methodproj1__` ``). +- **Vector index naming:** `format_index_name` produces `{index_name}_{value}` (e.g. `chunk_proj1`) — i.e. one index name per tenant. +- **Id encoding:** `format_id` produces `{prefix}:{value}:{id}` (non-default) vs `{prefix}::{id}` (default). + +From `codeproperty-graph.DeltaIngestor` (verified in source): + +- On a **changed** ingest, it writes under a **new tenant id** and then **purges the previous tenant** (`delete_tenant`). Tenant therefore behaves as a **versioned snapshot boundary**, and *replace is whole-tenant*. + +**Consequence of combining these:** tenant granularity must be compatible with "replace/purge the entire tenant on change." Any identifier that is not already ≤25 lowercase-alnum-plus-period characters (most repo/project names, e.g. `github.com/org/Repo-Name`) must be **derived** into a valid tenant id — which ties this decision to the canonical identity spine (ADR-0001 D5 / requirements R6). + +## 3. Options + +### Option A — Tenant per Project (coarse) +All repos of a Project share one tenant. +- **Pros:** trivial project-scoped retrieval (one tenant = one project); fewer tenants. +- **Cons:** collides with `DeltaIngestor` purge semantics — a change in *one* repo would purge the *whole* project's CPG unless the write path is changed to not purge. Higher blast radius. + +### Option B — Tenant per repo (or repo+version) — *recommended* +Each repo (optionally repo+version) is its own tenant; a Project is a **set** of repo-tenants grouped by a naming convention (e.g. shared prefix within the 25-char budget). +- **Pros:** aligns with `DeltaIngestor` per-repo skip/replace + purge; change blast radius is one repo; matches the existing design. +- **Cons:** project-scoped retrieval must fan out across the project's tenant set (the multi-graph planner must be tenant-group aware); requires a grouping convention and a derivation into valid tenant ids. + +### Option C — Tenant per Project, repo as label/property, versioning without tenant purge +Keep Project as the tenant; encode repo as a label/property; handle versioning via the toolkit's versioned-updates instead of tenant purge. +- **Pros:** project-scoped retrieval trivial; no tenant fan-out. +- **Cons:** diverges from `codeproperty-graph`'s delta design (would not use tenant purge); more custom work; reintroduces in-place update concerns we set out to avoid. + +## 4. Recommendation (for client approval) + +Adopt **Option B** (tenant per repo, optionally repo+version), with the **canonical identity spine deriving tenant ids** deterministically from canonical repo identity (normalise + short hash to satisfy the 25-char lowercase constraint). Group repos into a Project via a naming/registry convention so the planner can scope retrieval to a project's tenant set. This preserves the delta/purge design and bounds blast radius to a single repo. + +On the two-database split: **if** the two Neo4j DBs are "raw" and "enriched" CPG, that split **disappears** in the target — enrichment happens before load, so a single enriched graph (raw CPG + `SemanticCodeUnit` overlay) lives under the repo tenant. We would inform the client that two DBs collapse to one tenant-scoped enriched graph. If the two DBs are two distinct domains, they map to two tenants (or two graph shapes) — pending the clarification above. + +## 5. Consequences + +- Retrieval must be **tenant-group aware** to answer project-level questions (feeds the multi-graph planner, R8). +- The **canonical identity spine (R6) becomes a prerequisite** for tenant-id derivation, not just cross-graph linking. +- Delta purge blast radius is bounded to one repo (Option B). +- Vector indexes are per-tenant via `format_index_name` — see ADR-0003 for the store-topology interaction. + +## 6. Open questions for the client + +1. What do the two Neo4j databases per Project represent? +2. Is per-repo (or per-repo-per-version) tenant granularity acceptable, given tenant churn from delta purge? +3. What is the Project→repo grouping convention (prefix/registry)? +4. Are cross-project retrieval questions in scope (affects planner tenant scoping)? diff --git a/working/adr/ADR-0003-store-topology-embedding-divergence.md b/working/adr/ADR-0003-store-topology-embedding-divergence.md new file mode 100644 index 000000000..7f883edf0 --- /dev/null +++ b/working/adr/ADR-0003-store-topology-embedding-divergence.md @@ -0,0 +1,76 @@ +# ADR-0003: Graph + Vector Store Topology for CPG-RAG (Neptune Database vs Neptune Analytics) + +- **Status:** Proposed — client decision required +- **Date:** 2026-06-29 +- **Stance:** We propose and inform; the client decides (see ADR-0001 §1 Decision stance). +- **Relates to:** ADR-0001 (D5), ADR-0002, `requirements.md` R4/R8/NFRs, `working/neptune_vs_neptune_analytics_cpg_rag_architecture.md` + +--- + +## 1. Context + +The client anticipates using **both Amazon Neptune Database and Neptune Analytics**. This introduces a real CPG-RAG complication the client raised: **the embedding approach differs between the two engines**, so the retrieval/embedding design cannot be written as if "vectors live in the graph" in both cases. + +CPG-RAG needs **multiple embedding types, potentially at different dimensions** — lexical chunk, statement, method summary, endpoint summary, source-to-sink slice, and (Phase 2) finding summary — and **per-tenant separation** (ADR-0002). + +This ADR records the decision to be made about **where embeddings live** and **what role each engine plays**. We present options, a recommendation, and consequences; the client decides. + +## 2. Constraints (from the storage architecture doc and AWS docs it cites) + +- **Neptune Database** — durable graph truth; it is **not** the vector store. The toolkit separates `GraphStore` from `VectorStore`; the paired vector surface is typically **OpenSearch Serverless**. +- **OpenSearch Serverless** — supports **many indexes, multiple/arbitrary dimensions, metadata filtering**, and **per-tenant index naming** (`format_index_name` → `chunk_`). ACID-style index operations; frequent delta re-embeds are fine. +- **Neptune Analytics** — memory-optimised, supports **vector-in-graph** search, but: **one vector index per graph, fixed dimension set at graph creation, and vector updates are not ACID**. Loaded from Neptune/S3/snapshot. + +## 3. The divergence, stated precisely + +- **Embeddings in OpenSearch (Neptune DB path):** multiple embedding types/dimensions coexist; per-tenant separation via index-name suffix; delta re-embedding is safe. Fits CPG-RAG directly. +- **Embeddings in Neptune Analytics (vector-in-graph):** a single graph can hold **only one vector index at one fixed dimension**, so chunk + method-summary + source-to-sink embeddings (different models/dims) **cannot** coexist in one NA graph; per-tenant separation means **separate NA graphs** (not index suffixes); and the **non-ACID** vector updates clash with the delta re-embedding loop (R5). + +Therefore the CPG-RAG embedding design **must not assume vectors live in the graph**. + +## 4. Options + +### Option A — Neptune DB + OpenSearch only (no Neptune Analytics) +- **Pros:** simplest; fits multi-embedding, multi-tenant CPG-RAG; lowest cost; consistent with "embeddings are entry points, graph is proof." +- **Cons:** no in-memory graph algorithms / vector-in-graph analytics. + +### Option B — Role split (recommended if "use both" is required) +- **Neptune DB + OpenSearch** = durable graph truth + operational multi-embedding retrieval (the CPG-RAG hot path). +- **Neptune Analytics** = **on-demand analytical** engine, loaded from Neptune/S3 for graph algorithms (centrality, community, blast-radius, similar source-to-sink), optionally with a **single-purpose, single-dimension** vector index for a specific analysis. NA is **not** the primary vector home and **not** on the delta re-embedding path. +- **Pros:** keeps the operational path flexible and correct; still unlocks analytics where NA is genuinely valuable. +- **Cons:** two engines to operate; NA is memory-optimised (cost); requires deliberate load/snapshot boundaries. + +### Option C — Neptune Analytics as the unified graph+vector store +- **Pros:** one engine for graph + vectors. +- **Cons:** viable **only** if a single embedding type at a single dimension per graph is acceptable and non-atomic vector updates are tolerable — which conflicts with multi-embedding CPG-RAG and the delta loop. Not recommended as the operational vector home. + +## 5. Recommendation (for client approval) + +For the **Phase 1 operational retrieval path**, keep embeddings in **OpenSearch Serverless with Neptune Database** (Option A, or Option B if analytics are needed). Treat **Neptune Analytics as optional, analytical, and on-demand** (Option B) — never the operational vector home for multi-embedding CPG-RAG, and never on the delta re-embedding path. Consequently, the `SemanticCodeUnit` overlay embeddings (R4) target OpenSearch, keyed by `graph_node_id`, while Neptune holds the proof relationships. + +## 6. Consequences + +- The overlay/embedding component (R4) writes to OpenSearch, not into the graph; the design must not depend on vector-in-graph traversal for the operational path. +- Per-tenant vectors are clean in OpenSearch (index-name suffix); in NA, per-tenant means separate graphs — a further reason NA is not the operational vector home. +- If the client mandates a vector-in-graph analysis in NA, it is scoped to one embedding type/dimension on a **loaded snapshot**, isolated from the live delta pipeline. +- Cost: NA is memory-optimised/capacity-priced; Option B should gate NA usage to specific analytical jobs. + +## 7. Open questions for the client + +1. Does any required query genuinely need vector-in-graph traversal, or is vector-first-then-graph sufficient? +2. How many embedding types/dimensions are in Phase 1? +3. Is Neptune Analytics in scope for Phase 1 at all, or Phase 2 analytics only? +4. What is the cost ceiling for Neptune Analytics (memory-optimised capacity)? + +- **Direction set (pending client sign-off):** The client currently stores vectors + in-graph (Neo4j node property). We recommend a deliberate design shift to + **external vectors on Neptune Database + OpenSearch** (Option B; embeddings in + OpenSearch). Rationale — Neptune Analytics does not scale to hundreds of + long-lived, multi-tenant projects (new and old): + (a) per-graph single vector index + fixed dimension forces ~one NA graph per tenant; + (b) memory-optimised/capacity pricing penalises mostly-cold retained old projects; + (c) fixed dimension blocks embedding-model evolution across project vintages; + (d) non-ACID vector updates are risky under continuous per-repo delta re-embedding. + Neptune Analytics is retained for on-demand analytics on loaded subgraphs only. + Vector-in-graph traversal is not required: CPG-RAG uses vector-as-entry-point then + graph-as-proof (two-step), not vector inside traversal. diff --git a/working/adr/ADR-0004-store-agnostic-write-strategy.md b/working/adr/ADR-0004-store-agnostic-write-strategy.md new file mode 100644 index 000000000..ce5b51322 --- /dev/null +++ b/working/adr/ADR-0004-store-agnostic-write-strategy.md @@ -0,0 +1,85 @@ +# ADR-0004: Store-Agnostic Write/Read Strategy — Support BOTH Neptune Database and Neptune Analytics + +- **Status:** Proposed — client decision required +- **Date:** 2026-06-29 +- **Stance:** We propose and inform; the client decides (see ADR-0001 §1 Decision stance). +- **Relates to / reconciles:** ADR-0001 (D1, D2), ADR-0003 (store topology), requirements.md R3/R4/R9 + +--- + +## 1. Context + +ADR-0003 recorded a recommended direction (external vectors on Neptune Database + OpenSearch) +driven by scale. That recommendation stands **as a default**, but it must not become a +hard-coded exclusion of Neptune Analytics. The client must be able to run **both** engines, +and CPG-RAG must not care which is configured. + +The toolkit already supports this: `GraphStore` and `VectorStore` are separate, pluggable +abstractions with factories/registries — graph stores include Neptune Database, Neptune +Analytics, and Neo4j; vector stores include OpenSearch Serverless, Neptune Analytics, +pgvector, and S3 Vectors. The correct design leans on that seam rather than choosing a +single store. + +## 2. Decision + +1. **CPG-RAG depends only on the toolkit's `GraphStore` / `VectorStore` abstractions.** + It does not know or reference a concrete backend. +2. **The write strategy is polymorphic on the configured store topology** (Strategy pattern + behind the toolkit's store abstraction — OCP/DIP): + - **Graph + external vector store** (e.g. Neptune Database + OpenSearch): write the + `SemanticCodeUnit` node to the graph (summary text + id + `SUMMARIZES`/`cpgNodeId` + edges), and write its embedding to the vector store keyed by the node id. + *External-vector strategy.* + - **Vector-in-graph** (e.g. Neptune Analytics): write the embedding onto the graph + node/graph directly; no separate vector-store write. *In-graph strategy.* +3. **Both are first-class and selected by configuration, not by code changes.** Adding a + future backend must not require CPG-RAG changes. +4. **The read/retrieval path adapts symmetrically:** + - External-vector: kNN in the vector store → node id → graph traversal (two-step; + vector = entry point, graph = proof). + - Vector-in-graph: vector search within the graph engine, then traversal. + +## 3. Relationship to ADR-0003 (reconciliation) + +ADR-0004 supersedes the *tone* of ADR-0003's "direction set," not its analysis: + +- **Capability:** both topologies are supported (this ADR). +- **Recommended default at scale:** Neptune Database + OpenSearch, for the reasons in + ADR-0003 (hundreds of long-lived multi-tenant projects; per-graph single vector index; + memory pricing for cold data; fixed dimension vs model evolution; non-ACID vector updates + under delta churn). +- **Neptune Analytics:** fully supported — appropriate for smaller/isolated deployments and + for on-demand analytics on loaded subgraphs. + +So ADR-0003's recommendation is guidance for choosing the default; ADR-0004 guarantees the +client is never locked out of either engine. + +## 4. Consequences + +- CPG-RAG stays store-agnostic and toolkit-conformant; new `GraphStore` / `VectorStore` + implementations work without CPG-RAG changes. +- Each backend's constraints still apply where selected (ADR-0003): NA = one vector + index/graph, fixed dimension, non-ACID vector updates; OpenSearch = many + indexes/dimensions, per-tenant index suffix. +- The overlay/embedding component (R4) must not assume where the vector lives — it asks the + resolved write strategy. +- Testing must cover both strategies (external-vector and in-graph) against their respective + store contracts. + +## 5. Design detail to confirm (for design.md, not decided here) + +- **Capability detection:** how CPG-RAG determines "external vector store present" vs + "vector-in-graph" from the configured toolkit stores. This selects the strategy at runtime. +- **Dual-role NA:** when Neptune Analytics is configured as *both* graph and vector store, + CPG-RAG uses the in-graph strategy and performs no external vector write. +- **Id contract parity:** the shared node-id linkage (external-vector) and the on-node + embedding (in-graph) must expose the same logical `SemanticCodeUnit` identity so retrieval + is uniform across strategies. + +## 6. Open questions for the client + +1. Which deployments target which topology (default Neptune DB + OpenSearch at scale; + Neptune Analytics where)? +2. Are both strategies required in the same environment simultaneously, or per-environment + selection? +3. Any deployment that must switch topology later (migration path between strategies)? \ No newline at end of file diff --git a/working/adr/ADR-0006: External Extraction, CPG-RAG-Owned Schema-Driven Build b/working/adr/ADR-0006: External Extraction, CPG-RAG-Owned Schema-Driven Build new file mode 100644 index 000000000..88f56d84d --- /dev/null +++ b/working/adr/ADR-0006: External Extraction, CPG-RAG-Owned Schema-Driven Build @@ -0,0 +1,44 @@ +# ADR-0006: External Extraction, CPG-RAG-Owned Schema-Driven Build + +- **Status:** Proposed — client decision required +- **Date:** 2026-06-29 +- **Stance:** We propose and inform; the client decides (ADR-0001 §1). +- **Relates to:** ADR-0001 (D1 DRY/packaging), ADR-0004 (store strategy), ADR-0005 (embedding), requirements R3/R4/R9 + +## Context +Extraction is performed externally (Joern export + the client's Neo4j query/enrichment) and +emits a predefined, schema-conformant artifact (the format contract). The toolkit already +separates extract from build (lexical `extract()`/`build()`; document-graph +extract→transform→build→load), so a build can consume an externally-produced extract. The +lexical build's node builders are tuned to lexical node types, so CPG (different node model) +requires a build tailored to the CPG schema. + +## Decision +1. CPG-RAG owns a **build pipeline driven by a predefined external-extraction schema** and + validates incoming extracts against it (fail fast on mismatch). +2. That build is **composed from existing toolkit primitives, not reimplemented** (DRY / D1): + - document-graph schema-driven build (`PipelineExecutor`: schema/mapping → typed + nodes/edges → Cypher → graph) + its validation primitives; + - codeproperty-graph `from_joern` + `DeltaIngestor` (skip-or-replace, tenant purge); + - lexical-graph `GraphStore`/`VectorStore`, with build-time embedding (toolkit owns + vectorization — ADR-0005). +3. Net-new in CPG-RAG's build (the delta): the SemanticCodeUnit overlay builder (R4: + summary node + build-time embedding), the store-agnostic write-strategy resolver + (ADR-0004), and delta-manifest versioning incl. enrichment recipe (R5). +4. The build is store-agnostic and depends only on toolkit abstractions; dependency + direction is domain → foundation (D1). + +## Consequences +- CPG-RAG has an explicit, schema-validated integration boundary with the external extractor. +- No build infrastructure is duplicated; CPG-RAG adds only domain orchestration + overlay + + strategy resolver + delta versioning. +- The predefined extraction schema becomes a versioned contract shared by the external + extractor and CPG-RAG's build. + +## Non-goals +- Not a new build engine. Not reusing the lexical node-builders for the CPG node model. + +## Open questions for the client +1. Who owns and versions the predefined extraction schema (external extractor vs CPG-RAG)? +2. Does the extract arrive as Joern JSON directly, or a client-transformed shape (determines the mapping layer)? +3. Where is the schema validated — at extract-publish time, at build-ingest time, or both? \ No newline at end of file diff --git a/working/adr/ADR-0010-cross-cloud-build-neptune-public-endpoints.md b/working/adr/ADR-0010-cross-cloud-build-neptune-public-endpoints.md new file mode 100644 index 000000000..cd848dc53 --- /dev/null +++ b/working/adr/ADR-0010-cross-cloud-build-neptune-public-endpoints.md @@ -0,0 +1,57 @@ +# ADR-0010: Cross-Cloud Build Topology and Neptune Public Endpoints + +- Status: Proposed - client decision required +- Date: 2026-06-29 +- Stance: We propose and inform; the client decides (ADR-0001 section 1). +- Refines: ADR-0003 (store topology), ADR-0006 (external extraction/build), ADR-0008 (cloud strategy) + +## 1. Context +A client scenario: extraction runs on Azure EKS during an active Azure -> AWS migration; the client wants agents/build in Azure talking to AWS Neptune + OpenSearch. Two facts govern this: +1. Amazon Neptune now supports public endpoints (engine >= 1.4.6.x): per-instance publicly-accessible, IAM DB auth mandatory, security-group controlled, instances in public subnets with an internet-gateway route. Previously Neptune was strictly VPC-private. AWS positions public endpoints as a dev/test convenience. +2. The toolkit connects to Neptune Database via the neptunedata boto3 client over HTTPS with SigV4 (verified) - the same path a public endpoint serves. OpenSearch Serverless supports public endpoints with SigV4 + data-access policies as a standard configuration. + +Therefore the Azure -> AWS build is technically feasible with no toolkit code change. + +## 2. Decision +1. The toolkit requires no change to write to Neptune/OpenSearch over public endpoints - point the connection at the public endpoint URL and supply AWS credentials + region for SigV4. +2. Public-endpoint cross-cloud build is APPROVED for the interim/dev/test/POC period only. +3. For the end-state (production), keep Neptune private and use either: + (a) extract in Azure -> stage artifact in S3 -> build in AWS in-VPC (ADR-0006), or + (b) private connectivity (VPN / Direct Connect + ExpressRoute / Transit Gateway). +4. Any Neptune-touching component (build AND query/retrieval) inherits this; only extract may run cross-cloud freely. + +## 3. Requirements for the public-endpoint path (interim) +- Neptune engine >= 1.4.6.x; publicly-accessible on ALL instances (failover role-swap - see risks). +- IAM DB auth enabled. +- Security-group allowlist for the Azure EKS egress CIDR (stable NAT egress), not 0.0.0.0/0. +- Neptune instances in public subnets with an internet-gateway route. +- Azure EKS -> AWS IAM credentials via workload-identity federation (OIDC) preferred over static keys. +- OpenSearch Serverless public network policy + SigV4 data-access policy (IAM-principal-gated). + +## 4. Risks / caveats +- AWS positions public endpoints as dev/test convenience, not production. +- Public subnets for a proprietary CPG/evidence graph is a security-posture regression. +- Failover role-swap: public is per-instance; the cluster endpoint is public only if the writer is public; on failover a public-writer/private-reader flips and the build loses write access - HA-safe requires ALL instances public (maximum exposure). +- Write-heavy build over public internet: WAN latency per batch, throughput limits, transient errors, egress/data-transfer cost; painful at hundreds-of-projects scale. +- Org governance: an IAM/SCP guardrail may deny public instance creation. + +## 5. Component placement (interim vs end-state) +| Stage | Touches Neptune? | Interim | End-state | +|---|---|---|---| +| Joern extract -> JSON | No | Azure | Azure | +| Per-node enrich | No | Azure | Azure or AWS | +| Build (writes graph + vectors) | Yes | Azure->public endpoint OR AWS | AWS in-VPC | +| Query / retrieval / agents | Yes | Azure->public endpoint OR AWS | AWS in-VPC | +| Traversal-based enrich | Yes | AWS | AWS | + +## 6. Consequences +- POC/interim unblocked quickly with no toolkit change. +- The extract -> S3 -> in-AWS-build split (ADR-0006) remains the durable pattern; even with public endpoints the write-heavy build favors running in AWS. +- Model hosting cascades (ADR-0007): if build runs in AWS, host the embedding model in AWS; the generation model can remain in Azure with enrichment. + +## 7. Open questions for the client +1. Is public-endpoint use acceptable to the client's AWS governance (SCP)? +2. Interim only, or is a production cross-cloud build being contemplated? +3. HA requirement for Neptune during the interim (drives all-instances-public exposure)? +4. Expected build volume (WAN write cost at scale)? +5. Target date for the sunset to private endpoints. diff --git "a/working/adr/Embedding Ownership \342\200\224 Toolkit-Owned Vectorization vs Bring-Your-Own-Vector" "b/working/adr/Embedding Ownership \342\200\224 Toolkit-Owned Vectorization vs Bring-Your-Own-Vector" new file mode 100644 index 000000000..aa0d51e17 --- /dev/null +++ "b/working/adr/Embedding Ownership \342\200\224 Toolkit-Owned Vectorization vs Bring-Your-Own-Vector" @@ -0,0 +1,38 @@ +# ADR-0005: Embedding Ownership — Toolkit-Owned Vectorization vs Bring-Your-Own-Vector + +- **Status:** Proposed — client decision required +- **Date:** 2026-06-29 +- **Stance:** We propose and inform; the client decides (ADR-0001 §1). +- **Relates to:** ADR-0001 (D2), ADR-0004, requirements.md R2/R4 + +## Context +The client has a module that reads CPG from Neo4j, produces a unit, vectorises it, and writes +the vector in-graph. The toolkit's `VectorStore` write path (`add_embeddings`) instead OWNS +vectorisation: it embeds node text via the configured `embed_model`; there is no first-class +API to store a supplied vector as-is. Critically, the toolkit also embeds the QUERY at read +time with the same `embed_model`, so stored vectors and query vectors must come from the same +model or similarity search is invalid. + +## Decision (recommended) +Toolkit owns vectorisation. The client provides the `SemanticCodeUnit` summary TEXT; the +toolkit embeds at index and query time with one configured `embed_model`. The client's chosen +embedding model is configured AS the toolkit's `embed_model` via the model seam (ADR-0001 D2), +so the client keeps their model without a separate pre-vectorisation step. The client's +existing vectorise-and-write step is retired for the toolkit path. + +## Alternative (only if required) +Bring-your-own-vector: a supported write path that stores a pre-computed embedding without +re-embedding. Hard constraints: (a) the same model MUST be pinned as the query-time embedder; +(b) dimension must match the index. Higher complexity; not the default. + +## Consequences +- Recommended path keeps index/query embeddings consistent (correctness) and reuses the + toolkit as designed while still using the client's model. +- `document-graph` writes graph nodes/edges (Neptune), not OpenSearch vectors; the overlay + embedding (R4) spans a graph-node write plus a `VectorStore` write — the seam ADR-0004 covers. +- BYO-vector path would require the toolkit to enforce a model-parity contract. + +## Open questions for the client +1. Must existing vectors be reused, or is regenerating via the toolkit acceptable? +2. Which embedding model must be the toolkit `embed_model`, and is it injectable as a `BaseEmbedding`? +3. If BYO-vector is required, who guarantees query-time model parity? \ No newline at end of file diff --git a/working/adr/Non-Bedrock Model Providers and In-AWS Hosting.md b/working/adr/Non-Bedrock Model Providers and In-AWS Hosting.md new file mode 100644 index 000000000..ea945c0c0 --- /dev/null +++ b/working/adr/Non-Bedrock Model Providers and In-AWS Hosting.md @@ -0,0 +1,61 @@ +# ADR-0007: Non-Bedrock Model Providers and In-AWS Hosting + +- **Status:** Proposed — client decision required +- **Date:** 2026-06-29 +- **Stance:** We propose and inform; the client decides (ADR-0001 §1). +- **Relates to:** ADR-0001 (D2, D6), ADR-0004, ADR-0005, requirements R1/R2 + +## 1. Context +The graphrag-toolkit is AWS-native: Neptune has no public access, so the toolkit +runtime executes inside AWS. Model configuration *defaults* to Amazon Bedrock. The +question is whether the toolkit can use non-Bedrock models (the client runs a local +Ollama stack: a generation model, Mistral 7B, and an Ollama embedding model such as +nomic-embed-text). + +Verified in `config.py`: +- `EmbeddingType = Union[BaseEmbedding, str]`; `LLMType = Union[LLM, str]`. +- `to_embedding_model()` returns the instance unchanged when passed a `BaseEmbedding` + ("if isinstance(embed_model, BaseEmbedding): return embed_model"); it only constructs + `BedrockEmbedding` for a STRING input. `to_llm()` mirrors this for `LLM`. +- The vector index embeds via `self.embed_model` (the `BaseEmbedding` abstraction), at + build and at query time. + +Conclusion: Bedrock is the string-shorthand default, not a hard coupling. The embedding +seam is already provider-neutral. + +## 2. Decision +1. **Non-Bedrock providers are supported today via instance injection.** Inject a + `BaseEmbedding` (e.g. `OllamaEmbedding(model_name="nomic-embed-text", base_url=...)`) + into `GraphRAGConfig.embed_model`, and an `LLM` into the generation seam. No Bedrock + Custom Model Import is required — and for an embedding model it is generally not an + applicable import target. +2. **Host the model in AWS, reachable by the private-Neptune runtime.** Preferred: + self-host the same model (Ollama on EC2/ECS/EKS, or a SageMaker endpoint) inside the + AWS VPC, so build-time and query-time embedding calls stay in-region. Avoid calling a + client-local model across the network from AWS (latency, throughput, reachability). +3. **Do not rely on bring-your-own-vector** (ADR-0005): the toolkit embeds with the same + injected model at build and query, guaranteeing consistency. +4. **The embedding seam needs no rework; the enricher does.** The generation/enrichment + path is vendor-coupled today and must be made model-agnostic to consume the injected + `LLM` (requirements R1). The embedding path is already agnostic. + +## 3. The strategic choice (client-owned) +| Path | What | Trade-off | +|---|---|---| +| (a) Self-managed in AWS | Run the client's exact models (Ollama) on AWS compute | Preserves model + vector size; fastest migration; the client operates model servers | +| (b) Bedrock-native | Use Bedrock embeddings/LLMs (Titan/Cohere, Claude) | Fully managed; changes the model → re-embed + re-tune enrichment | + +Both are cloud deployments. (a) is IaaS (self-managed, in-region); (b) is managed/serverless. + +## 4. Consequences +- Provider choice is a configuration/injection decision, not a code change (embeddings). +- If models are hosted in AWS, there is no cross-network model dependency; throughput of + the in-AWS model servers must be sized for batch embedding across many projects. +- Optional upstream OCP enhancement (ADR-0001 D6): extend `to_llm`/`to_embedding_model` + string resolution to recognise non-Bedrock providers, so string config (not only + instance injection) works. + +## 5. Open questions for the client +1. Exact embedding model and dimension (set `embed_dimensions` to match). +2. Self-managed-in-AWS (a) or Bedrock-native (b)? +3. If self-managed, where hosted and expected batch throughput/latency? \ No newline at end of file diff --git a/working/architecture-addendum-cpg-rag-store-strategy.md b/working/architecture-addendum-cpg-rag-store-strategy.md new file mode 100644 index 000000000..4afaafe1b --- /dev/null +++ b/working/architecture-addendum-cpg-rag-store-strategy.md @@ -0,0 +1,40 @@ +# Architecture Addendum: Store-Agnostic CPG-RAG Write/Read Strategy + +**Status:** Draft for client alignment · **Relates to:** ADR-0003, ADR-0004, +`graphrag_com_redrawn_architecture.md` + +## Principle + +CPG-RAG composes the toolkit's `GraphStore` and `VectorStore` abstractions and never binds a +concrete backend. The **configured store topology determines the write and read strategy** +(Strategy pattern; OCP/DIP). Think *both*, not one. + +## Topology → strategy + +| Configured topology | Where the vector lives | Write strategy | Read strategy | +|---|---|---|---| +| Neptune DB + OpenSearch (external vector) | OpenSearch, keyed by node id | Write `SemanticCodeUnit` node to graph; write embedding to vector store keyed by id | kNN in OpenSearch → node id → Neptune traversal | +| Neptune Analytics (vector-in-graph) | On the graph node | Write `SemanticCodeUnit` node with embedding on the graph | Vector search in-graph → traversal | + +## Invariants (hold across both strategies) + +- Same logical `SemanticCodeUnit` identity and the same `SUMMARIZES` / `cpgNodeId` linkage to + the raw Joern node, regardless of where the vector lives. +- Embeddings are entry points; the graph is proof (two-step retrieval). +- Multi-tenancy honoured via the toolkit `TenantId` encoding. + +## Guidance (not exclusion) + +At hundreds-of-projects scale (new and old), **Neptune DB + OpenSearch is the recommended +default** (ADR-0003). **Neptune Analytics is fully supported** for smaller/isolated +deployments and on-demand analytics (ADR-0004). The client is never locked out of either. + +## Resolution flow + +```mermaid +flowchart TB + CFG[Configured toolkit stores] --> RES{Vector-in-graph capable\nand no external VectorStore?} + RES -- yes --> IG[In-graph write strategy\nembedding on node] + RES -- no --> EX[External-vector write strategy\nembedding in vector store, keyed by id] + IG --> R1[Read: vector search in-graph -> traversal] + EX --> R2[Read: kNN in vector store -> id -> graph traversal] \ No newline at end of file diff --git a/working/architecture.md b/working/architecture.md new file mode 100644 index 000000000..f0ee41825 --- /dev/null +++ b/working/architecture.md @@ -0,0 +1,402 @@ +# Interim Cross-Cloud Architecture — Azure Extract → AWS Build & Query +## graphrag-toolkit / CPG-RAG migration (Neo4j → Amazon Neptune) + +- **Status:** Proposal for client review +- **Audience:** Client architecture & security stakeholders; AWS; Deloitte +- **Date:** 2026-07-07 (revised) +- **Companion decisions:** ADR-0002…ADR-0010 + +--- + +## 1. Purpose +Propose the topology for migrating the client's code-property-graph (CPG) and knowledge-graph workloads off self-hosted Neo4j onto AWS (Amazon Neptune + Amazon OpenSearch), during an active Azure → AWS migration. Core recommendation — a clean responsibility split: + +> **Azure produces extract artifacts; AWS owns build and query.** +> Azure delivers JSON artifacts to Amazon S3; AWS builds the graph and serves retrieval. All graph/vector infrastructure remains private within the AWS VPC. + +## 2. Executive summary +- Joern extraction and enrichment remain in Azure EKS, reusing the client's existing process and models. Joern emits JSON (nodes + edges) — not a graph load — which is the natural hand-off. +- The client's process is repointed from "load into Neo4j" to "stage an artifact in S3": CPG JSON + semantic summaries (text) + a manifest. +- AWS runs the graphrag-toolkit build (S3 → Neptune + OpenSearch) and the query/retrieval + agents — deployed to the client's **AWS EKS cluster**, within the VPC. +- No toolkit code changes — this is a deployment/topology decision. +- Neptune and OpenSearch remain **private** (VPC). The only cross-cloud transfer is the S3 artifact upload (standard IAM/SigV4 authenticated). +- The client operates **two EKS clusters** during the migration: Azure EKS (extraction workloads) and AWS EKS (build/query workloads). Over time, workloads migrate from Azure EKS to AWS EKS. + +## 3. Context & constraints +- The client is migrating Azure → AWS; Neo4j → Neptune is one workstream of that move. +- Current stack (self-hosted, mostly Azure): Joern → JSON → Neo4j; enrich in Neo4j; embed; store vector-in-graph. Local Ollama models (generation + embedding). Azure Cosmos (MongoDB API) for agent orchestration. +- Amazon Neptune is VPC-private by default. This architecture keeps it that way. +- Amazon OpenSearch Serverless can be configured with VPC endpoints for private access. +- The graphrag-toolkit is AWS-native; it connects to Neptune via the neptunedata HTTPS API with SigV4 and to OpenSearch Serverless via SigV4 — both from within the VPC. +- S3 supports IAM/SigV4 authenticated uploads from anywhere — this is the standard cross-cloud hand-off mechanism. + +## 4. Design principles (traceable to ADRs) +1. Extraction is external; build is the toolkit's. (ADR-0006) +2. Graph = truth, embeddings = entry points; stores separated; write strategy is store-agnostic. (ADR-0003, ADR-0004) +3. The toolkit owns vectorization; the client's models are injected via the model seam. (ADR-0005, ADR-0007) +4. One-way dependency; the bespoke layer is insulated from toolkit churn. (ADR-0009) +5. Cloud strategy is native-trending; Neptune and OpenSearch stay private in VPC. (ADR-0008) +6. Cross-cloud seam is S3 only — a bulk artifact transfer, not chatty graph writes. (ADR-0006, ADR-0010) + +## 5. The architecture — the responsibility split + +### 5.1 Graph domains + +The client operates **two distinct graph domains** — each follows a different path: + +| Graph Domain | Source | Strategy | Why | +|---|---|---|---| +| **CPG domain graph** | Source code repos | Client extracts in Azure → portable JSON artifact → AWS loads | Proprietary Joern/CPG process; client investment preserved; no toolkit customization for extraction | +| **Lexical graph** | Confluence, documents | **Runs entirely in AWS** using graphrag-toolkit native pipeline | The toolkit already has a production lexical graph pipeline (document-graph). No reason to customize the toolkit for a proprietary extract when it handles this natively. | + +**Key distinction:** +- CPG: The toolkit does NOT own extraction — the client's Joern process is proprietary and stays in Azure. Deloitte builds a custom Domain Graph Module to load the portable artifact. +- Lexical: The toolkit DOES own extraction, build, and query — it already does this. Confluence content is made accessible to AWS (API access or content export). The entire pipeline runs in AWS EKS. + +### 5.2 Boundary — who does what, where + +| Concern | Runs in | Owner | Notes | +|---|---|---|---| +| **CPG Domain (proprietary extraction → portable artifact → AWS load)** | | | | +| Joern extraction → CPG nodes/edges | Azure EKS | Client | Proprietary process; emits structured JSON | +| Code slicing (snippets, line ranges) | Azure EKS | Client | Evidence slices for grounded retrieval | +| Per-node enrichment (summaries via generation model) | Azure EKS | Client | Multi-model: generation model produces summaries | +| Embedding generation (vectors for nodes/summaries) | Azure EKS | Client | Multi-model: embedding model produces vectors | +| Stage CPG artifact to S3 | Azure EKS → S3 | Client | Portable JSON boundary (section 6) | +| CPG build (load artifact → Neptune + OpenSearch) | AWS EKS (in-VPC) | Deloitte (toolkit) | Custom Domain Graph Module; deterministic load | +| **Lexical Domain (entirely AWS-native — toolkit pipeline)** | | | | +| Confluence content access | AWS EKS | Client (config) | API access to Confluence or content export to S3 | +| Lexical extraction + chunking | AWS EKS (in-VPC) | Deloitte (toolkit) | graphrag-toolkit native document-graph pipeline | +| Embedding generation | AWS EKS (in-VPC) | Deloitte (toolkit) | Uses toolkit's embedding seam; model hosted in AWS | +| Lexical build (graph + vectors → Neptune + OpenSearch) | AWS EKS (in-VPC) | Deloitte (toolkit) | graphrag-toolkit native build | +| **Query & Runtime (both domains)** | | | | +| Query / retrieval / agents | AWS EKS (in-VPC) | Client (infra) + Deloitte (toolkit) | Reads Neptune + OpenSearch across both graph domains | +| Query-time embedding (vectorize user question) | AWS EKS (in-VPC) | Client (model) + Deloitte (toolkit) | Must match stored vector space | +| Answer generation (LLM) | AWS EKS (in-VPC) | Client (model) + Deloitte (toolkit) | Response generation from evidence | + +The only cross-cloud seam is the **CPG artifact upload to S3**. The lexical domain never leaves AWS. + +### 5.3 Data flow +```mermaid +flowchart LR + subgraph AZURE [Azure EKS - client owned] + subgraph CPG [CPG Domain - proprietary extraction] + SRC[Source code] --> JOERN[Joern extract] + JOERN --> SLICE[Code slicing] + SLICE --> ENR_CPG[Enrich: summaries - generation model] + ENR_CPG --> EMB_CPG[Embed: vectors - embedding model] + EMB_CPG --> ART_CPG[CPG Artifact - nodes, edges, vectors, summaries, slices] + end + end + ART_CPG -->|S3 upload, IAM/SigV4| S3[(AWS S3 artifact bucket)] + subgraph AWS_EKS [AWS EKS - client owned, VPC-private] + S3 --> CPG_BUILD[CPG Domain Graph Module - custom loader] + subgraph LEX [Lexical Domain - toolkit native pipeline] + CONF[Confluence content] --> LEX_EXT[graphrag-toolkit lexical extraction + chunking] + LEX_EXT --> LEX_EMB[Embedding - toolkit seam] + LEX_EMB --> LEX_BUILD[graphrag-toolkit lexical build] + end + CPG_BUILD --> NEP[(Amazon Neptune - private endpoint)] + LEX_BUILD --> NEP + CPG_BUILD --> AOSS[(OpenSearch Serverless - VPC endpoint)] + LEX_BUILD --> AOSS + NEP --> Q[Query / retrieval + agents] + AOSS --> Q + Q --> EMB_Q[Query-time embedding model] + Q --> LLM[Answer generation LLM] + end +``` + +- **CPG domain**: crosses the cloud boundary once (S3 artifact upload). Build is a custom loader. +- **Lexical domain**: never leaves AWS. The graphrag-toolkit's native document-graph pipeline handles extraction, build, and query entirely within AWS EKS. +- **Query layer**: unified across both domains — graph traversal + vector search over the combined graph. + +### 5.4 Compute topology — two EKS clusters + +| Cluster | Cloud | Owner | Workloads | Lifecycle | +|---|---|---|---|---| +| Azure EKS | Azure | Client | CPG extraction + enrichment + embedding; S3 upload | Existing; CPG extraction migrates to AWS EKS over time | +| AWS EKS | AWS | Client | Lexical pipeline (toolkit native); CPG Domain Graph Module (build); query/retrieval; agents; embedding model; LLM | New; provisioned for this workload; grows as Azure workloads migrate | + +- The **client owns and operates both EKS clusters** — infrastructure, configuration, scaling, and networking. +- The **client owns the multi-model extraction pipeline** in Azure EKS (generation model for summaries, embedding model for vectors) for both CPG and Lexical domains. +- **Deloitte delivers** the graphrag-toolkit containers/Helm charts (Domain Graph Module) that run in the client's AWS EKS. +- **AWS provisions** the backing services (Neptune, OpenSearch, S3) that the AWS EKS workloads connect to. +- Over time, the client migrates extraction and enrichment workloads from Azure EKS to AWS EKS. The architecture supports this with no changes — only the S3 upload step becomes a local (in-VPC) operation instead of cross-cloud. + +### 5.3 Why Neptune and OpenSearch must remain private — no public endpoint build + +**Building a graph and vector index over a public endpoint from one cloud provider to another is not production-viable.** This is a fundamental architectural constraint, not a preference: + +1. **Write amplification over WAN:** A graph build is not a single bulk load — it is thousands of individual mutations (node creates, edge creates, property updates, vector inserts). Each write is a round-trip. Over a public endpoint from Azure to AWS, every round-trip incurs 20–80ms of WAN latency. A build that takes minutes in-VPC would take hours over the internet. + +2. **Transient failure at scale:** Public internet connections experience packet loss, TCP resets, DNS failures, and throttling. A build performing thousands of sequential writes will hit these failures repeatedly. Retry logic helps, but the error rate makes the process unreliable for production workloads. + +3. **Data transfer cost:** Graph builds are write-heavy in both directions (request + response for every mutation). Cross-cloud egress is billed on both sides. At hundreds of repos, this becomes a material ongoing cost for no architectural benefit. + +4. **Security exposure of a proprietary graph:** A code-property-graph containing proprietary source code structure, control flow, and semantic summaries — accessible over the public internet — is an unacceptable security posture for production, regardless of IAM gating. Defense-in-depth requires network-layer isolation. + +5. **No resilience guarantee:** Neptune public endpoints are positioned by AWS as a dev/test convenience. Failover behavior with public endpoints has caveats (role-swap breaks writer accessibility). This is not a foundation for a production build pipeline. + +**The correct pattern:** Ship the artifact (JSON) to S3 in one bulk transfer, then build locally within the AWS VPC where Neptune and OpenSearch are milliseconds away over private networking. This is faster, cheaper, more reliable, and more secure. + +### 5.4 What actually crosses the cloud boundary + +Only **one thing** crosses from Azure to AWS: the S3 artifact upload. + +| Cross-cloud? | Operation | Protocol | Notes | +|---|---|---|---| +| ✅ Yes | Artifact upload (JSON files → S3) | HTTPS, IAM/SigV4 | One bulk transfer per repo per job. Standard pattern. | +| ❌ No | Graph writes (nodes, edges, properties) | Private VPC | Thousands of mutations — must be local to Neptune | +| ❌ No | Vector inserts (embeddings → OpenSearch) | Private VPC | Must be local to OpenSearch | +| ❌ No | Query / retrieval | Private VPC | Must be co-located with graph + vectors | + +## 6. The hand-off contract (the S3 artifact) + +The client produces a **portable JSON domain graph artifact** — a complete, self-describing bundle that allows AWS to reconstruct the graph and vector enrichment without participating in extraction or embedding. + +### 6.1 CPG artifact structure +A per-repo, per-job bundle: + +``` +s3:///cpg-exports/// + manifest.json # artifact type, schema version, repo, commit, embedding model, dimensions, vector strategy + nodes.jsonl # CPG nodes with stable identifiers (not Neo4j internal IDs) + edges.jsonl # CPG relationships between stable node IDs + vectors.jsonl # precomputed vectors with embedding metadata + summaries.jsonl # method/file/slice summaries + code_slices.jsonl # code snippets, line ranges, evidence slices + findings.jsonl # scanner/security findings (if available) + lineage.jsonl # analysis run, repo, commit, source artifact metadata +``` + +### 6.2 Lexical domain — no artifact needed +The lexical graph does **not** use the portable JSON artifact pattern. The graphrag-toolkit's native document-graph pipeline handles extraction, chunking, embedding, and build internally. The client only needs to make Confluence content accessible to the AWS EKS cluster (API access or content export to S3). + +This is intentional: it makes no sense to customize the AWS graphrag-toolkit for a proprietary extraction process when the toolkit already handles lexical/document content natively and better. + +### 6.3 Why vectors are in the artifact (not generated in AWS) +The client currently generates embeddings as part of their enrichment process (vector-in-graph pattern in Neo4j). This does not change. The client ships **precomputed vectors** in `vectors.jsonl`. Consequences: + +- **AWS build does NOT need the client's embedding model.** The graph and vector load is deterministic — no model inference at build time. +- **AWS query DOES need a compatible embedding model** to vectorize user questions at query time (same model, same dimensions, same similarity function). +- **The manifest declares** the embedding model, dimensions, and similarity function — AWS validates compatibility before loading. +- **If the client changes their embedding model**, all vectors must be regenerated and the artifact re-shipped (the manifest version forces a full re-ingest). + +### 6.4 Key contract requirements +| Requirement | Purpose | +|---|---| +| Stable `cpg_node_id` (not Neo4j internal IDs) | Portable cross-system identity; join key between graph and vectors | +| `embedding_model` + `embedding_dimensions` + `similarity_function` in manifest | AWS validates vector index compatibility | +| `code_hash` on nodes | Enables delta detection (skip unchanged code) | +| `analysis_run_id` in lineage | Tracks which extraction/enrichment run produced the artifact | +| `embedding_target` on each vector | Describes what was embedded (raw code, summary, slice, finding) | +| Provenance fields (repo, commit, file, line_start, line_end) | Evidence traceability | +| Schema version in manifest | Contract evolution without breaking changes | + +### 6.5 What changed from Neo4j +The client's process is identical except the persistence boundary: + +| Current (Neo4j) | Target (Portable JSON) | +|---|---| +| Write CPG nodes to Neo4j | Write CPG nodes to `nodes.jsonl` | +| Write CPG edges to Neo4j | Write CPG edges to `edges.jsonl` | +| Store vectors as Neo4j node properties | Write vectors to `vectors.jsonl` | +| Store summaries on Neo4j nodes | Write summaries to `summaries.jsonl` | +| Store code snippets in Neo4j | Write code slices to `code_slices.jsonl` | +| Neo4j is the integration boundary | Portable JSON is the integration boundary | + +The orchestration remains the same. Only the database-specific writer changes. + +## 7. Networking & connectivity + +### 7.1 Primary architecture (recommended) +- **Azure EKS → S3:** Write artifacts over S3's public endpoint with IAM/SigV4. Standard pattern — no special networking required. +- **Neptune:** Private endpoint within VPC. Accessible only from the client's AWS EKS pods (via VPC networking). +- **OpenSearch Serverless:** VPC endpoint. Accessible only from the client's AWS EKS pods. +- **AWS EKS → Neptune/OpenSearch:** Pods run in VPC subnets with direct private access to Neptune and OpenSearch. No internet traversal. +- **AWS EKS → S3:** Via S3 gateway endpoint (free, no NAT required). + +### 7.2 Credential flow (Azure EKS → S3) +- Azure EKS → AWS IAM credentials via workload-identity federation (OIDC) to an AWS IAM role — not static keys. +- The IAM role has `s3:PutObject` permission on the artifact bucket only. No access to Neptune or OpenSearch. +- TLS in transit; CloudTrail audit on the S3 bucket. + +### 7.3 Credential flow (AWS EKS → Neptune/OpenSearch) +- AWS EKS pods use IAM Roles for Service Accounts (IRSA) — Kubernetes service accounts mapped to IAM roles. +- IAM roles scoped to Neptune and OpenSearch access only. +- No static credentials in pod environment variables. + +### 7.4 Migration trajectory +As the client migrates workloads from Azure EKS to AWS EKS: +- Extraction and enrichment move to AWS EKS. +- The S3 upload becomes a local VPC operation (S3 gateway endpoint) instead of cross-cloud. +- No architecture changes required — the same containers, same artifact format, same S3 bucket. +- Azure EKS can be decommissioned when all workloads have migrated. + +## 8. Security posture +- **Neptune:** Private VPC endpoint. No public access. IAM DB auth enabled. Security group allows only AWS EKS pod subnets. +- **OpenSearch Serverless:** VPC endpoint. Data-access policy scoped to AWS EKS pod IAM roles (IRSA). +- **S3 artifact bucket:** IAM/SigV4. Bucket policy allows PutObject from Azure workload role only. Versioning + KMS encryption. +- **No cross-cloud access to graph stores.** Azure EKS never talks to Neptune or OpenSearch directly. +- **Credentials:** Azure EKS → AWS via OIDC federation (no static keys). AWS EKS pods use IRSA (IAM Roles for Service Accounts). +- **Audit:** CloudTrail on all API calls; S3 access logging; Neptune audit logs enabled. + +## 9. Model hosting (ADR-0007) — a consequence of the split + +Because the CPG domain ships **precomputed vectors** in the artifact, and the lexical domain runs natively in AWS, the model dependencies differ per domain: + +### 9.1 CPG domain (precomputed vectors) +| Stage | Needs embedding model? | Needs generation model? | Where | +|---|---|---|---| +| Extraction (Joern) | No | No | Azure EKS | +| Enrichment (summaries) | No | Yes (generation) | Azure EKS | +| Embedding generation | Yes | No | Azure EKS | +| CPG build (graph + vector load) | **No** | **No** | AWS EKS | +| CPG query-time embedding | **Yes** | No | AWS EKS | +| CPG answer generation | No | **Yes** | AWS EKS | + +### 9.2 Lexical domain (toolkit-native, all in AWS) +| Stage | Needs embedding model? | Needs generation model? | Where | +|---|---|---|---| +| Lexical extraction + chunking | No | Yes (summarization) | AWS EKS | +| Lexical embedding | Yes | No | AWS EKS | +| Lexical build | No | No | AWS EKS | +| Lexical query-time embedding | Yes | No | AWS EKS | +| Lexical answer generation | No | Yes | AWS EKS | + +### 9.3 Model deployment summary +| Model | Azure EKS | AWS EKS | Purpose | +|---|---|---|---| +| Generation (e.g. Mistral 7B) | ✅ CPG enrichment | ✅ Lexical extraction + answer generation | Summaries, answers | +| Embedding (e.g. nomic-embed) | ✅ CPG vector generation | ✅ Lexical build + query-time (both domains) | Vectors | + +The client is responsible for deploying models in both clusters. Models are injected through the toolkit's provider seam (LLM / BaseEmbedding), which is provider-neutral. + +## 10. Toolkit deployment topology in AWS EKS — three personalities + +The graphrag-toolkit is deployed as **one container image** with **three runtime personalities** in the client's AWS EKS cluster. Same code, different configuration — scaled independently: + +| Personality | Role | Scaling Pattern | Config | +|---|---|---|---| +| **Extract** | Lexical extraction + chunking + embedding from Confluence | Job-based (batch); scales with content volume | `TOOLKIT_MODE=extract`; needs generation + embedding model access; reads Confluence API | +| **Build** | Load artifacts into Neptune + OpenSearch (CPG from S3; lexical from extract output) | Job-based (batch); scales with artifact volume | `TOOLKIT_MODE=build`; needs Neptune + OpenSearch write; reads S3 | +| **Query** | Serve retrieval requests (graph traversal + vector search + answer generation) | Request-based (HPA); scales with query traffic | `TOOLKIT_MODE=query`; needs Neptune + OpenSearch read; needs embedding + LLM model access | + +### 10.1 Why split into personalities +- **Independent scaling:** Extract and build are batch/job workloads (bursty, CPU/memory heavy). Query is a long-running service (latency-sensitive, horizontally scalable). +- **Isolation:** A build job consuming resources does not degrade query latency. +- **Security:** Build has write access to Neptune/OpenSearch; query has read-only access. Least-privilege per pod via IRSA. +- **Lifecycle:** Extract runs when new content arrives. Build runs when new artifacts land in S3. Query runs continuously. + +### 10.2 Deployment in EKS +``` +AWS EKS Cluster (client-owned) +├── namespace: graphrag-extract +│ └── Job/CronJob: toolkit (mode=extract) +│ → reads Confluence API +│ → writes extracted chunks to internal staging (S3 or local) +│ → needs: generation model, embedding model +│ +├── namespace: graphrag-build +│ └── Job: toolkit (mode=build) +│ → reads S3 artifacts (CPG portable JSON + lexical extract output) +│ → writes to Neptune (private) + OpenSearch (VPC endpoint) +│ → needs: Neptune write, OpenSearch write +│ +├── namespace: graphrag-query +│ └── Deployment (HPA): toolkit (mode=query) +│ → reads Neptune (private) + OpenSearch (VPC endpoint) +│ → needs: embedding model (query-time), generation model (answers) +│ → serves retrieval API to agents +│ +└── namespace: models + ├── Deployment: embedding-model (e.g. Ollama + nomic-embed) + └── Deployment: generation-model (e.g. Ollama + Mistral 7B) +``` + +### 10.3 IRSA (IAM Roles for Service Accounts) per personality +| Personality | IAM Permissions | +|---|---| +| Extract | S3 read/write (staging), Confluence API (external) | +| Build | S3 read (artifacts), Neptune write, OpenSearch write | +| Query | Neptune read, OpenSearch read — **no write access** | + +## 10. Multi-tenancy & identity (ADR-0002) +- Logical separation via the toolkit's TenantId (label + index-name encoding) within a single Neptune cluster and OpenSearch collection — no per-project database. +- Recommended granularity: tenant per repo (or repo+version), grouped into a Project by a naming convention; a deterministic derivation maps repo identity to a valid TenantId (<= 25 lowercase chars). Bounds delta-replace blast radius to one repo. + +## 11. Delta / idempotency (ADR-0004, ADR-0005) +- The build is skip-or-replace: unchanged repos are skipped (zero writes); changed repos are re-ingested under a new tenant and the prior tenant purged. +- The manifest signature keys on method signatures + enrichment recipe version, so improving the summariser forces re-ingest rather than silently serving stale overlay. + +## 12. Store topology (ADR-0003, ADR-0004) +- Recommended: Neptune Database (durable graph truth) + OpenSearch Serverless (vectors). +- The overlay write strategy is store-agnostic: external-vector (OpenSearch) or vector-in-graph (Neptune Analytics). At hundreds-of-projects scale, Neptune DB + OpenSearch is the default; Neptune Analytics is reserved for on-demand analytics. + +## 13. Toolkit vs bespoke, and future-proofing +- Reused from the toolkit (no rebuild): build pipeline, embedding, storage, checkpoint/extraction format. +- Bespoke (CPG-RAG): the "compiler" that turns the CPG into semantic summaries, and the federation that links the code graph to the document/knowledge graph. +- Future-proofing (cheap, structural): preserving provenance, stable identity, temporal versioning, and using the multi-source-capable ingest path costs little now and keeps future evidence/analytics capabilities open. Not built in this phase; simply not foreclosed. + +## 14. Risks & mitigations +| Risk | Mitigation | +|---|---| +| S3 upload failure from Azure | Retry with exponential backoff; manifest checksums for integrity validation | +| Embedding model unreachable from build | Host embedding model in AWS (co-located with build in same VPC) | +| Embedding dimension mismatch | Confirm model dimension; set embed_dimensions; contract test | +| Toolkit churn breaks bespoke layer | Pin versions; adapter layer; contract tests (ADR-0009) | +| Stale overlay after summariser change | Enrichment recipe version in the manifest | +| Build throughput at scale | Size compute to expected volume; S3 event-driven triggers; parallelism | +| Credential leak (Azure OIDC role) | Scope to s3:PutObject on one bucket only; no Neptune/OpenSearch access | + +## 15. Phasing +- Phase 1 (this document): migrate off Neo4j to Neptune/OpenSearch via the split; Neptune and OpenSearch private in VPC; graphrag-toolkit deployed to client's AWS EKS. Done future-ready (provenance/identity/time preserved). +- Phase 2 (proposed after Phase 1): federate the code graph with the document/knowledge graph. +- Phase 3 (over time): client migrates extraction/enrichment from Azure EKS to AWS EKS. No architecture change required. +- Later capability is out of scope here and proposed separately. + +## 16. Ownership summary + +| Component | Owner | Responsibility | +|---|---|---| +| Azure EKS cluster | Client | Provision, configure, operate, scale | +| Azure workloads (Joern, enrichment, S3 upload) | Client | Develop, deploy, operate | +| AWS EKS cluster | Client | Provision, configure, operate, scale, networking | +| AWS infrastructure (Neptune, OpenSearch, S3, IAM) | AWS | Provision, configure, maintain | +| graphrag-toolkit customization (containers, Helm charts) | Deloitte | Develop, deliver, support | +| Deployment of toolkit to AWS EKS | Client | Deploy containers/charts to their EKS | +| Embedding model (AWS EKS) | Client | Deploy, operate, scale | +| Generation model (Azure EKS) | Client | Existing; no change | +| Architecture approval | Client | Review, decide, sign-off | + +## 17. Decisions we need from the client +1. Confirm the embedding model and its vector dimension. +2. Enrichment: per-node (fits the single build pass) or traversal-based (adds an in-AWS post-load pass)? +3. Tenant granularity and the Project→repo grouping convention. +4. Expected volume (repos/projects) — sizes the build throughput. +5. Build trigger preference: S3 event-driven (automatic) or scheduled/manual? +6. AWS EKS cluster configuration: node instance types, scaling policy, namespaces for toolkit workloads. +7. Timeline for migrating extraction/enrichment from Azure EKS to AWS EKS. + +## Appendix A — Decision record index +ADR-0002 Tenant strategy; ADR-0003 Store topology; ADR-0004 Store-agnostic write; ADR-0005 Embedding ownership; ADR-0006 External extraction / toolkit build; ADR-0007 Model providers & in-AWS hosting; ADR-0008 Cloud strategy; ADR-0009 Dependency discipline; ADR-0010 Cross-cloud seam (S3 artifact). + +--- + +## Appendix B — Contingency: Public endpoints (if Azure must query Neptune directly) + +> **This section applies ONLY if the client requires Azure-based agents or queries to read from Neptune/OpenSearch directly during the migration — i.e., before query compute moves to AWS. This is NOT the recommended path.** + +If this contingency is needed: +- Neptune engine >= 1.4.6.x supports public endpoints (IAM auth mandatory). +- All instances must be publicly accessible (failover safety). +- Security-group allowlist to Azure EKS egress CIDR only. +- OpenSearch Serverless public network policy + SigV4 data-access policy. +- Time-boxed with a documented sunset date. Disable public access once query compute moves to AWS. +- See ADR-0010 for full requirements, risks, and caveats. + +**Client decisions required for this contingency:** +1. Is public-endpoint use acceptable to security/governance (any SCP on public Neptune)? +2. Neptune HA required during the interim (drives all-instances-public exposure)? +3. Target date for the sunset to private endpoints. diff --git a/working/cloud-native-vs-self-hosted-assessment.md b/working/cloud-native-vs-self-hosted-assessment.md new file mode 100644 index 000000000..140c267f0 --- /dev/null +++ b/working/cloud-native-vs-self-hosted-assessment.md @@ -0,0 +1,47 @@ +# Migration Mapping: Local / Hybrid → AWS + +**Status:** Draft for client alignment · **Relates to:** ADR-0002, ADR-0003, ADR-0004, +ADR-0005, ADR-0006, ADR-0007 + +## Purpose +A component-by-component map from the client's current stack to AWS targets, with the +decision and status for each. This ties the ADRs into one migration picture. + +## Mapping +| Component | Current (client) | AWS target | Decision / status | +|---|---|---|---| +| Code extraction | Joern → nodes.json / edges.json | Unchanged (external) | Preserved (ADR-0006) | +| Graph store | Neo4j (self-hosted) | Neptune Database (default) or Neptune Analytics | Planned; store-agnostic (ADR-0003/0004) | +| Enrichment (summaries) | In Neo4j (read-modify-write) | Build-time transform using injected model | Re-homed; enricher R1 rework; per-node vs traversal fork | +| Generation model | Local Ollama (Mistral 7B) | AWS-hosted Ollama (self-managed) or Bedrock LLM | Decision (ADR-0007) | +| Embedding model | Local Ollama embedding | AWS-hosted Ollama (self-managed) or Bedrock embedding | Decision (ADR-0007); confirm model+dimension | +| Vectors | Vector-in-graph (Neo4j) | OpenSearch (Neptune DB) or vector-in-graph (Neptune Analytics) | Store-agnostic write strategy (ADR-0004) | +| Vectorization ownership | Client pipeline | Toolkit embeds at build+query | No BYO-vector (ADR-0005) | +| Semantic overlay | Client "compiler" (what to summarise) | CPG-RAG overlay builder (SemanticCodeUnit) | Net-new, reuses client logic (R4) | +| Tenant separation | Two Neo4j DBs per project | Logical tenants via TenantId | tenant-per-repo recommended (ADR-0002) | +| Orchestration store | Azure Cosmos DB (MongoDB API) | DynamoDB (AWS-native) or keep cross-cloud | Open decision (earlier) | + +## What is preserved +- Joern extraction (unchanged). +- Enrichment *logic* and model *choice* (Mistral generation + Ollama embedding). +- Vector size / embedding model semantics. + +## What changes +- Storage: Neo4j → Neptune; vectors → OpenSearch or in-graph. +- Enrichment *location*: from in-graph read-modify-write to a build-time transform. +- Vectorization *owner*: from the client's pipeline to the toolkit (build + query). +- Model *hosting location*: from local to AWS. + +## The effort variables (size these before committing) +1. **Enrichment: per-node vs traversal-based.** Per-node → pre-load transform (light). + Traversal-based → post-load overlay pass (real re-engineering). +2. **Model hosting: self-managed-in-AWS vs Bedrock-native.** (a) preserves the model, + self-managed; (b) fully managed, requires re-embed. +3. **Orchestration store: DynamoDB vs cross-cloud Cosmos.** Cross-cloud adds latency/egress. + +## Sequencing (suggested) +1. Stand up Neptune + (OpenSearch or NA) + in-AWS model hosting. +2. Build path: Joern JSON → document-graph/codeproperty-graph → Neptune, tenant-per-repo, delta. +3. Overlay + embedding at build (toolkit owns vectorization). +4. Retrieval + provenance; join to the lexical (Confluence) graph via identity (Phase 2). +5. Resolve the Cosmos/orchestration decision. \ No newline at end of file diff --git a/working/federated_semantics_graph_architecture.md b/working/federated_semantics_graph_architecture.md new file mode 100644 index 000000000..530009cac --- /dev/null +++ b/working/federated_semantics_graph_architecture.md @@ -0,0 +1,1012 @@ +# Federated Semantics Graph Architecture + +**Document status:** Draft for client alignment +**Audience:** Client architects, platform engineers, GraphRAG developers, security engineering, and application-security teams +**Scope:** Combining Confluence multimodal content, Code Property Graphs, SBOM/CVE/scanner evidence, and domain inventory into one retrieval and reasoning architecture + + +## 1. Executive Summary + +The client requirement combines two different information types: + +1. **Confluence and documentation data** — human-authored pages, diagrams, images, tables, attachments, runbooks, and design notes. (Targeted for MVP1, prototype delivered) +2. **Source-code-derived Code Property Graph data** — AST, call graph, control flow, data flow, methods, classes, endpoints, source/sink paths, and vulnerability-relevant program behavior. (Partial target delivered for MVP1, prototypes delivered) + +These should not be forced into a single extraction model. + +The recommended architecture is a **Federated Semantics Graph**: (Targeted MVP2) + +- Confluence is modeled as a **Multimodal Lexical Semantics Graph**. (Targeted MVP1, Neo4j solution delievered) +- Source code is modeled as a **Program Semantics Graph**, implemented through a **Code Property Graph** and semantic overlay. (Targeted MVP1) +- SBOM, CVEs, scanner results, tickets, exceptions, and attestations are modeled as an **Evidence Semantics Graph**. (Targeted MVP2) +- Applications, repositories, APIs, services, owners, controls, risks, and findings are modeled as the **Domain Semantics Graph**. (Targeted MVP1, Neo4j solution delivered) +- The Domain Semantics Graph serves as the canonical identity spine connecting the other graph families. + +The retrieval pattern is **Multi-Graph RAG**: a query planner selects the appropriate graph/index, performs vector search only as an entry point, traverses graph relationships to find evidence, and produces answers with provenance back to source documents, code locations, and evidence artifacts. (Targeted MVP2) + + +## 2. Architecture Principle + +> Do not collapse Confluence, source code, SBOM/CVE data, and findings into one flat graph. +> +> Keep each graph specialized, then join them through canonical domain identity and typed evidence relationships. + +The system should preserve three separate kinds of truth: + +| Truth Type | Source | Graph Family | Role | +|---|---|---|---| +| Lexical truth | Confluence, documents, diagrams, images, attachments | Multimodal Lexical Semantics Graph | What the documentation says | +| Program truth | Source code and Code Property Graph | Program Semantics Graph | What the code does | +| Evidence truth | SBOM, CVE, scanners, tickets, exceptions, attestations | Evidence Semantics Graph | What evidence exists | +| Domain truth | CMDB, service catalog, app inventory, ownership, controls | Domain Semantics Graph | What business/security object this is | + + +## 3. Terminology + +Root family → Semantics Graph +Graph families → Lexical / Program / Domain / Evidence +Instances → Multimodal Lexical Graph, Code Property Graph, Domain Identity Graph, Evidence Graph +Pattern → Federated Semantics Graph / Multi-Graph RAG + +| Source | Graph family | Instance | +| --------------------------------------- | ------------------------- | ------------------------------------------- | +| Confluence, docs, diagrams, images | Lexical Semantics Graph | Multimodal Lexical Semantics Graph | +| Source code / CPG | Program Semantics Graph | Code Property Graph + Semantic Code Overlay | +| Apps, repos, services, APIs, controls | Domain Semantics Graph | Canonical Domain / Identity Graph | +| SBOM, CVE, scans, tickets, attestations | Evidence Semantics Graph | Evidence / Provenance Graph | +| End-to-end retrieval | Federated Semantics Graph | Multi-Graph RAG | + + + +## 4. Graph Family Taxonomy + +```mermaid +graph TD + SG[Semantics Graph Family] + + SG --> LSG[Multimodal Lexical Semantics Graph] + SG --> PSG[Program Semantics Graph] + SG --> DSG[Domain Semantics Graph] + SG --> ESG[Evidence Semantics Graph] + + LSG --> CONF[Confluence Pages] + LSG --> DOCS[Documents / PDFs] + LSG --> IMG[Images / Diagrams] + LSG --> TABLES[Tables / Attachments] + + PSG --> CPG[Code Property Graph] + PSG --> AST[AST] + PSG --> CFG[Control Flow] + PSG --> DDG[Data Flow] + PSG --> CALLS[Call Graph] + + DSG --> APP[Application] + DSG --> API[API Endpoint] + DSG --> REPO[Repository] + DSG --> CTRL[Control] + DSG --> RISK[Risk] + DSG --> OWNER[Owner / Team] + + ESG --> SBOM[SBOM] + ESG --> CVE[CVE] + ESG --> SCAN[Scanner Finding] + ESG --> TICKET[Ticket] + ESG --> ATTEST[Attestation / Exception] +``` + + +## 5. Target Logical Architecture + +```mermaid +graph LR + subgraph Sources[Source Systems] + CONF_SRC[Confluence] + REPO_SRC[Source Repositories] + SBOM_SRC[SBOM / Package Manifests] + CVE_SRC[CVE Feeds] + SCAN_SRC[Security Scanners] + CMDB_SRC[CMDB / Service Catalog] + TICKET_SRC[Jira / Tickets] + end + + subgraph Extraction[Specialized Extraction Pipelines] + LEX_EXT[Multimodal Lexical Extractor] + CPG_EXT[CPG Extractor] + EVD_EXT[Evidence Extractor] + DOM_EXT[Domain Inventory Extractor] + end + + subgraph Graphs[Specialized Graph Families] + LEX_GRAPH[Multimodal Lexical Semantics Graph] + PROG_GRAPH[Program Semantics Graph / CPG] + EVD_GRAPH[Evidence Semantics Graph] + DOM_GRAPH[Domain Semantics Graph / Identity Spine] + end + + subgraph Indexes[Vector and Search Indexes] + LEX_VEC[Lexical Vector Index] + PROG_VEC[Program Semantic Vector Index] + EVD_VEC[Evidence Vector Index] + DOM_VEC[Domain Object Vector Index] + TEXT_INDEX[Full Text / Keyword Index] + end + + subgraph Retrieval[Multi-Graph RAG Runtime] + PLANNER[Query Planner] + RESOLVER[Identity Resolver] + TRAVERSAL[Graph Traversal Engine] + ASSEMBLER[Context Assembler] + PROVENANCE[Provenance Validator] + end + + subgraph Output[Answer Layer] + ANSWER[Grounded Answer] + PATH[Graph Path] + SOURCES[Source Evidence] + GAPS[Confidence / Gaps] + end + + CONF_SRC --> LEX_EXT --> LEX_GRAPH --> LEX_VEC + CONF_SRC --> LEX_EXT --> TEXT_INDEX + + REPO_SRC --> CPG_EXT --> PROG_GRAPH --> PROG_VEC + + SBOM_SRC --> EVD_EXT + CVE_SRC --> EVD_EXT + SCAN_SRC --> EVD_EXT + TICKET_SRC --> EVD_EXT + EVD_EXT --> EVD_GRAPH --> EVD_VEC + + CMDB_SRC --> DOM_EXT --> DOM_GRAPH --> DOM_VEC + + LEX_GRAPH <--> DOM_GRAPH + PROG_GRAPH <--> DOM_GRAPH + EVD_GRAPH <--> DOM_GRAPH + + PLANNER --> LEX_VEC + PLANNER --> PROG_VEC + PLANNER --> EVD_VEC + PLANNER --> DOM_VEC + PLANNER --> TEXT_INDEX + + PLANNER --> RESOLVER --> TRAVERSAL --> ASSEMBLER --> PROVENANCE + + PROVENANCE --> ANSWER + PROVENANCE --> PATH + PROVENANCE --> SOURCES + PROVENANCE --> GAPS +``` + + +## 6. Specialized Extraction Pipelines + +### 6.1 Confluence: Multimodal Lexical Semantics Graph + +Confluence extraction should produce a lexical graph because the source is human-authored content. + +#### Source Types + +- Pages +- Sections +- Paragraphs +- Tables +- Images +- Architecture diagrams +- Sequence diagrams +- Attachments +- Comments +- Page hierarchy +- Page metadata + +#### Output Objects + +- `ConfluencePage` +- `Section` +- `TextChunk` +- `Statement` +- `Fact` +- `Topic` +- `TableChunk` +- `ImageArtifact` +- `DiagramArtifact` +- `AttachmentArtifact` +- `ExtractedEntity` +- `ExtractedRelationship` + +#### Embedding Targets + +Embed meaningful lexical units, not arbitrary fragments. + +Good embedding targets: + +- Page summary +- Section summary +- Text chunk +- Statement +- Table summary +- Diagram summary +- Image caption plus extracted text +- Attachment summary + +Avoid embedding: + +- Page title only +- Image filename only +- Raw HTML +- Raw table cells without context +- Duplicated boilerplate + +```mermaid +graph TD + CONF[Confluence Page] + CONF --> SEC[Section] + SEC --> CHUNK[Text Chunk] + CHUNK --> STMT[Statement] + STMT --> FACT[Fact] + SEC --> TOPIC[Topic] + CONF --> TABLE[Table Artifact] + CONF --> IMG[Image / Diagram Artifact] + CONF --> ATTACH[Attachment] + + TABLE --> TABLE_SUM[Table Summary] + IMG --> OCR[Extracted Text / OCR] + IMG --> VISUAL[Visual Summary] + ATTACH --> ATTACH_SUM[Attachment Summary] + + CHUNK --> LEX_EMB[Embedding] + STMT --> STMT_EMB[Embedding] + TABLE_SUM --> TABLE_EMB[Embedding] + VISUAL --> IMG_EMB[Embedding] +``` + + +### 6.2 Source Code: Program Semantics Graph / CPG + +Source-code extraction should produce a Program Semantics Graph, not a lexical graph. + +The raw CPG preserves program structure and behavior. A semantic overlay turns raw graph structures into retrieval-grade objects. + +#### Source Types + +- Repositories +- Branches +- Commits +- Source files +- Package manifests +- Build files +- Code ownership metadata + +#### Raw CPG Objects + +- `File` +- `Namespace` +- `TypeDecl` +- `Method` +- `Parameter` +- `Call` +- `Identifier` +- `Literal` +- `ControlStructure` +- `Block` +- `Local` +- `Return` +- `AST_EDGE` +- `CALL_EDGE` +- `CFG_EDGE` +- `DDG_EDGE` +- `CDG_EDGE` + +#### Semantic Overlay Objects + +- `MethodSemanticUnit` +- `ClassSemanticUnit` +- `EndpointSemanticUnit` +- `CallChainSummary` +- `DataFlowSlice` +- `SourceSinkPath` +- `AuthGuardPattern` +- `SanitizerPattern` +- `TrustBoundaryCrossing` +- `VulnerabilityPatternCandidate` + +#### Embedding Targets + +Embed higher-level program-semantic units. + +Good embedding targets: + +- Method behavior summary +- Endpoint behavior summary +- Class/module summary +- Source-to-sink path summary +- Data-flow slice summary +- Call-chain summary +- Authorization pattern summary +- Vulnerability candidate rationale + +Avoid embedding: + +- Every AST node +- Every identifier +- Every literal +- Every raw edge +- Method name only +- Raw JSON dumps of CPG nodes + +```mermaid +graph TD + REPO[Repository] + REPO --> FILE[Source File] + FILE --> CPG[Raw Code Property Graph] + + CPG --> AST[AST] + CPG --> CALL[Call Graph] + CPG --> CFG[Control Flow] + CPG --> DDG[Data Flow] + CPG --> CDG[Control Dependence] + + CPG --> METHOD[Method] + METHOD --> MSU[Method Semantic Unit] + METHOD --> DFS[Data Flow Slice] + METHOD --> SSP[Source-to-Sink Path] + METHOD --> EP[Endpoint Semantic Unit] + + MSU --> M_EMB[Method Embedding] + DFS --> DFS_EMB[Data Flow Embedding] + SSP --> SSP_EMB[Path Embedding] + EP --> EP_EMB[Endpoint Embedding] +``` + + +### 6.3 Evidence: Evidence Semantics Graph + +Evidence extraction should normalize scanner and assessment data into evidence objects that can support findings, controls, risks, and verdicts. + +#### Source Types + +- SBOMs +- CVE feeds +- Dependency scans +- SAST findings +- DAST findings +- IaC findings +- Container findings +- Cloud configuration findings +- Tickets +- Exceptions +- Attestations +- Manual assessment notes + +#### Output Objects + +- `EvidenceObject` +- `Finding` +- `ScannerFinding` +- `SBOMComponent` +- `Package` +- `CVE` +- `Vulnerability` +- `Exception` +- `Ticket` +- `Attestation` +- `AssessmentResult` + +#### Embedding Targets + +- Finding summary +- Vulnerability summary +- Evidence bundle summary +- Exception rationale +- Ticket summary +- Assessment rationale + +```mermaid +graph TD + SBOM[SBOM] --> PKG[Package / Component] + CVE_FEED[CVE Feed] --> CVE[CVE] + SCAN[Scanner Result] --> FIND[Finding] + TICKET[Ticket] --> FIND + EXC[Exception] --> FIND + ATTEST[Attestation] --> EVID[Evidence Object] + + CVE --> PKG + PKG --> FIND + FIND --> EVID + FIND --> FIND_SUM[Finding Summary] + FIND_SUM --> FIND_EMB[Finding Embedding] +``` + +--- + +### 6.4 Domain: Canonical Identity Spine + +The Domain Semantics Graph is the join layer. It should be populated from authoritative systems where possible. + +#### Canonical Objects + +- `Application` +- `Service` +- `Repository` +- `APIEndpoint` +- `Environment` +- `BusinessCapability` +- `Owner` +- `Team` +- `Control` +- `ControlIntent` +- `Risk` +- `RiskScenario` +- `Finding` +- `EvidenceObject` +- `DataClassification` +- `SystemBoundary` + +#### Why This Layer Matters + +The same system may appear under many names: + +- `payment-service` +- `Payment Service` +- `payments-api` +- `github.com/org/payment-service` +- `PAY-APP-042` +- `prod-payment-api` + +These must resolve to one canonical domain identity. + +```mermaid +graph TD + ALIAS1[payment-service] + ALIAS2[Payment Service] + ALIAS3[payments-api] + ALIAS4[github.com/org/payment-service] + ALIAS5[PAY-APP-042] + + CANON[Canonical Application: Payment Service] + + ALIAS1 --> CANON + ALIAS2 --> CANON + ALIAS3 --> CANON + ALIAS4 --> CANON + ALIAS5 --> CANON + + CANON --> REPO[Repository] + CANON --> API[API Endpoint] + CANON --> OWNER[Owner / Team] + CANON --> CTRL[Applicable Controls] + CANON --> RISK[Risk Scenarios] + CANON --> FIND[Findings] +``` + + +## 7. Cross-Graph Relationship Model + +The system is joined through typed relationships. + +```mermaid +graph TD + APP[Application] + REPO[Repository] + API[API Endpoint] + METHOD[CPG Method] + FLOW[Data Flow Slice] + CONF_PAGE[Confluence Page] + CONF_SEC[Confluence Section] + STMT[Statement] + CTRL[Control Intent] + RISK[Risk Scenario] + FIND[Finding] + EVID[Evidence Object] + CVE[CVE] + PKG[Package] + TICKET[Ticket] + + APP -->|IMPLEMENTED_BY| REPO + REPO -->|HAS_PROGRAM_GRAPH| METHOD + API -->|IMPLEMENTED_BY| METHOD + API -->|DOCUMENTED_BY| CONF_SEC + CONF_PAGE -->|HAS_SECTION| CONF_SEC + CONF_SEC -->|ASSERTS| STMT + STMT -->|SUPPORTS| CTRL + CTRL -->|MITIGATES| RISK + METHOD -->|HAS_DATA_FLOW| FLOW + FLOW -->|EVIDENCES| FIND + FIND -->|AFFECTS| APP + FIND -->|VIOLATES| CTRL + FIND -->|EVIDENCED_BY| EVID + CVE -->|AFFECTS| PKG + PKG -->|USED_BY| REPO + TICKET -->|TRACKS| FIND +``` + + +## 8. Retrieval Architecture + +The retrieval system must act as a planner across multiple graph and vector indexes. + +### 8.1 Retrieval Modes + +| Query Type | Primary Graph | Secondary Graphs | Example | +|---|---|---|---| +| Documentation question | Lexical Graph | Domain Graph | “What does the runbook say about refunds?” | +| Code behavior question | Program Graph / CPG | Domain + Evidence | “Where is authorization checked?” | +| Security finding question | Evidence Graph | Program + Domain + Lexical | “What proves this finding?” | +| Application risk question | Domain Graph | Evidence + Program + Lexical | “Which applications have similar access risk?” | +| Multi-hop governance question | Domain Graph | All | “Does this app satisfy privileged access controls?” | + +### 8.2 Runtime Flow + +```mermaid +sequenceDiagram + participant User + participant Planner as Query Planner + participant Domain as Domain Graph + participant Lex as Lexical Graph / Vector Index + participant Prog as Program Graph / CPG Index + participant Evid as Evidence Graph + participant Traversal as Graph Traversal Engine + participant Assembler as Context Assembler + participant LLM as LLM Answer Layer + + User->>Planner: Ask question + Planner->>Domain: Resolve application/API/control/finding candidates + Planner->>Lex: Search Confluence docs if documentation evidence is needed + Planner->>Prog: Search CPG semantic units if code behavior is needed + Planner->>Evid: Search findings/CVEs/SBOM/tickets if evidence is needed + Domain-->>Planner: Canonical domain nodes + Lex-->>Planner: Lexical candidates + Prog-->>Planner: Program candidates + Evid-->>Planner: Evidence candidates + Planner->>Traversal: Traverse typed relationships across graphs + Traversal-->>Assembler: Graph paths and source references + Assembler->>Assembler: Deduplicate, rank, validate provenance + Assembler->>LLM: Provide grounded context + LLM-->>User: Answer with evidence, graph path, and gaps +``` + + +## 9. Example End-to-End Query + +### Question + +> Does the payment refund endpoint enforce authorization correctly? + +### Expected Multi-Graph Plan + +```mermaid +graph TD + Q[User Question: Refund authorization] + + Q --> DOM_SEARCH[Domain Search] + DOM_SEARCH --> APP[Payment Application] + DOM_SEARCH --> API[Refund API Endpoint] + DOM_SEARCH --> CTRL[Authorization Control] + + Q --> LEX_SEARCH[Lexical Search] + LEX_SEARCH --> DOC[Confluence Refund Runbook] + DOC --> EXPECT[Statement: Refund requires Finance or Manager role] + + Q --> PROG_SEARCH[Program Graph Search] + PROG_SEARCH --> METHOD[RefundController.issueRefund] + METHOD --> FLOW[Request role/user context to refund execution path] + FLOW --> GUARD{Authorization Guard Present?} + + Q --> EVD_SEARCH[Evidence Search] + EVD_SEARCH --> FIND[Finding: Missing role validation] + EVD_SEARCH --> TICKET[Jira Ticket SEC-123] + + APP --> API + API --> METHOD + EXPECT --> CTRL + FLOW --> FIND + FIND --> CTRL + + GUARD -->|No| GAP[Gap: Authorization not proven before refund execution] + GAP --> ANSWER[Grounded Answer] +``` + +### Expected Answer Shape + +The answer must include: + +- Conclusion +- Code evidence +- Documentation evidence +- Finding/evidence references +- Graph path +- Confidence +- Gaps or missing evidence + +Example: + +```text +Conclusion: Authorization enforcement is not fully proven for the refund endpoint. + +Evidence: +- Confluence states that refund actions require Finance or Manager role approval. +- The API endpoint maps to RefundController.issueRefund(). +- The CPG path shows request data reaches the refund execution path. +- No authorization guard was observed before refund execution in the traversed control-flow path. +- Existing finding SEC-123 also references missing role validation. + +Graph path: +Application → API Endpoint → CPG Method → Data Flow Slice → Finding → Control + +Gap: +No confirmed guard/sanitizer node was found on the controlling path before refund execution. +``` + + +## 10. Embedding Strategy + +Embeddings should be used as semantic entry points, not as proof. + +### 10.1 Embedding Targets by Graph + +| Graph | Embed | Do Not Embed | +|---|---|---| +| Lexical Graph | Chunks, statements, section summaries, image summaries, table summaries | Raw HTML, filenames, duplicated boilerplate | +| Program Graph | Method summaries, endpoint summaries, data-flow slices, call-chain summaries, source-to-sink path summaries | Every AST node, every literal, every raw edge | +| Evidence Graph | Finding summaries, CVE summaries, ticket summaries, exception rationales | CVE ID alone, scanner ID alone | +| Domain Graph | Application summaries, API descriptions, control intent, risk scenarios | Random inventory IDs, names without context | + +### 10.2 Embedding Text Contracts + +Every embedded object must have a stable text surface. + +#### Lexical Example + +```text +Object: ConfluenceSection +Embedding text: +Title: Refund Authorization +Page: Payment Operations Runbook +Summary: Refund operations require Finance or Manager role approval before execution. +Extracted facts: Refunds above threshold require dual approval. +``` + +#### Program Example + +```text +Object: MethodSemanticUnit +Embedding text: +Method: RefundController.issueRefund +Summary: Handles refund request and calls payment reversal logic. +Observed behavior: Request user context reaches refund execution path. +Authorization: No confirmed role guard observed before processRefund call. +Source: payment-service/src/refund/RefundController.java +``` + +#### Evidence Example + +```text +Object: Finding +Embedding text: +Finding: Missing authorization guard on refund endpoint +Summary: Refund execution path can be reached without confirmed role validation. +Affected application: Payment Service +Related control: Privileged transaction authorization +Evidence: CPG data-flow path and scanner result SEC-123 +``` + + +## 11. Provenance and Evidence Requirements + +Every answer must trace back to evidence. + +### 11.1 Required Provenance Fields + +| Evidence Type | Required Provenance | +|---|---| +| Confluence | page ID, page title, version, section, chunk ID, URL, timestamp | +| Image/Diagram | image URI, source page, extracted text, visual summary, model used, timestamp | +| Code | repo, commit SHA, branch, file path, method, line range, CPG node ID | +| CPG path | source node, sink node, path edges, guard/sanitizer evidence, traversal timestamp | +| SBOM | artifact ID, package name, version, supplier, generated timestamp | +| CVE | CVE ID, affected package/version, severity source, retrieval timestamp | +| Scanner finding | scanner, finding ID, rule ID, severity, target, evidence URI | +| Ticket | ticket ID, status, owner, linked finding, timestamps | + +### 11.2 Provenance Flow + +```mermaid +graph LR + ANSWER[Answer Claim] + ANSWER --> SUPPORT[Supported By] + SUPPORT --> GP[Graph Path] + SUPPORT --> EVID[Evidence Object] + EVID --> SOURCE[Source Artifact] + SOURCE --> LOC[Exact Location] + + LOC --> CONF_LOC[Confluence Page / Section / Version] + LOC --> CODE_LOC[Repo / Commit / File / Line] + LOC --> SCAN_LOC[Scanner Finding ID] + LOC --> SBOM_LOC[SBOM Component] + LOC --> TICKET_LOC[Ticket ID] +``` + +--- + +## 12. Storage and Indexing Responsibilities + +This document is storage-neutral, but the recommended responsibility split is: + +| Capability | Recommended Store Responsibility | +|---|---| +| Raw source artifacts | S3 or equivalent artifact store | +| Lexical graph relationships | Graph database | +| Program graph / CPG relationships | Graph database or CPG-native store | +| Domain identity spine | Graph database | +| Evidence relationships | Graph database | +| Vector similarity | Vector index or graph-native vector index | +| Full-text search | Search index | +| Operational state | Operational database | +| Provenance metadata | Artifact store + graph metadata | + +The important design decision is not the product; it is the separation of responsibility: + +```text +Artifact store = source truth +Graph store = relationship truth +Vector index = similarity entry point +Search index = keyword/discovery truth +Operational store = workflow/state truth +``` + + +## 13. How AWS GraphRAG Toolkit Fits + +The AWS GraphRAG Toolkit is useful, but it should not be treated as the only extraction model. + +### 13.1 Suitable Uses + +Use toolkit-style lexical GraphRAG for: + +- Confluence pages +- PDFs +- Documents +- Architecture notes +- Runbooks +- Images and diagrams after text/visual summarization +- Statements, facts, topics, and entity extraction + +Use BYOKG-style patterns for: + +- Existing CPG graph +- Existing domain graph +- Existing evidence graph +- Multi-strategy graph retrieval over already-built knowledge graphs + +### 13.2 Custom Layers Required + +We still need custom layers for: + +- CPG extraction and normalization +- Semantic Code Overlay generation +- CPG path summarization +- Source-to-sink path indexing +- Canonical identity resolution +- Cross-graph linking +- Query planning across graph families +- Provenance validation +- Multi-graph context assembly + + +## 14. Required Components + +```mermaid +graph TD + subgraph BuildTime[Build-Time Components] + CONN[Source Connectors] + EXTRACT[Extraction Workers] + CANON[Canonical Identity Resolver] + LINKER[Cross-Graph Linker] + SUMM[Semantic Summarizers] + EMBED[Embedding Workers] + GRAPH_LOAD[Graph Loader] + IDX_LOAD[Index Loader] + end + + subgraph RunTime[Runtime Components] + API[GraphRAG API] + AUTH[AuthN/AuthZ] + PLAN[Query Planner] + SEARCH[Vector/Search Router] + GRAPHQ[Graph Query Engine] + CTX[Context Assembler] + PROV[Provenance Validator] + LLM[LLM Gateway] + OBS[Observability] + end + + CONN --> EXTRACT --> CANON --> LINKER --> SUMM --> EMBED + LINKER --> GRAPH_LOAD + EMBED --> IDX_LOAD + + API --> AUTH --> PLAN --> SEARCH + PLAN --> GRAPHQ + SEARCH --> CTX + GRAPHQ --> CTX + CTX --> PROV --> LLM + API --> OBS + PLAN --> OBS + LLM --> OBS +``` + + +## 15. Phase 1 Implementation Scope + +### 15.1 Goal + +Prove that a single user question can traverse: + +1. Confluence documentation +2. Code Property Graph evidence +3. Domain identity spine +4. Evidence/finding records + +and produce a grounded answer with provenance. + +### 15.2 Minimum Phase 1 Data Set + +- 1 application +- 1 repository +- 1 CPG export +- 5–10 Confluence pages +- 1 SBOM +- 5–10 scanner findings or synthetic findings +- 3–5 controls or control intents +- 1–2 representative questions + +### 15.3 Phase 1 Acceptance Criteria + +A successful Phase 1 must demonstrate: + +- Confluence pages are chunked and vectorized as lexical artifacts. +- Images/diagrams receive text or visual summaries before embedding. +- CPG is loaded as a Program Semantics Graph. +- Higher-level CPG semantic units are created and embedded. +- Domain nodes canonicalize application, repository, API, control, and finding identity. +- Cross-graph links connect Confluence, CPG, evidence, and domain objects. +- Query planner chooses the correct graph/index based on question intent. +- Answer includes source references and graph path. +- Missing evidence is reported as a gap, not hallucinated. + + +## 16. Key Client Questions + +### 16.1 Confluence / Lexical Graph + +1. Which Confluence spaces are in scope for Phase 1? +2. Are page permissions required to be preserved during retrieval? +3. Do we need to ingest page comments? +4. Do we need version history, or only latest page versions? +5. Which attachment types are in scope: PDF, Word, Excel, images, draw.io, Lucid, PlantUML, Mermaid? +6. Are diagrams embedded as images, source files, or both? +7. Should image extraction include OCR, visual captioning, or both? +8. What metadata should be retained from Confluence: author, last updated, labels, space, page tree, permissions? + +### 16.2 CPG / Program Graph + +1. Which CPG extractor is being used? +2. What languages are in scope? +3. What graph schema is produced today? +4. Does the CPG include AST, CFG, DDG, CDG, and call graph? +5. Are line numbers and file paths preserved? +6. Is commit SHA captured at extraction time? +7. Are source-to-sink paths already calculated, or must we calculate them? +8. Are authorization guards, sanitizers, and trust boundaries already labeled? +9. What CPG node types are expected to be embedded today? +10. Are embeddings attached to raw nodes or semantic summaries? + +### 16.3 Domain Identity + +1. What is the authoritative application inventory? +2. How do we map repositories to applications? +3. How do we map API endpoints to services and methods? +4. How do we map Confluence pages to applications? +5. How do we map scanner findings to applications and repositories? +6. What alias sources exist? +7. Who owns canonical identity resolution? +8. Are teams and owners authoritative from CMDB, GitHub, LDAP, or another source? + +### 16.4 Evidence + +1. Which evidence types are required in Phase 1? +2. Are SBOMs available per repository, build, release, or application? +3. Which scanner outputs are available? +4. Are scanner findings normalized today? +5. Are CVE severities enriched with EPSS, KEV, exploitability, or business criticality? +6. Are exceptions and compensating controls available? +7. Do findings need a signed or immutable evidence trail? +8. Are tickets authoritative for remediation status? + +### 16.5 Retrieval and Answering + +1. What are the top 5 questions the system must answer first? +2. Should retrieval start with domain objects, Confluence text, CPG semantic units, or evidence findings? +3. Do answers require exact code lines? +4. Do answers require exact Confluence sections? +5. Should the system return graph paths? +6. Should the system return confidence scores? +7. What should happen when evidence is incomplete or conflicting? +8. Should answers be allowed without source evidence? + +### 16.6 Security and Governance + +1. Must Confluence permissions be enforced in retrieval? +2. Must source-code repository permissions be enforced in retrieval? +3. Are there tenant or business-unit boundaries? +4. Are embeddings allowed to contain sensitive code or documentation content? +5. Is there a data retention requirement for embeddings? +6. Are generated summaries considered derived sensitive data? +7. What audit logs are required for retrieval and answer generation? +8. Who can see graph paths and evidence details? + + +## 17. Recommended Naming + +Use the following terms consistently with the client: + +```text +Family: + Semantics Graph + +Instances: + Multimodal Lexical Semantics Graph + Program Semantics Graph + Domain Semantics Graph + Evidence Semantics Graph + +Concrete implementation: + Code Property Graph for program semantics + +Combined architecture: + Federated Semantics Graph + or Semantic Evidence Fabric + +Retrieval pattern: + Multi-Graph RAG + CPG-RAG for the code-specific retrieval path +``` + +Recommended client-facing sentence: + +> We should treat Confluence as a Multimodal Lexical Semantics Graph and source code as a Program Semantics Graph implemented through the Code Property Graph. These should be joined through a Domain Semantics Graph, with SBOM, CVE, scanner findings, tickets, and attestations modeled as an Evidence Semantics Graph. The combined system is a Federated Semantics Graph using Multi-Graph RAG. + + +## 18. Architecture Decision Summary + +| Decision | Recommendation | +|---|---| +| One graph or multiple graph families? | Multiple specialized graph families joined by domain identity. | +| Is Confluence lexical? | Yes. Model as Multimodal Lexical Semantics Graph. | +| Is CPG lexical? | No. Model as Program Semantics Graph. | +| Should CPG nodes be embedded? | Only higher-level semantic units, not every raw node. | +| What joins everything? | Domain Semantics Graph / canonical identity spine. | +| What provides proof? | Source artifacts, CPG paths, evidence objects, and provenance links. | +| What is the runtime pattern? | Multi-Graph RAG with query planning and graph traversal. | +| What custom layer is needed? | Semantic Federation Layer plus CPG Semantic Retrieval Adapter. | + + +## 19. Final Position + +The correct architecture is not a single lexical graph and not a single vectorized CPG. + +The correct architecture is a **Federated Semantics Graph**: + +- **Confluence** becomes a **Multimodal Lexical Semantics Graph**. +- **Source code** becomes a **Program Semantics Graph** through the CPG. +- **SBOM, CVEs, findings, tickets, and attestations** become an **Evidence Semantics Graph**. +- **Applications, repositories, APIs, controls, risks, owners, and findings** become the **Domain Semantics Graph**. + +The Domain Semantics Graph is the spine that connects the others. Embeddings provide semantic entry points. Graph traversal provides relationship proof. Source artifacts provide evidence. The answer layer must return conclusions with graph paths, source references, and explicit gaps. + + +## 20. References + +- AWS GraphRAG Toolkit GitHub repository: https://github.com/awslabs/graphrag-toolkit +- AWS Database Blog: Introducing the GraphRAG Toolkit: https://aws.amazon.com/blogs/database/introducing-the-graphrag-toolkit/ +- GraphRAG.com Lexical Graph reference: https://graphrag.com/reference/knowledge-graph/lexical-graph/ +- AWS BYOKG-RAG announcement: https://aws.amazon.com/about-aws/whats-new/2025/08/amazon-neptune-supports-byokg-rag-toolkit/ + diff --git a/working/graphrag_com_redrawn_architecture.md b/working/graphrag_com_redrawn_architecture.md new file mode 100644 index 000000000..836a4384a --- /dev/null +++ b/working/graphrag_com_redrawn_architecture.md @@ -0,0 +1,575 @@ +# GraphRAG.com-Aligned Architecture: Lexical Graph + Domain Graph + +## 1. Purpose + +This document redraws the proposed architecture using the terminology and graph-shape model from GraphRAG.com. + +The key correction is that we should not introduce competing top-level graph families. For client and developer alignment, we should use the GraphRAG.com graph-shape vocabulary: + +- **Lexical Graph**: graph built from documents, chunks, embeddings, and optionally extracted entities. +- **Domain Graph**: graph built from structured domain entities and relationships. + +Under this model: + +- Confluence is modeled as a **Lexical Graph**. +- The Code Property Graph is modeled as a **Domain Graph instance for the program/code domain**. +- SBOM, CVE, scan findings, tickets, attestations, controls, applications, repositories, and APIs are modeled as **Domain Graph instances**. +- The combined system is a **Multi-Graph GraphRAG architecture** that retrieves across lexical and domain graphs. + +### Scope and status + +This document describes the **target reference architecture** — the end-state graph shapes and retrieval model. It is intentionally forward-looking in places. Two sections keep it grounded in what actually exists: + +- **Section 13 — Concrete Technology Binding** maps every abstract element to the real stack (`lexical-graph`, `document-graph`, `codeproperty-graph`, Neptune Database, OpenSearch Serverless, S3, Bedrock). +- **Section 14 — Implementation Status: Have vs Build** states what is implemented today versus what remains — principally the CPG-RAG overlay and the multi-graph federation layer. + +Companion documents: `neptune_vs_neptune_analytics_cpg_rag_architecture.md` (storage split) and `graphrag_phased_architecture.md` (Phase 1 / Phase 2 scope). + +--- + +## 2. GraphRAG.com Grounding + +GraphRAG.com defines GraphRAG as a set of RAG patterns that leverage graph structure for retrieval. Different patterns require different graph shapes. + +For this architecture, the important graph shapes are: + +| GraphRAG.com graph type | Meaning | Client-specific use | +|---|---|---| +| **Lexical Graph** | Document/chunk-centered graph with embeddings and source lineage. | Confluence, documentation, diagrams, tables, images, attachments. | +| **Domain Graph** | Structured graph of real-world/domain entities and relationships. | Applications, APIs, repos, controls, CPG, SBOM, CVE, scans, findings, tickets, evidence. | + +The architecture therefore combines one or more Lexical Graphs with one or more Domain Graphs. + +--- + +## 3. Corrected Terminology + +| Term | GraphRAG.com alignment | Definition | +|---|---|---| +| Lexical Graph | Graph shape | A document/chunk graph used for human-authored content. Chunks hold human-readable text and embeddings. | +| Multimodal Lexical Graph | Lexical Graph instance | A Lexical Graph extended for Confluence pages, diagrams, images, tables, screenshots, and attachments. | +| Domain Graph | Graph shape | A structured graph of real-world or domain-specific entities and relationships. | +| Program Domain Graph | Domain Graph instance | A Domain Graph representing program/code behavior. The Code Property Graph is the concrete implementation. | +| Code Property Graph | Domain Graph implementation | Structured graph of code syntax, call flow, control flow, and data flow. | +| Semantic Code Overlay | Domain Graph overlay | Higher-level code behavior summaries, method summaries, endpoint summaries, call-chain summaries, and source-to-sink summaries attached above raw CPG nodes. | +| Security/Evidence Domain Graph | Domain Graph instance | Structured graph for SBOM, CVEs, scans, tickets, findings, exceptions, attestations, and assessment results. | +| Canonical Domain Graph | Domain Graph instance | Identity spine for applications, services, repositories, APIs, controls, owners, environments, and teams. | +| Multi-Graph GraphRAG | Retrieval architecture | Retrieval pattern that searches and traverses across Lexical Graphs and Domain Graphs to assemble grounded context. | + +--- + +## 4. Architecture at a Glance + +```mermaid +flowchart TB + Q[User Question] --> ORCH[GraphRAG Retrieval Orchestrator] + + ORCH --> LRET[Lexical Retrieval] + ORCH --> DRET[Domain Retrieval] + ORCH --> XRET[Cross-Graph Traversal] + + subgraph LG[Lexical Graph Type] + direction TB + CONF[Confluence Pages] + DOCS[Docs / Attachments] + IMG[Images / Diagrams / Tables] + CHUNKS[Chunks + Embeddings] + LENTS[Extracted Entities / Topics / Statements] + + CONF --> CHUNKS + DOCS --> CHUNKS + IMG --> CHUNKS + CHUNKS --> LENTS + end + + subgraph DG[Domain Graph Type] + direction TB + + subgraph CANON[Canonical Domain Graph] + APP[Application] + SVC[Service] + API[API Endpoint] + REPO[Repository] + CTRL[Control] + OWNER[Owner / Team] + end + + subgraph PROG[Program Domain Graph] + CPG[Code Property Graph] + METHOD[Method / Class / Module] + FLOW[Call Flow / Data Flow / Control Flow] + SLICE[Source-to-Sink Slice] + CODESEM[Semantic Code Overlay] + end + + subgraph EVID[Security / Evidence Domain Graph] + SBOM[SBOM Package] + CVE[CVE] + SCAN[Scanner Finding] + TICKET[Ticket / Exception] + ATTEST[Attestation / Assessment] + FINDING[Security Finding] + end + end + + LRET --> CHUNKS + DRET --> APP + DRET --> CPG + DRET --> FINDING + + LENTS -.mentions / describes.-> APP + LENTS -.mentions / describes.-> API + LENTS -.mentions / describes.-> CTRL + + APP --> SVC + APP --> REPO + SVC --> API + REPO --> CPG + API --> METHOD + METHOD --> FLOW + FLOW --> SLICE + CODESEM --> METHOD + CODESEM --> SLICE + + SBOM --> REPO + CVE --> SBOM + SCAN --> FINDING + TICKET --> FINDING + ATTEST --> FINDING + FINDING --> APP + FINDING --> CTRL + FINDING --> SLICE + + XRET --> CTX[Context Assembly] + CHUNKS --> CTX + APP --> CTX + API --> CTX + CPG --> CTX + FINDING --> CTX + CTX --> LLM[LLM Answer] + LLM --> ANS[Grounded Answer + Provenance] +``` + +--- + +## 5. Source-to-Graph Mapping + +```mermaid +flowchart LR + subgraph SOURCES[Source Systems] + CONF[Confluence] + CODE[Source Code Repositories] + SBOM_SRC[SBOM Sources] + SCAN_SRC[Security Scanners] + TICKET_SRC[Ticketing Systems] + CMDB[CMDB / Service Catalog] + CTRL_SRC[Controls / Policies] + end + + subgraph LGRAPH[Lexical Graph] + L_DOC[Document] + L_CHUNK[Chunk + Embedding] + L_ENTITY[Extracted Entity] + L_STMT[Statement / Fact] + L_MEDIA[Image / Diagram / Table Summary] + end + + subgraph DGRAPH[Domain Graphs] + D_APP[Application] + D_REPO[Repository] + D_API[API Endpoint] + D_CTRL[Control] + D_CPG[Code Property Graph] + D_METHOD[Method / Flow / Slice] + D_SBOM[Package] + D_CVE[CVE] + D_FINDING[Finding] + D_OWNER[Owner / Team] + end + + CONF --> L_DOC + L_DOC --> L_CHUNK + L_DOC --> L_MEDIA + L_CHUNK --> L_ENTITY + L_CHUNK --> L_STMT + L_MEDIA --> L_ENTITY + + CODE --> D_CPG + D_CPG --> D_METHOD + + SBOM_SRC --> D_SBOM + SCAN_SRC --> D_FINDING + TICKET_SRC --> D_FINDING + CMDB --> D_APP + CMDB --> D_REPO + CMDB --> D_OWNER + CTRL_SRC --> D_CTRL + + L_ENTITY -.entity linking.-> D_APP + L_ENTITY -.entity linking.-> D_API + L_ENTITY -.entity linking.-> D_CTRL + L_STMT -.supports.-> D_FINDING + + D_APP --> D_REPO + D_REPO --> D_CPG + D_APP --> D_API + D_API --> D_METHOD + D_SBOM --> D_REPO + D_CVE --> D_SBOM + D_FINDING --> D_APP + D_FINDING --> D_CTRL + D_FINDING --> D_METHOD + D_APP --> D_OWNER +``` + +> Correctness note: the dashed `entity linking` / `supports` edges are the **target** cross-graph links. Today that linking is derived from lexical **entity extraction** (fuzzy), not from a canonical identity spine — `lexical-graph` does not preserve custom node metadata through indexing, so a chunk cannot yet carry a stable `graph_node_id` back to a canonical domain node. Until the identity/overlay layer exists (Section 14), treat these as candidate links, not proof. + +--- + +## 6. Retrieval Flow + +This architecture uses multiple GraphRAG retrieval patterns. The query planner decides which retriever to use first and how to expand context. + +```mermaid +sequenceDiagram + actor User + participant Planner as Query Planner + participant LexVec as Lexical Vector Search + participant LexGraph as Lexical Graph Traversal + participant DomSearch as Domain Graph Search + participant CPG as CPG Traversal + participant Evidence as Evidence Graph + participant Assembler as Context Assembler + participant LLM as LLM + + User->>Planner: Ask question + Planner->>Planner: Classify intent: documentation, code behavior, evidence, control, app, or hybrid + + alt Documentation / Confluence question + Planner->>LexVec: Search chunk/table/image embeddings + LexVec->>LexGraph: Return chunk entry points + LexGraph->>LexGraph: Traverse to document, section, extracted entities + end + + alt Domain / app / control question + Planner->>DomSearch: Search canonical domain entities + DomSearch->>DomSearch: Traverse app, repo, API, owner, control relationships + end + + alt Code behavior question + Planner->>DomSearch: Find app/repo/API/method candidates + DomSearch->>CPG: Traverse CPG paths + CPG->>CPG: Validate call flow, control flow, data flow, source/sink path + end + + alt Evidence / risk / finding question + Planner->>Evidence: Search findings, SBOM, CVE, scans, tickets, attestations + Evidence->>Evidence: Traverse to affected app, repo, package, control, method + end + + LexGraph->>Assembler: Lexical evidence + DomSearch->>Assembler: Domain context + CPG->>Assembler: Code proof path + Evidence->>Assembler: Evidence/provenance + + Assembler->>LLM: Grounded context package + LLM->>User: Answer with source, graph path, and gaps +``` + +--- + +## 7. Retrieval Pattern Mapping + +| Question type | Initial graph shape | Retrieval pattern | Expansion path | +|---|---|---|---| +| “What does Confluence say about X?” | Lexical Graph | Basic Retriever / Graph-Enhanced Vector Search | Chunk → document → extracted entity → domain object | +| “Which app owns this API?” | Domain Graph | Pattern Matching / structured domain query | API → service → application → owner | +| “Where is this endpoint implemented?” | Domain Graph | Domain query + CPG traversal | API → repo → CPG method → source location | +| “Does this endpoint enforce authorization?” | Domain Graph | CPG traversal + evidence retrieval | API → method → control flow/data flow → guard/sink → finding | +| “Which CVEs affect this app?” | Domain Graph | Structured evidence traversal | App → repo → package → CVE → finding | +| “What evidence supports this finding?” | Domain Graph + Lexical Graph | Multi-graph retrieval | Finding → CPG slice / scan / ticket / Confluence statement | +| “What changed from documentation to code?” | Lexical + Domain Graphs | Cross-graph comparison | Confluence statement → API/control → CPG implementation path | + +--- + +## 8. Embedding Strategy + +Embeddings should be used as entry points, not as proof. + +```mermaid +flowchart TB + subgraph EMBED[Embedding Targets] + LC[Lexical Chunk] + TS[Table Summary] + IS[Image / Diagram Summary] + MS[Method Summary] + ES[Endpoint Summary] + DFS[Data-Flow Slice Summary] + FS[Finding Summary] + CI[Control Intent Summary] + AS[Application Summary] + end + + subgraph NOEMBED[Do Not Embed as Primary Units] + AST[Raw AST Node] + LIT[Literal] + IDENT[Identifier] + EDGE[Raw Edge] + CVEID[CVE ID Alone] + CTRLID[Control ID Alone] + end + + LC --> VEC[Vector Index] + TS --> VEC + IS --> VEC + MS --> VEC + ES --> VEC + DFS --> VEC + FS --> VEC + CI --> VEC + AS --> VEC + + VEC --> ENTRY[Graph Entry Points] + ENTRY --> TRAV[Graph Traversal] + TRAV --> PROOF[Evidence / Provenance / Source Proof] +``` + +--- + +## 9. Developer Implementation View + +Status legend: green = implemented today, amber = to build, gray = managed infrastructure, red = cross-cloud (to reconcile). Full breakdown in Section 14; the current-vs-recommended ingestion flow is in Section 15. + +```mermaid +flowchart TB + subgraph INGEST[Ingestion & Enrichment Pipelines] + CONN[Confluence Connector] + MM[Lexical / Multimodal Extractor] + JOERN[Joern CPG Extractor] + OVL[document-graph Enrichment + Semantic Code Overlay] + CPGEXT[CPG Delta Ingestor - codeproperty-graph] + SDOM[Structured Domain Ingest - document-graph] + EVIDEXT[Evidence Extractors] + IDRES[Canonical Identity Resolver] + end + + subgraph MODELS[Model Providers] + MISTRAL[Local LLM: Mistral 7B\nCPG enrichment] + BEDROCK[Amazon Bedrock\nLexical embeddings + LLM] + end + + subgraph STORES[Storage / Indexes] + NEPTUNE[(Neptune Database\nGraph truth: lexical + domain)] + NA[(Neptune Analytics\nAlgorithms + vector-in-graph)] + AOSS[(OpenSearch Serverless\nVector indexes)] + S3[(S3\nRaw docs, code, CPG JSON exports)] + end + + subgraph SERVICES[Runtime Services] + PLAN[Multi-Graph Query Planner] + RET[Retriever Registry] + TRAV[Graph Traversal Service] + CTX[Context Assembler] + PROV[Provenance Service] + GAPI[GraphRAG API] + COSMOS[(Cosmos DB - Azure\nOrchestration knowledge)] + end + + CONN --> MM + MM --> NEPTUNE + MM --> BEDROCK + BEDROCK --> AOSS + MM --> S3 + + JOERN --> S3 + S3 --> OVL + OVL --> MISTRAL + OVL --> BEDROCK + OVL --> CPGEXT + CPGEXT --> NEPTUNE + OVL --> AOSS + + SDOM --> NEPTUNE + EVIDEXT --> NEPTUNE + EVIDEXT --> AOSS + EVIDEXT --> S3 + IDRES --> NEPTUNE + + GAPI --> PLAN + PLAN --> COSMOS + PLAN --> RET + RET --> AOSS + RET --> NEPTUNE + RET --> NA + RET --> TRAV + TRAV --> CTX + CTX --> PROV + PROV --> GAPI + + classDef have fill:#D9F2E6,stroke:#2E7D32,color:#1B4332,stroke-width:1.5px; + classDef build fill:#FFE0B2,stroke:#B26A00,color:#5C3A00,stroke-width:1.5px; + classDef infra fill:#ECEFF1,stroke:#607D8B,color:#263238,stroke-width:1.5px; + classDef xcloud fill:#F8D7DA,stroke:#B02A37,color:#5C1119,stroke-width:1.5px; + + class CONN,MM,JOERN,CPGEXT,SDOM have; + class OVL,EVIDEXT,IDRES,PLAN,RET,TRAV,CTX,PROV,GAPI build; + class MISTRAL,BEDROCK,NEPTUNE,NA,AOSS,S3 infra; + class COSMOS xcloud; +``` + +> Enrichment happens **before** Neptune, not inside it. Joern writes CPG JSON to S3; `document-graph` enriches that JSON — calling the **local Mistral 7B** model through a pluggable model-provider interface — and only the finished, enriched graph is bulk-loaded into Neptune by `codeproperty-graph`. `codeproperty-graph` never mutates the graph for enrichment, and code-summary embeddings are written to OpenSearch, not Neptune. This retires the client's Neo4j enrichment step (Section 15). + +--- + +## 10. Critical Design Rule + +Do not collapse every artifact into one generic graph. + +The corrected rule is: + +```text +Confluence = Lexical Graph +Code Property Graph = Domain Graph instance for the program/code domain +SBOM/CVE/findings/tickets = Domain Graph instances for security/evidence +Applications/repos/APIs/controls/owners = Canonical Domain Graph identity spine +Combined retrieval = Multi-Graph GraphRAG +``` + +--- + +## 11. Client Questions to Confirm + +### Lexical Graph / Confluence + +1. Which Confluence spaces are in scope for Phase 1? +2. Should we ingest pages only, or also comments, attachments, embedded diagrams, PDFs, and images? +3. Do diagrams need visual interpretation, OCR, or both? +4. What metadata must be preserved: author, modified date, page version, space, labels, permissions? +5. Should Confluence permissions be enforced at retrieval time? + +### Domain Graph / Program CPG + +1. Which CPG extractor is being used? +2. What languages and repository types are in scope? +3. Are AST, call graph, control flow, and data flow all available? +4. What is the stable identifier for repo, commit, file, method, and graph node? +5. Which higher-level semantic code units should be generated: method summaries, endpoint summaries, data-flow slices, source-to-sink paths, or vulnerability patterns? + +### Domain Graph / Security Evidence + +1. Which evidence sources are in scope: SBOM, CVE, scans, tickets, exceptions, attestations, assessments? +2. What is the canonical finding model? +3. How do findings link to applications, repositories, methods, packages, controls, and tickets? +4. What evidence is considered authoritative versus advisory? +5. How is evidence versioned and invalidated? + +### Cross-Graph Identity + +1. What is the canonical application identifier? +2. How do Confluence app names map to CMDB/service catalog app names? +3. How do repositories map to applications and services? +4. How do API endpoints map to CPG methods? +5. How are aliases, renamed services, archived apps, and duplicate entities handled? + +### Retrieval and Answering + +1. Should retrieval start with lexical chunks, domain entities, or both? +2. Which questions require deterministic graph traversal instead of vector similarity? +3. What provenance must be returned with every answer? +4. Should confidence scoring include vector score, graph path strength, evidence freshness, and authority? +5. What should the system do when Confluence documentation disagrees with code behavior? + +--- + +## 12. Recommended Phase 1 Scope + +Phase 1 should prove the architecture with a narrow vertical slice: + +```text +One application +One repository +One CPG extraction +One Confluence space +One SBOM source +One scanner/finding source +One control family, for example authentication/authorization +``` + +The success criteria should be the ability to answer questions such as: + +1. Which Confluence pages describe this application or API? +2. Where is this API implemented in code? +3. What code path handles the request? +4. Is the expected control described in documentation? +5. Is the control visible in the CPG traversal? +6. Are there findings, CVEs, packages, or tickets related to this implementation? +7. What evidence supports the final answer? + + +--- + +## 13. Concrete Technology Binding + +The abstract graph shapes bind to a specific toolkit + AWS stack. This mirrors the storage split defined in `neptune_vs_neptune_analytics_cpg_rag_architecture.md`. + +| Architecture element | Concrete implementation | +|---|---| +| Lexical Graph engine | graphrag-toolkit `lexical-graph` (`LexicalGraphIndex`, `LexicalGraphQueryEngine`) | +| Structured Domain Graph | `document-graph` (schema-driven ingest, transformers, normalizers, multi-tenancy) | +| Program Domain Graph (CPG) | `codeproperty-graph` (delta ingest → Neptune; `delta_ingestor`, `graph_diff`, `manifest_manager`, `models`, `schema`, `tenant_ops`) | +| CPG extractor | Joern — emits CPG as JSON artifacts to S3 | +| CPG enrichment model | **Local LLM — Mistral 7B** (self-hosted inference endpoint), interfaced by `document-graph` through a pluggable model provider | +| Graph store (lexical + domain truth) | Amazon Neptune Database — single cluster, logical separation by tenant/label encoding (`__Type__tenant_id__`) | +| Vector store | Amazon OpenSearch Serverless (AOSS) | +| Object / artifact store | Amazon S3 (raw docs, code snapshots, CPG JSON exports, images) | +| Lexical embeddings + generation | Amazon Bedrock | +| Analytics engine | Amazon Neptune Analytics — graph algorithms and vector-in-graph analytics (target) | +| Orchestration knowledge store | Azure Cosmos DB (current) — cross-cloud; AWS-native reconciliation is an open decision | +| Retired | Neo4j — client currently enriches CPG in Neo4j; retired in favor of JSON-stage enrichment + Neptune | + +Key bindings: + +- The GraphRAG.com `GraphStore` and `VectorStore` are **separate instances**: Neptune Database is the durable graph truth; OpenSearch Serverless is the vector surface. Bulk embeddings do not live in Neptune. +- Lexical Graph and Domain Graphs are distinct graph *shapes* but can share **one** Neptune cluster, separated logically by tenant/label rather than by separate databases. +- **Enrichment happens before Neptune, not inside a graph database.** Joern emits CPG JSON to S3; `document-graph` enriches that JSON by calling the local **Mistral 7B** model, and only the finished graph is bulk-loaded into Neptune. This retires Neo4j-based enrichment and avoids read-modify-write churn against a live graph — the time and cost saving (Section 15). +- **Model providers are pluggable.** CPG enrichment uses a self-hosted **Mistral 7B**; lexical embeddings/LLM currently use **Bedrock**. Whether the lexical models also move to local inference is an open item. +- **Neptune Analytics** is part of the target for graph algorithms and vector-in-graph analytics; it does not replace Neptune Database as the durable truth or OpenSearch as the primary vector surface (companion storage doc). +- **Cosmos DB (Azure)** is the one cross-cloud dependency in an otherwise AWS stack — Azure↔AWS egress, added latency, split identity boundary. Reconciling it to an AWS-native store (e.g. DynamoDB) is an open decision (Section 14). +- Embeddings are entry points; Neptune traversal is proof — consistent with Section 8. +- Neptune version must be ≥ 1.4.x for the nested-`UNWIND` support the lexical-graph toolkit relies on; `any()` predicates are unsupported, and batched writes should be validated before enabling. + +--- + +## 14. Implementation Status: Have vs Build + +| Capability | Graph shape | Provided by (today) | Status | +|---|---|---|---| +| Lexical ingest, chunking, embeddings, single-graph retrieval | Lexical Graph | `lexical-graph` | Have | +| Structured domain graph, schema-driven ingest, multi-tenancy | Domain Graph | `document-graph` | Have | +| Raw CPG ingest (delta, diff, manifests, tenant ops) | Program Domain Graph | `codeproperty-graph` | Have | +| Durable graph store | — | Amazon Neptune Database | Have (deployed) | +| Vector store | — | Amazon OpenSearch Serverless | Have (deployed) | +| Object / artifact store | — | Amazon S3 | Have | +| CPG extraction to JSON | Program Domain Graph | Joern | Have | +| Lexical embeddings + LLM | — | Amazon Bedrock | Have | +| Analytics engine | — | Amazon Neptune Analytics | Target (not yet deployed) | +| `document-graph` ↔ local LLM (Mistral 7B) enrichment interface | Program Domain overlay | pluggable model provider | Build | +| Multimodal extraction (diagrams / tables / images, OCR) | Lexical Graph | `lexical-graph` extension | Partial | +| Semantic Code Overlay (method / endpoint / source-to-sink summaries + embeddings), enriched on JSON pre-load via Mistral 7B | Program Domain overlay | — | Build | +| Canonical identity spine + resolver (app / service / repo / API / control / owner) | Canonical Domain Graph | minimal identity in `codeproperty-graph` models; no resolver | Build | +| Cross-graph linking (lexical entity ↔ canonical ↔ CPG semantic) | Multi-graph | entity-extraction correlation only (fuzzy) | Build | +| Multi-graph query planner / federated retrieval + provenance | Multi-Graph GraphRAG | `lexical-graph` query engine is single-graph | Build | +| Security / Evidence domain graph (SBOM / CVE / scan / ticket / attestation) | Evidence Domain Graph | — | Build (Phase 2) | +| CPG enrichment in Neo4j | Program Domain Graph | client current-state | Retire → Neptune | +| Orchestration knowledge store | — | Azure Cosmos DB | Have (cross-cloud) — AWS reconciliation open | + +### The missing layer + +The three base capabilities exist as independent parts: lexical retrieval (`lexical-graph`), a structured domain graph (`document-graph`), and raw CPG ingest (`codeproperty-graph`). What does not yet exist is the layer that turns three independent graphs into one CPG-RAG system: + +1. **Semantic Code Overlay builder** — raw CPG → method / endpoint / source-to-sink summaries as `SemanticCodeUnit` nodes, embedded (never raw AST nodes or edges). +2. **Canonical identity spine + resolver** — application / service / repository / API / control / owner identity that both lexical and code entities resolve to. +3. **Cross-graph linker** — replaces today's entity-extraction correlation with explicit canonical-identity edges written at ingest. +4. **Multi-graph query planner** — vector entry across both indexes, then federated traversal (lexical → canonical → CPG proof path) with provenance assembly. + +Items 1–4 are the amber nodes in Section 9. They sit above `lexical-graph`, `document-graph`, and `codeproperty-graph`; everything below them (lexical retrieval, CPG ingest, and the Neptune / OpenSearch / S3 / Bedrock stack) already exists or is decided. diff --git a/working/neptune_vs_neptune_analytics_cpg_rag_architecture.md b/working/neptune_vs_neptune_analytics_cpg_rag_architecture.md new file mode 100644 index 000000000..10d807189 --- /dev/null +++ b/working/neptune_vs_neptune_analytics_cpg_rag_architecture.md @@ -0,0 +1,586 @@ +# CPG-RAG Storage Architecture: Neptune Database, OpenSearch, and Neptune Analytics + +## 1. Purpose + +This document explains how embeddings should be overlaid on the document graph and CPG graph in a CPG-RAG architecture, and why the storage architecture should usually be **split by requirement** rather than putting everything into either **Neptune Database** or **Neptune Analytics**. + +The main architectural point is: + +> **Neptune Database should usually be the durable graph truth. OpenSearch should usually be the scalable vector-search surface. Neptune Analytics should be used selectively for high-speed analytical graph/vector workloads, not as the default home for every graph and vector.** + +This is especially important because **Neptune Analytics is memory-optimized** and priced around memory-optimized capacity. It is powerful, but it should be introduced where the workload actually needs in-memory graph analytics, native vector search inside graph traversal, or large-scale graph algorithms. + +--- + +## 2. Context: What We Are Building + +The client architecture has two Phase 1 graph areas: + +1. **Confluence Lexical Graph** + - Human-authored content. + - Pages, sections, chunks, diagrams, images, tables, and extracted entities/statements. + - Embeddings are created for chunks, section summaries, table summaries, image/diagram summaries, and possibly statements. + +2. **CPG / Program Domain Graph** + - Source-code graph. + - Code Property Graph with AST, call graph, control flow, and data flow. + - A **Semantic Code Overlay** is created above the raw CPG. + - Embeddings are created for higher-level code semantics, not raw AST nodes. + +The important design principle: + +```text +Embedding = semantic entry point +Graph = relationship truth and proof path +Source = evidence truth +``` + +For example, an embedding may help us find a method summary that appears related to authorization, but the CPG traversal must prove whether an authorization guard exists on the relevant execution path. + +--- + +## 3. What an Embedding Looks Like + +An embedding record is not the graph itself. It is a semantic index record that points back to a graph node. + +### 3.1 Document graph embedding example + +```json +{ + "id": "chunk::confluence::PAY::page-123::v17::chunk-004", + "graph_node_id": "chunk::confluence::PAY::page-123::v17::chunk-004", + "index_type": "lexical_chunk", + "source_type": "confluence", + "text": "The Refund API requires Finance role approval before issuing refunds.", + "embedding": [0.013, -0.042, 0.118, "..."], + "metadata": { + "tenant_id": "client-a", + "space_key": "PAY", + "page_id": "123", + "page_version": 17, + "section": "Refund authorization", + "source_uri": "https://confluence/..." + } +} +``` + +The matching graph node in Neptune might be: + +```cypher +(:Chunk { + id: "chunk::confluence::PAY::page-123::v17::chunk-004", + sourceType: "confluence", + pageId: "123", + pageVersion: 17, + sourceUri: "https://confluence/..." +}) +``` + +OpenSearch answers: + +```text +Which chunks are semantically similar to this question? +``` + +Neptune answers: + +```text +What is this chunk connected to? +``` + +Example graph traversal: + +```cypher +(:Chunk)-[:PART_OF]->(:Section)-[:PART_OF]->(:ConfluencePage) +(:Chunk)-[:MENTIONS]->(:APIEndpoint) +(:Chunk)-[:SUPPORTS]->(:Statement) +(:APIEndpoint)-[:IMPLEMENTED_BY]->(:Method) +``` + +### 3.2 CPG semantic overlay embedding example + +Do **not** embed raw low-level CPG nodes as the primary embedding unit: + +```json +{ + "id": "cpg::node::123456", + "label": "CALL", + "name": "assignRole", + "embedding": ["..."] +} +``` + +That is too small and too context-poor. + +Instead, embed a higher-level semantic code unit: + +```json +{ + "id": "code-semantic::payment-api::a1b2::RefundController.issueRefund", + "graph_node_id": "semantic-code-unit::RefundController.issueRefund", + "index_type": "code_method_summary", + "source_type": "cpg", + "text": "HTTP endpoint handler for issuing refunds. Accepts refund request input, loads payment transaction, checks transaction status, and calls PaymentGateway.refund. No authorization guard is observed before the refund operation.", + "embedding": [0.021, -0.018, 0.064, "..."], + "metadata": { + "tenant_id": "client-a", + "application_id": "payment-service", + "repository": "payment-api", + "commit": "a1b2c3d4", + "file": "src/main/java/com/acme/RefundController.java", + "method": "RefundController.issueRefund", + "language": "java", + "semantic_unit_type": "method_summary", + "cpg_node_id": "cpg::method::98765" + } +} +``` + +The matching graph structure in Neptune: + +```cypher +(:SemanticCodeUnit { + id: "semantic-code-unit::RefundController.issueRefund", + type: "method_summary", + repository: "payment-api", + commit: "a1b2c3d4" +}) +-[:SUMMARIZES]-> +(:Method { + cpgNodeId: "cpg::method::98765", + name: "RefundController.issueRefund", + file: "src/main/java/com/acme/RefundController.java" +}) +``` + +The CPG then preserves proof relationships: + +```cypher +(:Method)-[:CALLS]->(:Method) +(:Method)-[:HAS_CONTROL_FLOW]->(:ControlFlowPath) +(:Method)-[:HAS_DATA_FLOW]->(:DataFlowPath) +(:DataFlowPath)-[:REACHES]->(:Sink) +``` + +--- + +## 4. AWS Service Roles + +## 4.1 Neptune Database + +**Primary role:** durable operational graph store. + +Use Neptune Database for: + +- Durable graph truth. +- Canonical graph relationships. +- Application, repository, API, method, chunk, statement, and evidence relationships. +- Repeated operational query workloads. +- Multi-AZ durability and managed graph database operation. +- Production graph state that must be stable and auditable. + +AWS describes Neptune Database as a managed graph database with read replicas, point-in-time recovery, continuous backup to S3, and replication across Availability Zones.[^aws-neptune-db] + +### Good fit + +```text +Application → Repository → CPG → Method → DataFlowPath +ConfluenceChunk → Mentions → APIEndpoint +Finding → EvidencedBy → CPGPath +Package → UsedBy → Repository +``` + +### Weak fit + +Neptune Database is not where we should put large-scale vector search if OpenSearch or S3 Vectors is the better vector store. It is also not the best engine for fast in-memory whole-graph analytics such as PageRank-style algorithms, large community detection, or high-speed exploratory graph algorithms. + +--- + +## 4.2 OpenSearch Serverless + +**Primary role:** scalable vector search and text/hybrid retrieval surface. + +Use OpenSearch for: + +- Document chunk embeddings. +- Statement embeddings. +- Diagram/table/image summary embeddings. +- Semantic code overlay embeddings. +- Finding or evidence-summary embeddings in Phase 2. +- Metadata-filtered vector search. +- Hybrid keyword + vector search where needed. + +In the GraphRAG Toolkit storage model, the lexical graph uses separate `GraphStore` and `VectorStore` instances. The docs describe the graph store as Neptune/Neptune Analytics-compatible and the vector store as OpenSearch Serverless, Neptune Analytics, Postgres/pgvector, or S3 Vectors-compatible.[^toolkit-storage] + +### Good fit + +```text +Question embedding + → OpenSearch vector search + → top-k chunk / statement / code-summary hits + → graph_node_id pointers + → Neptune graph traversal +``` + +### Weak fit + +OpenSearch is not the graph truth. It should not be used to answer relationship questions by itself. It can return candidate IDs, but Neptune should assemble the graph path and provenance. + +--- + +## 4.3 Neptune Analytics + +**Primary role:** in-memory graph analytics and optional unified graph + vector analysis. + +Use Neptune Analytics for: + +- High-speed analytical graph queries. +- Investigatory/exploratory graph workloads. +- Large in-memory graph processing. +- Built-in graph algorithms. +- Native vector search inside graph traversal. +- Temporary or dedicated analytical graphs loaded from Neptune Database or S3. + +AWS describes Neptune Analytics as a **memory-optimized graph database engine for analytics** that stores large graph datasets in memory and supports graph algorithms, low-latency graph queries, and vector search within graph traversals.[^na-what-is] + +AWS also explicitly positions Neptune Analytics as complementary to Neptune Database, and notes that data can be loaded from a Neptune Database graph, a Neptune snapshot, or S3.[^na-what-is] + +### Good fit + +```text +Run large graph algorithms: + PageRank, centrality, community detection, path analysis + +Run high-speed investigative analysis: + Similar methods across thousands of repos + Similar source-to-sink paths across all applications + SBOM vulnerability blast-radius analysis + Large-scale graph-vector exploration +``` + +### Weak fit + +Neptune Analytics should not automatically become the primary home for everything. It is memory-optimized and capacity-driven. It can be expensive if used as an always-on production graph for workloads that do not need in-memory analytics. + +Also, Neptune Analytics has some vector-index constraints. AWS documentation states that a Neptune Analytics graph can have only one vector index, created at graph creation time, with a fixed dimension.[^na-vector-index] + +This matters if we want different embedding dimensions, multiple embedding models, or many separate vector indexes for document chunks, statements, code summaries, findings, and evidence bundles. + +--- + +## 5. Why Not Put Everything in Neptune Analytics? + +Do **not** put everything in Neptune Analytics by default because the workload requirements are different. + +| Requirement | Best default | Reason | +|---|---|---| +| Durable graph truth | Neptune Database | Managed durable graph store with backups, HA, and operational graph query support. | +| High-volume vector search over chunks and summaries | OpenSearch Serverless | Mature vector/text retrieval surface with metadata filtering and scalable index separation. | +| Large-scale graph algorithms | Neptune Analytics | In-memory analytics engine designed for fast graph algorithms and exploratory graph analysis. | +| Native vector + graph traversal in one engine | Neptune Analytics | Useful when the graph/vector loop must run inside one in-memory graph engine. | +| Cheap cold artifact storage | S3 | Raw documents, code snapshots, CPG exports, and batch artifacts should live in S3. | +| Phase 1 operational GraphRAG | Neptune Database + OpenSearch | Strong split between graph truth and vector search. | +| Periodic deep analysis | Neptune Analytics on demand | Load/export subset or snapshot only when needed. | + +Neptune Analytics can reduce operational complexity when graph and vector search need to be unified inside one engine, but that does not mean it is the right default for all data. AWS notes that vector similarity in Neptune Analytics can reduce overhead because you do not manage separate stores or sync pipelines.[^na-vector-similarity] That benefit must be weighed against cost, vector-index constraints, and workload fit. + +--- + +## 6. Why Not Put Everything in Neptune Database? + +Do **not** put everything in Neptune Database either. + +Neptune Database should be the durable graph truth, but it should not become: + +- a massive vector store for all embeddings, +- a document search engine, +- a high-volume lexical retrieval engine, +- or an in-memory analytics engine. + +The GraphRAG Toolkit’s own storage model separates `GraphStore` from `VectorStore`, which reflects the same architecture principle.[^toolkit-storage] + +In a CPG-RAG system, Neptune Database should hold relationships such as: + +```text +Chunk → Mentions → APIEndpoint +APIEndpoint → ImplementedBy → Method +Method → HasDataFlow → SourceSinkPath +Finding → EvidencedBy → SourceSinkPath +Application → Owns → Repository +Repository → Contains → Package +``` + +OpenSearch should hold semantic indexes such as: + +```text +chunk embeddings +statement embeddings +diagram summary embeddings +method summary embeddings +endpoint summary embeddings +data-flow slice embeddings +finding summary embeddings +``` + +Neptune Analytics should be introduced when a workload requires in-memory graph-vector analytics or large graph algorithms. + +--- + +## 7. Recommended Split Architecture + +```mermaid +flowchart TB + Q[User Question] + PLAN[Query Planner] + OS[(OpenSearch Serverless\nVector + Hybrid Search)] + NDB[(Neptune Database\nDurable Graph Truth)] + NA[(Neptune Analytics\nOptional In-Memory Analytics)] + S3[(S3\nRaw Artifacts + Exports)] + CTX[Context Assembler] + LLM[LLM Answer] + + Q --> PLAN + PLAN --> OS + OS -->|graph_node_id hits| NDB + NDB -->|graph paths + provenance| CTX + CTX --> LLM + + S3 -->|raw docs / code / CPG exports| NDB + S3 -->|batch graph loads / snapshots| NA + NDB -->|snapshot / export / subset| NA + NA -->|analytics results / derived relationships| NDB + NA -->|optional analytical context| CTX + + classDef operational fill:#D9F2E6,stroke:#2E7D32,color:#1B4332,stroke-width:1.5px; + classDef analytics fill:#FFBF00,stroke:#B26A00,color:#5C3A00,stroke-width:1.5px; + classDef shared fill:#ECEFF1,stroke:#607D8B,color:#263238,stroke-width:1.5px; + + class OS,NDB,S3 operational; + class NA analytics; + class Q,PLAN,CTX,LLM shared; +``` + +### Recommended default for Phase 1 + +```text +Neptune Database = durable graph truth +OpenSearch = vector search over lexical chunks and semantic code overlay +S3 = raw artifacts, CPG exports, document snapshots +Neptune Analytics = optional, not default +``` + +### Recommended default for Phase 2 + +```text +Neptune Database = durable cross-artifact graph truth +OpenSearch = vector search over documents, code summaries, findings, evidence summaries +S3 = raw evidence and historical exports +Neptune Analytics = on-demand or dedicated analytics for large subgraph analysis +``` + +--- + +## 8. When to Use Neptune Analytics + +Use Neptune Analytics when at least one of these is true: + +1. We need graph algorithms across large subgraphs. +2. We need fast investigative traversal over many applications, repositories, packages, and findings. +3. We need native vector search and graph traversal in one in-memory engine. +4. We need to analyze a snapshot or exported subgraph from S3 or Neptune Database. +5. The workload is exploratory, data-science-heavy, or periodic rather than always-on operational traffic. + +Example workloads: + +```text +Find applications most central to vulnerability blast radius. +Find communities of repositories sharing risky dependency patterns. +Find similar source-to-sink paths across all repositories. +Rank controls by graph centrality across findings and assets. +Run SBOM vulnerability graph algorithms across large dependency graphs. +``` + +AWS documentation describes Neptune Analytics as suitable for investigatory, exploratory, and data-science workloads requiring fast iteration, algorithmic processing, or vector search on graph data.[^na-what-is] + +--- + +## 9. When to Avoid Neptune Analytics + +Avoid Neptune Analytics as the default when: + +1. We only need durable relationship storage. +2. The workload is routine operational GraphRAG. +3. OpenSearch can perform vector search and Neptune Database can perform the required graph traversal. +4. We need multiple independent vector indexes with different dimensions or embedding models. +5. The graph is large, always-on, and cost-sensitive, but does not require in-memory analytics. +6. The data changes frequently and vector update consistency matters. + +Neptune Analytics vector-index updates are documented as not ACID-compliant in the same way as graph updates; vector embedding inserts, deletes, and updates are non-atomic and not isolated.[^na-vector-index] + +That does not make the feature unusable. It means the architecture should deliberately decide when native graph-vector storage is worth the tradeoff. + +--- + +## 10. Decision Matrix + +| Workload | Neptune Database | OpenSearch | Neptune Analytics | Recommendation | +|---|---:|---:|---:|---| +| Confluence chunk retrieval | Low | High | Possible | Use OpenSearch for vectors; Neptune for graph links. | +| CPG relationship traversal | High | Low | Medium | Use Neptune Database as durable CPG/domain graph. | +| CPG semantic overlay vector search | Low | High | Possible | Use OpenSearch first; use Neptune Analytics only if graph-vector traversal must be unified. | +| Source-to-sink proof path | High | Low | Medium | Use Neptune Database for proof path; optionally accelerate with Analytics for large-scale investigation. | +| SBOM/CVE blast-radius analytics | Medium | Medium | High | Phase 2 candidate for Neptune Analytics. | +| Cross-application graph algorithms | Medium | Low | High | Use Neptune Analytics selectively. | +| Always-on operational GraphRAG | High | High | Low/Medium | Neptune Database + OpenSearch is the default. | +| Periodic analytics job | Medium | Low | High | Load subset/snapshot into Neptune Analytics on demand. | +| Full all-in-one graph + vectors | Medium | Medium | High | Only if cost and vector-index constraints are acceptable. | + +--- + +## 11. Recommended Architectural Position + +The recommended position is **not**: + +```text +Put everything in Neptune Analytics. +``` + +And it is also **not**: + +```text +Put everything in Neptune Database. +``` + +The recommended position is: + +```text +Use Neptune Database for durable graph truth. +Use OpenSearch for most vector retrieval. +Use S3 for raw artifacts and batch exports. +Use Neptune Analytics selectively for analytical graph/vector workloads. +``` + +This gives us a better balance of: + +- cost control, +- separation of concerns, +- operational durability, +- retrieval performance, +- analytical capability, +- and future flexibility. + +--- + +## 12. CPG-RAG Retrieval Flow With the Split + +```mermaid +sequenceDiagram + actor User + participant Planner as Query Planner + participant OS as OpenSearch Vector Indexes + participant NDB as Neptune Database + participant NA as Neptune Analytics Optional + participant S3 as S3 Raw Artifacts + participant Ctx as Context Assembler + participant LLM as LLM + + User->>Planner: Ask code/document question + Planner->>OS: Search chunk + semantic code overlay embeddings + OS-->>Planner: Return graph_node_id candidates + Planner->>NDB: Traverse from graph_node_ids + NDB-->>Planner: Return doc links, API, repo, method, CPG path + + alt Deep analytics required + Planner->>NA: Run large graph/vector analytics on selected subgraph + NA-->>Planner: Return ranked paths, communities, or blast-radius results + end + + Planner->>S3: Fetch raw source artifacts if needed + S3-->>Ctx: Confluence page/version, source code, CPG export, evidence file + NDB-->>Ctx: Graph path and provenance + OS-->>Ctx: Matched semantic text surfaces + Planner-->>Ctx: Retrieval plan and confidence signals + Ctx->>LLM: Grounded context package + LLM-->>User: Answer with provenance and gaps +``` + +--- + +## 13. Practical Phase 1 Recommendation + +For Phase 1, the simplest defensible deployment is: + +```text +Graph store: + Neptune Database + +Vector store: + OpenSearch Serverless + +Raw artifacts: + S3 + +Optional analytics: + Neptune Analytics disabled initially, or used only for a small proof-of-value subgraph +``` + +This lets us prove: + +1. Confluence lexical graph ingestion. +2. CPG graph ingestion. +3. Semantic Code Overlay generation. +4. Embeddings for document chunks and code semantic units. +5. OpenSearch vector retrieval. +6. Neptune graph traversal. +7. Code/document provenance. + +Only after that should we evaluate whether Neptune Analytics is justified. + +--- + +## 14. Questions to Ask Before Using Neptune Analytics + +1. Which queries cannot be satisfied by OpenSearch vector search plus Neptune Database traversal? +2. Do we need graph algorithms such as centrality, community detection, or large path analysis? +3. Do we need native vector search inside graph traversal, or is vector-first then graph-traverse sufficient? +4. How large is the graph subset that must be analyzed in memory? +5. Is the workload continuous, scheduled, or ad hoc? +6. What is the cost ceiling for analytics graphs? +7. Can the analytics workload run on an exported subset instead of the full graph? +8. Do we need more than one vector index or more than one embedding dimension? +9. How frequently do vectors change? +10. Are non-atomic vector-index updates acceptable for this use case? + +--- + +## 15. Final Recommendation + +The best architecture is a **split architecture based on workload requirements**: + +```text +Operational GraphRAG path: + OpenSearch → Neptune Database → Context Assembler → LLM + +Analytical GraphRAG path: + Neptune Database / S3 export → Neptune Analytics → derived insights → Neptune Database / Context Assembler +``` + +This avoids overpaying for in-memory analytics where ordinary graph traversal and vector search are sufficient, while still preserving a path to Neptune Analytics for the workloads where it is genuinely valuable. + +--- + +## 16. References + +[^aws-neptune-db]: AWS Documentation, **What Is Amazon Neptune?** Neptune Database is described as highly available with read replicas, point-in-time recovery, continuous backup to S3, and replication across Availability Zones. https://docs.aws.amazon.com/neptune/latest/userguide/intro.html + +[^na-what-is]: AWS Documentation, **What is Neptune Analytics?** Neptune Analytics is described as a memory-optimized graph database engine for analytics that stores graph datasets in memory and supports graph algorithms, low-latency graph queries, and vector search within graph traversals. https://docs.aws.amazon.com/neptune-analytics/latest/userguide/what-is-neptune-analytics.html + +[^na-db-vs-analytics]: AWS Documentation, **When to use Neptune Analytics and when to use Neptune Database.** AWS positions Neptune Database for scalable/high-availability graph database workloads and Neptune Analytics for in-memory analytics over existing graph databases or graph datasets. https://docs.aws.amazon.com/neptune-analytics/latest/userguide/neptune-analytics-vs-neptune-database.html + +[^na-vector-index]: AWS Documentation, **Vector indexing in Neptune Analytics.** Neptune Analytics supports one vector index per graph, created at graph creation time, with fixed dimension. The same page also notes vector updates are not ACID-compliant in the same way as graph updates. https://docs.aws.amazon.com/neptune-analytics/latest/userguide/vector-index.html + +[^na-vector-similarity]: AWS Documentation, **Working with vector similarity in Neptune Analytics.** AWS describes vector similarity search as enabling embeddings to be associated with graph nodes and integrated with graph queries for domain-specific context. https://docs.aws.amazon.com/neptune-analytics/latest/userguide/vector-similarity.html + +[^toolkit-storage]: AWS Labs GraphRAG Toolkit, **Lexical Graph Storage Model.** The toolkit uses separate `GraphStore` and `VectorStore` instances; graph stores include Neptune Database and Neptune Analytics, and vector stores include OpenSearch Serverless, Neptune Analytics, Postgres/pgvector, and S3 Vectors. https://awslabs.github.io/graphrag-toolkit/lexical-graph/storage-model/ + +[^toolkit-byokg-indexing]: AWS Labs GraphRAG Toolkit, **BYOKG-RAG Indexing.** The toolkit describes dense indexes for semantic similarity, fuzzy string indexes, and graph-store indexes where embeddings can live directly in Neptune Analytics. https://awslabs.github.io/graphrag-toolkit/byokg-rag/indexing/ diff --git a/working/specs/cpg-rag-missing-pieces/requirements.md b/working/specs/cpg-rag-missing-pieces/requirements.md new file mode 100644 index 000000000..05c7c7a72 --- /dev/null +++ b/working/specs/cpg-rag-missing-pieces/requirements.md @@ -0,0 +1,143 @@ +# CPG-RAG Delta — Requirements Specification + +- **Status:** Draft for approval +- **Date:** 2026-06-29 +- **Derives from:** `working/adr/ADR-0001-cpg-rag-delta.md` (decisions D1–D6) +- **Companion (next):** `working/specs/cpg-rag-missing-pieces/design.md` + +This SPEC states *what* the CPG-RAG delta must satisfy and *how we will know it is satisfied*. It does not prescribe implementation; that is the design document. Every requirement traces to an ADR decision and carries testable acceptance criteria. + +--- + +## 1. Scope + +**In scope:** the delta between the general-purpose GraphRAG Toolkit and a client CPG-RAG capability — model-agnostic enrichment, local-model injection, enrich-before-Neptune CPG ingestion, semantic code overlay, delta-contract correctness, canonical identity spine, cross-graph linking, and multi-graph retrieval with provenance. + +**Out of scope:** the toolkit's existing, conformant capabilities (reused as-is); client agent orchestration state (MongoDB API / A2A) on the retrieval hot path; evidence-graph (SBOM/CVE/findings) beyond interface stubs — deferred to a later phase. + +**Non-goals:** forking or specialising the toolkit core; introducing a parallel model abstraction; embedding raw AST/edges; mutating a live graph in place. + +--- + +## 2. Definitions + +- **Model seam** — the toolkit's `LLM` / `BaseEmbedding` abstraction resolved via `GraphRAGConfig` (`LLMType`, `EmbeddingType`, `to_llm`, `to_embedding_model`). +- **Enricher** — a `document-graph` `TransformerProvider` that adds derived fields to records. +- **Semantic Code Overlay** — higher-level code-behaviour summaries (`SemanticCodeUnit`) built above raw Joern CPG nodes. +- **Delta ingest** — `codeproperty-graph.DeltaIngestor` skip-or-replace loading keyed on a manifest signature. +- **Enrichment recipe** — the identity of everything that determines enrichment output (model id, prompt version, parameters, overlay builder version). + +--- + +## 3. Phasing + +- **Phase 1 (approvable and buildable now):** R1, R2, R3, R4, R5, R9. Conformant enrichment + local model injection + enrich-before-Neptune + semantic code overlay + correct delta contract, packaged correctly. +- **Phase 2 (requires client identity/source input):** R6, R7, R8. Identity spine, cross-graph linker, multi-graph planner + provenance. + +--- + +## 4. Functional requirements + +### R1 — Model-agnostic enrichment (traces to D2) +As a pipeline author, I want CPG/document enrichment to depend on the toolkit model seam, so that no transformer is coupled to a vendor SDK. + +Acceptance criteria: +- The enrichment transformer SHALL accept an injected `LLM` (and, where it vectorises, a `BaseEmbedding`) and SHALL NOT import a vendor SDK (`openai`, `boto3`) directly. +- WHEN the injected model changes, THEN enrichment behaviour SHALL change with no edit to the enricher class. +- The existing OpenAI- and Bedrock-coupled enricher plugins SHALL be superseded by a single model-agnostic enricher; duplicated transform loops SHALL be removed. +- Unit tests SHALL verify enrichment with a stubbed `LLM` (no network), asserting the derived fields and error handling. + +### R2 — Local model provider injection (traces to D2) +As an operator, I want to run enrichment against a self-hosted Mistral 7B, so that enrichment can run without a cloud model dependency. + +Acceptance criteria: +- A local Mistral 7B SHALL be usable purely as an injected `LLM` implementation via the existing seam, with no CPG-specific code path. +- Selection between local Mistral 7B, Bedrock, and any other provider SHALL be configuration/injection only. +- WHEN no provider is explicitly configured, THEN the toolkit default SHALL remain unchanged (no regression to existing users). + +### R3 — Enrich-before-Neptune CPG pipeline (traces to D3) +As an architect, I want Joern CPG JSON enriched before loading to Neptune, so that the Neo4j read-modify-write step is retired. + +Acceptance criteria: +- The pipeline SHALL consume Joern node/edge JSON, enrich records via R1, and load the finished graph to Neptune via `DeltaIngestor`. +- No stage SHALL read-modify-write a live graph database for enrichment. +- The flow SHALL reuse `codeproperty-graph.from_joern` and the `document-graph` transform pipeline without duplicating their logic. +- An end-to-end test SHALL demonstrate: Joern JSON in → enriched, typed graph in Neptune out, with zero graph-DB round-trip during enrichment. + +### R4 — Semantic Code Overlay (traces to D5) +As a retrieval designer, I want method/endpoint/source-to-sink summaries as first-class graph nodes with embeddings, so that retrieval has meaningful entry points. + +Acceptance criteria: +- A `SemanticCodeUnit` node type SHALL represent method, endpoint, and source-to-sink summaries, linked to the raw CPG node it summarises (e.g. `SUMMARIZES → Method`). +- Only summaries SHALL be embedded; raw AST nodes, identifiers, literals, and edges SHALL NOT be embedded as primary units. +- Summary embeddings SHALL be written to the vector store (OpenSearch), and each SHALL carry a stable pointer back to its graph node. +- Each `SemanticCodeUnit` SHALL carry provenance metadata (repo, commit, file, method, cpg node id). + +### R5 — Delta idempotency contract with enrichment recipe version (traces to D4) +As an operator, I want re-ingest to trigger when either code or enrichment changes, so that overlay never goes stale. + +Acceptance criteria: +- The manifest signature SHALL be a function of BOTH the method-signature set AND the enrichment recipe version. +- WHEN code is unchanged but the enrichment recipe changes, THEN the delta decision SHALL be INGEST (not SKIP). +- WHEN both code and recipe are unchanged, THEN the decision SHALL be SKIP with zero writes to Neptune and the vector store. +- A migration path SHALL exist for manifests created before the signature change. + +### R6 — Canonical identity spine (traces to D5, Phase 2) +As an integrator, I want a canonical identity for applications/services/repos/APIs/controls/owners, so that graphs join on stable identity rather than fuzzy names. + +Acceptance criteria: +- A canonical identity model SHALL exist as its own bounded context with a resolver mapping aliases to canonical ids. +- Resolution SHALL be deterministic and auditable (input alias → canonical id → source of authority). +- Lexical and code entities SHALL be able to resolve to a canonical id. + +### R7 — Cross-graph linker (traces to D5, Phase 2) +As a retrieval designer, I want explicit canonical-identity edges written at ingest, so that cross-graph links are proof, not guesses. + +Acceptance criteria: +- Cross-graph edges SHALL be written from canonical identities at ingest time, replacing runtime fuzzy correlation. +- A link SHALL record its basis (resolved identity vs candidate) so consumers can distinguish proof from candidate. + +### R8 — Multi-graph query planner + provenance (traces to D5, Phase 2) +As a user, I want questions answered across lexical, domain, CPG, and evidence graphs with provenance, so that answers are grounded and traceable. + +Acceptance criteria: +- The planner SHALL classify intent and route to the appropriate index/graph, using vectors as entry points and graph traversal as proof. +- Every answer SHALL return provenance: source artifact, graph path, and explicit gaps where evidence is missing. +- The planner SHALL compose existing retrievers rather than reimplementing store access. + +### R9 — Packaging and dependency direction (traces to D1, D6) +As a maintainer, I want client CPG-RAG isolated from the toolkit core, so that the toolkit stays general-purpose. + +Acceptance criteria: +- CPG-RAG composition SHALL live in a separate package depending on the toolkit; dependencies SHALL be one-directional (domain → foundation). +- Enhancements that are genuinely generic SHALL be contributed upstream additively (OCP), preserving existing behaviour and defaults. +- No client-specific logic SHALL be introduced into `lexical-graph`. + +--- + +## 5. Non-functional requirements + +- **SOLID conformance:** components single-responsibility, depend on abstractions, extend via registration not modification. Reviewed against the toolkit's existing provider/factory/registry patterns. +- **No vendor coupling:** no vendor SDK imports outside a provider adapter. +- **Multi-tenancy:** all writes SHALL respect the toolkit's tenant label encoding; delta replace SHALL purge the superseded tenant. +- **Embeddings as entry points:** vectors locate; the graph proves. No requirement shall depend on vector similarity as proof. +- **Provenance & auditability:** overlay, links, and answers SHALL be traceable to source. +- **Security:** retrieval SHALL be able to honour source permissions (Confluence, repo) where required; secrets via a secret store, never in code. +- **Observability & cost:** SKIP on unchanged input SHALL incur zero model, Neptune, and vector-store cost; ingest SHALL emit a delta summary. + +--- + +## 6. Definition of done (per phase) + +- **Phase 1:** R1–R5 and R9 met with automated tests; a documented Joern-JSON → enriched-Neptune run using an injected local Mistral 7B provider; no vendor SDK imports in transformers; delta SKIP/INGEST correct across code and recipe changes. +- **Phase 2:** R6–R8 met with tests; a single question traverses lexical + CPG + identity and returns a grounded answer with provenance and gaps. + +--- + +## 7. Open questions for the client + +1. Joern scope: languages, and which layers are exported (AST/CFG/CDG/REACHING_DEF/CALL)? Are file/line and commit SHA always present? +2. Local Mistral 7B serving: which runtime (Ollama / vLLM / TGI) and endpoint contract, for the injected provider? +3. Which semantic units are required first: method, endpoint, source-to-sink, or authorization-pattern summaries? +4. Identity authority (Phase 2): what is the canonical application/repo/API source, and what alias sources exist? +5. Retrieval permissions: must Confluence/repo permissions be enforced at retrieval time? diff --git a/working/task-list.md b/working/task-list.md new file mode 100644 index 000000000..8c3acd9f4 --- /dev/null +++ b/working/task-list.md @@ -0,0 +1,205 @@ +# Task List — CPG-RAG Migration (Neo4j → Amazon Neptune) +## graphrag-toolkit / Cross-Cloud Architecture Implementation + +- **Date:** 2026-07-07 +- **Reference:** architecture.md, ADR-0010 + +--- + +## Ownership Legend + +| Owner | Scope | +|-------|-------| +| **Client** | All Azure EKS processes, AWS EKS provisioning/configuration/deployment, architecture approval | +| **AWS** | AWS infrastructure (Neptune, OpenSearch, S3, IAM, VPC) | +| **Deloitte** | graphrag-toolkit customization, containers/Helm charts, integration advisory | + +--- + +## Phase 1: Architecture & Approval + +| # | Task | Owner | Depends On | Status | +|---|------|-------|------------|--------| +| 1.1 | Review architecture document (architecture.md) | Client | — | ☐ | +| 1.2 | Review ADR-0010 (cross-cloud seam — S3 artifact) | Client | — | ☐ | +| 1.3 | Decision: Confirm embedding model and vector dimension | Client | — | ☐ | +| 1.4 | Decision: Enrichment strategy — per-node or traversal-based? | Client | — | ☐ | +| 1.5 | Decision: Tenant granularity (per-repo, per-repo+version, per-project) | Client | — | ☐ | +| 1.6 | Decision: Expected volume (repos/projects) for throughput sizing | Client | — | ☐ | +| 1.7 | Decision: Build trigger preference (S3 event-driven or scheduled/manual) | Client | — | ☐ | +| 1.8 | Decision: AWS EKS cluster config (instance types, scaling, namespaces) | Client | — | ☐ | +| 1.9 | Decision: Timeline for migrating extraction from Azure EKS to AWS EKS | Client | — | ☐ | +| 1.10 | Sign-off on architecture | Client | 1.1–1.9 | ☐ | + +--- + +## Phase 2: Azure EKS — Extraction & Enrichment Pipeline + +| # | Task | Owner | Depends On | Status | +|---|------|-------|------------|--------| +| 2.1 | Confirm Joern extraction pipeline outputs (nodes.json, edges.json) | Client | 1.10 | ☐ | +| 2.2 | Implement per-node enrichment (generation model → summary text) | Client | 2.1 | ☐ | +| 2.3 | Define and validate the S3 artifact schema (section 6 contract) | Client + Deloitte | 2.1 | ☐ | +| 2.4 | Implement manifest generation (repo, commit, method_signatures, recipe version) | Client | 2.3 | ☐ | +| 2.5 | Implement artifact bundle writer (nodes + edges + semantic_units + manifest) | Client | 2.3, 2.4 | ☐ | +| 2.6 | Configure Azure workload-identity federation (OIDC → AWS IAM role) | Client | 3.2 | ☐ | +| 2.7 | Implement S3 upload from Azure EKS (IAM/SigV4 authenticated) | Client | 2.5, 2.6 | ☐ | +| 2.8 | Test artifact upload — validate schema, checksums, manifest integrity | Client | 2.7 | ☐ | +| 2.9 | Document Azure EKS pipeline runbook (trigger, retry, monitoring) | Client | 2.8 | ☐ | + +--- + +## Phase 3: AWS Infrastructure Provisioning + +| # | Task | Owner | Depends On | Status | +|---|------|-------|------------|--------| +| 3.1 | Provision VPC (private subnets, NAT gateway, route tables) | AWS | 1.10 | ☐ | +| 3.2 | Create IAM role for Azure workload-identity federation (OIDC trust, S3 PutObject only) | AWS | 1.10 | ☐ | +| 3.3 | Create S3 artifact bucket with versioning, lifecycle, KMS encryption | AWS | 3.1 | ☐ | +| 3.4 | Create S3 gateway endpoint (VPC, for EKS pod access — no NAT needed) | AWS | 3.1 | ☐ | +| 3.5 | Provision Neptune DB cluster (engine >= 1.4.6.x, serverless, private) | AWS | 3.1 | ☐ | +| 3.6 | Enable Neptune IAM DB authentication | AWS | 3.5 | ☐ | +| 3.7 | Configure Neptune security group (allow only EKS pod subnets) | AWS | 3.5, 4.1 | ☐ | +| 3.8 | Create Neptune cluster parameter group | AWS | 3.5 | ☐ | +| 3.9 | Create Neptune DB subnet group (private subnets) | AWS | 3.1, 3.5 | ☐ | +| 3.10 | Provision OpenSearch Serverless collection (VECTORSEARCH type, VPC endpoint) | AWS | 3.1 | ☐ | +| 3.11 | Configure OpenSearch VPC endpoint (private access from EKS) | AWS | 3.10 | ☐ | +| 3.12 | Configure OpenSearch encryption policy | AWS | 3.10 | ☐ | +| 3.13 | Configure OpenSearch data-access policy (EKS pod IAM roles via IRSA) | AWS | 3.10, 4.3 | ☐ | +| 3.14 | Create IAM roles for EKS pods — IRSA (build role, query role) | AWS | 3.5, 3.10 | ☐ | +| 3.15 | Enable CloudTrail logging for Neptune and OpenSearch access | AWS | 3.5, 3.10 | ☐ | +| 3.16 | Enable Neptune audit logs | AWS | 3.5 | ☐ | +| 3.17 | Enable S3 access logging on artifact bucket | AWS | 3.3 | ☐ | +| 3.18 | Validate Neptune private endpoint connectivity from VPC test instance | AWS | 3.5, 3.7 | ☐ | +| 3.19 | Validate OpenSearch VPC endpoint connectivity | AWS | 3.11, 3.13 | ☐ | +| 3.20 | Document infrastructure — endpoints, ARNs, IRSA role ARNs, VPC config | AWS | 3.18, 3.19 | ☐ | + +--- + +## Phase 4: AWS EKS — Client Cluster Setup + +| # | Task | Owner | Depends On | Status | +|---|------|-------|------------|--------| +| 4.1 | Provision AWS EKS cluster (private subnets, managed node groups) | Client | 3.1 | ☐ | +| 4.2 | Configure EKS node instance types and scaling policy (per decision 1.8) | Client | 4.1 | ☐ | +| 4.3 | Configure IRSA (IAM Roles for Service Accounts) on EKS cluster | Client | 4.1, 3.14 | ☐ | +| 4.4 | Create Kubernetes namespaces for toolkit workloads | Client | 4.1 | ☐ | +| 4.5 | Validate EKS pod → Neptune private endpoint connectivity | Client | 4.1, 3.18 | ☐ | +| 4.6 | Validate EKS pod → OpenSearch VPC endpoint connectivity | Client | 4.1, 3.19 | ☐ | +| 4.7 | Validate EKS pod → S3 (via gateway endpoint) connectivity | Client | 4.1, 3.4 | ☐ | +| 4.8 | Deploy embedding model to AWS EKS (e.g. Ollama + nomic-embed) | Client | 4.1, 1.3 | ☐ | +| 4.9 | Validate embedding model service endpoint from within EKS | Client | 4.8 | ☐ | +| 4.10 | Document EKS cluster config — endpoints, namespaces, IRSA mappings | Client | 4.1–4.9 | ☐ | + +--- + +## Phase 5: graphrag-toolkit Customization (Deloitte delivers) + +| # | Task | Owner | Depends On | Status | +|---|------|-------|------------|--------| +| 5.1 | Implement S3 artifact reader (ingest nodes/edges/semantic_units from contract) | Deloitte | 2.3 | ☐ | +| 5.2 | Implement CPG-to-graph compiler (Joern nodes/edges → Neptune graph model) | Deloitte | 5.1 | ☐ | +| 5.3 | Implement SemanticCodeUnit overlay (summary text → graph overlay nodes) | Deloitte | 5.2 | ☐ | +| 5.4 | Configure embedding model provider (client's model via toolkit seam) | Deloitte | 1.3, 4.8 | ☐ | +| 5.5 | Validate embed_dimensions matches client model output | Deloitte | 5.4 | ☐ | +| 5.6 | Implement tenant derivation (repo identity → TenantId, <= 25 chars) | Deloitte | 1.5 | ☐ | +| 5.7 | Implement delta/skip-or-replace logic (manifest signature comparison) | Deloitte | 5.1, 2.4 | ☐ | +| 5.8 | Implement tenant purge (remove prior tenant on re-ingest) | Deloitte | 5.6 | ☐ | +| 5.9 | Configure Neptune Database writer (neptunedata API, SigV4, private endpoint) | Deloitte | 3.20 | ☐ | +| 5.10 | Configure OpenSearch Serverless writer (vector store, SigV4, VPC endpoint) | Deloitte | 3.20 | ☐ | +| 5.11 | Implement provenance preservation (repo, commit, file, method, timestamp) | Deloitte | 5.2 | ☐ | +| 5.12 | Implement query/retrieval pipeline (graph traversal + vector search) | Deloitte | 5.9, 5.10 | ☐ | +| 5.13 | Implement traversal-based enrichment pass (if decision 1.4 = traversal) | Deloitte | 5.9, 1.4 | ☐ | +| 5.14 | Build agent integration (expose retrieval for downstream agents) | Deloitte | 5.12 | ☐ | +| 5.15 | Pin toolkit version; implement adapter layer for bespoke isolation | Deloitte | 5.1–5.14 | ☐ | +| 5.16 | Write contract tests (artifact schema validation, build output validation) | Deloitte | 5.1, 5.2 | ☐ | +| 5.17 | Package as containers + Helm charts for EKS deployment | Deloitte | 5.1–5.16 | ☐ | +| 5.18 | Document customization — configuration, model seam, tenant rules, Helm values | Deloitte | 5.17 | ☐ | + +--- + +## Phase 6: AWS EKS Deployment (Client deploys Deloitte deliverables) + +| # | Task | Owner | Depends On | Status | +|---|------|-------|------------|--------| +| 6.1 | Deploy build pipeline containers/Helm chart to AWS EKS | Client | 5.17, 4.10 | ☐ | +| 6.2 | Configure build pipeline environment (endpoints, IRSA, region, Helm values) | Client | 6.1, 3.20 | ☐ | +| 6.3 | Deploy query/retrieval service containers/Helm chart to AWS EKS | Client | 5.17, 4.10 | ☐ | +| 6.4 | Configure query service environment (endpoints, IRSA, Helm values) | Client | 6.3, 3.20 | ☐ | +| 6.5 | Deploy agent service (if separate from retrieval) | Client | 5.17, 6.3 | ☐ | +| 6.6 | Configure S3 event trigger for build (new artifact → start build job) | Client | 6.1, 3.3 | ☐ | +| 6.7 | Configure monitoring & alerting (CloudWatch, build failures, latency) | Client | 6.1, 6.3 | ☐ | +| 6.8 | Configure auto-scaling (HPA for query service pods) | Client | 6.3 | ☐ | +| 6.9 | Document deployment runbook (deploy, rollback, Helm upgrades) | Client | 6.1–6.8 | ☐ | + +--- + +## Phase 7: Integration Testing + +| # | Task | Owner | Depends On | Status | +|---|------|-------|------------|--------| +| 7.1 | End-to-end test: Azure EKS extract → S3 upload → AWS EKS build → Neptune/OpenSearch | Client + Deloitte | 6.2, 2.8 | ☐ | +| 7.2 | Validate graph correctness (nodes, edges, labels, properties) | Deloitte | 7.1 | ☐ | +| 7.3 | Validate vector index (embedding dimensions, similarity search) | Deloitte | 7.1 | ☐ | +| 7.4 | Validate delta/idempotency (re-run same repo = skip; changed repo = replace) | Deloitte | 7.1 | ☐ | +| 7.5 | Validate multi-tenant isolation (two repos in same cluster, no bleed) | Deloitte | 7.1 | ☐ | +| 7.6 | Validate query/retrieval accuracy (known queries, expected results) | Deloitte + Client | 7.2, 7.3 | ☐ | +| 7.7 | Performance test: build at expected volume (decision 1.6 sizing) | Client + Deloitte | 7.1, 1.6 | ☐ | +| 7.8 | Security validation: no public endpoints, IRSA working, SG correct | Client + AWS | 7.1 | ☐ | +| 7.9 | Validate private connectivity: EKS pods → Neptune, OpenSearch, S3 (no internet) | Client | 7.1 | ☐ | +| 7.10 | Sign-off on integration test results | Client | 7.1–7.9 | ☐ | + +--- + +## Phase 8: Production Cutover & Migration + +| # | Task | Owner | Depends On | Status | +|---|------|-------|------------|--------| +| 8.1 | Migrate production workloads (Azure EKS pipelines point to S3 bucket) | Client | 7.10 | ☐ | +| 8.2 | Decommission Neo4j (after validation period) | Client | 8.1 | ☐ | +| 8.3 | Monitor production build/query (first 2 weeks) | Client + Deloitte | 8.1 | ☐ | +| 8.4 | Migrate extraction/enrichment from Azure EKS to AWS EKS (per decision 1.9) | Client | 8.3 | ☐ | +| 8.5 | Revoke Azure workload-identity federation trust (after Azure EKS decommission) | AWS + Client | 8.4 | ☐ | +| 8.6 | Final security audit (private only, no stale credentials, IRSA verified) | Client + AWS | 8.4, 8.5 | ☐ | +| 8.7 | Document end-state architecture (all workloads in AWS EKS) | Deloitte | 8.6 | ☐ | +| 8.8 | Close-out: confirm migration complete, archive interim config | Client | 8.6, 8.7 | ☐ | + +--- + +## Summary — Ownership Matrix + +| Phase | Client | AWS | Deloitte | +|-------|--------|-----|----------| +| 1. Architecture & Approval | Decisions, sign-off | — | Advisory | +| 2. Azure EKS Pipeline | Build & operate | — | Schema co-design | +| 3. AWS Infrastructure | — | Provision & configure | — | +| 4. AWS EKS Cluster | Provision, configure, operate | — | — | +| 5. Toolkit Customization | — | — | Build & deliver | +| 6. AWS EKS Deployment | Deploy & operate | — | Support | +| 7. Integration Testing | Execute & validate | Infra validation | Build validation | +| 8. Production & Migration | Operate, migrate, cutover | Revoke access, audit | Document | + +--- + +## Dependencies & Critical Path + +``` +1.10 (Architecture sign-off) + ├── 2.x (Azure EKS pipeline) ───── 2.8 (artifact upload tested) + ├── 3.x (AWS infra) ────────────── 3.20 (infra documented) + └── 4.x (AWS EKS cluster) ──────── 4.10 (EKS ready) + │ + 5.x (toolkit customization) ────── 5.17 (containers/Helm delivered) + │ + 6.x (AWS EKS deployment) ───────── 6.2 (build deployed) + │ + 7.x (Integration testing) ──────── 7.10 (sign-off) + │ + 8.x (Production cutover & migration) +``` + +- Phases 2, 3, 4 can run in **parallel** after architecture sign-off (1.10). +- Phase 5 (Deloitte) can start in parallel once artifact schema is agreed (2.3). +- Phase 6 requires outputs from phases 3, 4, and 5. +- Phase 7 requires all prior phases complete. +- Phase 8 is sequential after successful integration testing.