Remove LlamaIndex as mandatory runtime dependency

examples/lexical-graph-hybrid-dev/tests/run_notebooks.py

@@ -24,8 +24,28 @@
 ]
 
 # (notebook_index, cell_index) -> reason
-CUDA_SKIPS = {(4, 22): "GPU/CUDA BGEReranker"}
+CUDA_SKIPS = {
+    (4, 20): "GPU/CUDA reranker (RerankingBeamGraphSearch)",
+    (4, 22): "GPU/CUDA BGEReranker",
+    (4, 25): "GPU/CUDA reranker (SpaCy + reranker)",
+}
 BATCH_SKIPS = {(1, 9): "Requires SOURCE_DIR with PDF files"}
+INFRA_SKIPS = {
+    # 03-Cloud-Build: requires S3 bucket with extracted data from setup-bedrock-batch.sh
+    (3, 4): "Requires S3 extract-build data (setup-bedrock-batch.sh)",
+    # 04-Cloud-Querying: requires populated graph from successful build
+    (4, 4): "Requires populated graph (build not run)",
+    (4, 5): "Requires populated graph (depends on cell 4)",
+    (4, 6): "Requires populated graph (depends on cell 4)",
+    (4, 8): "Requires populated graph (build not run)",
+    (4, 11): "Requires S3 prompt files (setup-bedrock-batch.sh)",
+    (4, 16): "Requires populated graph (build not run)",
+    (4, 18): "Requires populated graph (build not run)",
+}
+ENV_SKIPS = {
+    # 04-Cloud-Querying: requires SYSTEM_PROMPT_ARN from create_custom_prompt.sh
+    (4, 14): "Requires SYSTEM_PROMPT_ARN env var (create_custom_prompt.sh)",
+}
 
 
 def extract_output(cell):
@@ -132,6 +152,8 @@ def main():
     parser.add_argument("--output-dir", default="/home/jovyan/notebooks")
     parser.add_argument("--skip-cuda", default="true", choices=["true", "false"])
     parser.add_argument("--skip-batch", default="true", choices=["true", "false"])
+    parser.add_argument("--skip-infra", default="true", choices=["true", "false"])
+    parser.add_argument("--skip-env", default="true", choices=["true", "false"])
     parser.add_argument("--notebooks", nargs="*", help="Specific notebooks to run")
     args = parser.parse_args()
 
@@ -143,6 +165,10 @@ def main():
         skip_cells.update(CUDA_SKIPS)
     if args.skip_batch == "true":
         skip_cells.update(BATCH_SKIPS)
+    if args.skip_infra == "true":
+        skip_cells.update(INFRA_SKIPS)
+    if args.skip_env == "true":
+        skip_cells.update(ENV_SKIPS)
 
     report = []
     for nb_idx, nb_name in enumerate(ALL_NOTEBOOKS):

lexical-graph/README.md

@@ -37,14 +37,38 @@ $ pip install https://github.com/awslabs/graphrag-toolkit/archive/refs/tags/grap
 
 If you're running on AWS, you must run your application in an AWS region containing the Amazon Bedrock foundation models used by the lexical graph (see the [configuration](https://awslabs.github.io/graphrag-toolkit/lexical-graph/configuration/#graphragconfig) section in the documentation for details on the default models used), and must [enable access](https://docs.aws.amazon.com/bedrock/latest/userguide/model-access.html) to these models before running any part of the solution.
 
+### Optional extras
+
+The core library installs without any llama-index dependency. Install optional extras for additional functionality:
+
+```bash
+# Core install (no llama-index required)
+$ pip install graphrag-lexical-graph
+
+# With LlamaIndex reader support
+$ pip install graphrag-lexical-graph[readers]
+
+# With LlamaIndex chunking support (SentenceSplitter)
+$ pip install graphrag-lexical-graph[chunking]
+
+# With LlamaIndex LLM/embedding providers
+$ pip install graphrag-lexical-graph[llms]
+
+# With OpenSearch vector store support
+$ pip install graphrag-lexical-graph[opensearch]
+
+# Full compatibility (all llama-index integrations)
+$ pip install graphrag-lexical-graph[all]
+```
+
 ### Additional dependencies
 
 You will need to install additional dependencies for specific graph and vector store backends:
 
 #### Amazon OpenSearch Serverless
 
 ```bash
-$ pip install opensearch-py llama-index-vector-stores-opensearch
+$ pip install graphrag-lexical-graph[opensearch]
 ```
 
 #### Postgres with pgvector

lexical-graph/pyproject.toml

@@ -18,6 +18,25 @@ license = "Apache-2.0"
 files = ["src/graphrag_toolkit/lexical_graph/requirements.txt"]
 
 [project.optional-dependencies]
+readers = [
+    "llama-index-core>=0.14.0",
+]
+chunking = [
+    "llama-index-core>=0.14.0",
+]
+opensearch = [
+    "llama-index-core>=0.14.0",
+    "opensearch-py>=2.0.0",
+]
+llms = [
+    "llama-index-core>=0.14.0",
+    "llama-index-llms-anthropic>=0.11.0",
+]
+all = [
+    "llama-index-core>=0.14.0",
+    "llama-index-llms-anthropic>=0.11.0",
+    "opensearch-py>=2.0.0",
+]
 test = [
     "pytest>=7.0.0",
     "pytest-cov>=4.0.0",

lexical-graph/src/graphrag_toolkit/core/__init__.py

@@ -0,0 +1,95 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+from graphrag_toolkit.core.callbacks import CallbackRegistry
+from graphrag_toolkit.core.compat import (
+    BaseComponent,
+    BaseNode,
+    NodeRelationship,
+    RelatedNodeInfo,
+    TextNode,
+)
+from graphrag_toolkit.core.embedding import (
+    BedrockEmbeddingProvider,
+    EmbeddingProvider,
+)
+from graphrag_toolkit.core.extractor import Extractor
+from graphrag_toolkit.core.llm import (
+    BedrockLLMProvider,
+    LLMProvider,
+    LLMResponse,
+)
+from graphrag_toolkit.core.pipeline import Pipeline, async_run_pipeline, run_pipeline
+from graphrag_toolkit.core.postprocessor import PostProcessor
+from graphrag_toolkit.core.prompt import (
+    ChatPromptTemplate,
+    PromptTemplate,
+)
+from graphrag_toolkit.core.query_engine import QueryEngine
+from graphrag_toolkit.core.reader import Reader
+from graphrag_toolkit.core.response import (
+    RESPONSE_TYPE,
+    Response,
+    StreamingResponse,
+    TokenGen,
+)
+from graphrag_toolkit.core.retriever import Retriever
+from graphrag_toolkit.core.transform import Transform
+from graphrag_toolkit.core.types import (
+    Document,
+    Node,
+    NodeRef,
+    NodeWithScore,
+    QueryBundle,
+)
+from graphrag_toolkit.core.utils import iter_batch, run_jobs
+from graphrag_toolkit.core.vector_store_types import (
+    FilterCondition,
+    FilterOperator,
+    MetadataFilter,
+    MetadataFilters,
+    VectorStoreQueryMode,
+    VectorStoreQueryResult,
+)
+
+__all__ = [
+    "BaseComponent",
+    "BaseNode",
+    "BedrockEmbeddingProvider",
+    "BedrockLLMProvider",
+    "CallbackRegistry",
+    "ChatPromptTemplate",
+    "Document",
+    "EmbeddingProvider",
+    "Extractor",
+    "FilterCondition",
+    "FilterOperator",
+    "LLMProvider",
+    "LLMResponse",
+    "MetadataFilter",
+    "MetadataFilters",
+    "Node",
+    "NodeRef",
+    "NodeRelationship",
+    "NodeWithScore",
+    "Pipeline",
+    "PostProcessor",
+    "PromptTemplate",
+    "QueryBundle",
+    "QueryEngine",
+    "RESPONSE_TYPE",
+    "Reader",
+    "RelatedNodeInfo",
+    "Response",
+    "Retriever",
+    "StreamingResponse",
+    "TextNode",
+    "TokenGen",
+    "Transform",
+    "VectorStoreQueryMode",
+    "VectorStoreQueryResult",
+    "async_run_pipeline",
+    "iter_batch",
+    "run_jobs",
+    "run_pipeline",
+]

lexical-graph/src/graphrag_toolkit/core/async_utils.py

@@ -0,0 +1,22 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Async utilities for safe coroutine execution."""
+
+import asyncio
+
+
+def run_async(coro):
+    """Run a coroutine safely, whether or not an event loop is already running.
+
+    Handles Jupyter notebooks and async frameworks where asyncio.run() would fail.
+    """
+    try:
+        asyncio.get_running_loop()
+        # Loop already running — execute in a new thread
+        import concurrent.futures
+        with concurrent.futures.ThreadPoolExecutor(1) as pool:
+            return pool.submit(asyncio.run, coro).result()
+    except RuntimeError:
+        # No running loop — safe to use asyncio.run()
+        return asyncio.run(coro)

lexical-graph/src/graphrag_toolkit/core/callbacks.py

@@ -0,0 +1,48 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Callback registry for observability events."""
+
+from __future__ import annotations
+
+from typing import Callable
+
+# Event type constants
+LLM_START = "llm_start"
+LLM_END = "llm_end"
+EMBEDDING_START = "embedding_start"
+EMBEDDING_END = "embedding_end"
+RETRIEVAL_START = "retrieval_start"
+RETRIEVAL_END = "retrieval_end"
+TRANSFORM_START = "transform_start"
+TRANSFORM_END = "transform_end"
+
+
+class CallbackRegistry:
+    """Class-level registry for event handlers."""
+
+    _handlers: list[Callable[[str, dict], None]] = []
+
+    @classmethod
+    def register(cls, handler: Callable[[str, dict], None]) -> None:
+        """Add a handler."""
+        cls._handlers.append(handler)
+
+    @classmethod
+    def emit(cls, event_type: str, payload: dict) -> None:
+        """Call all registered handlers."""
+        for handler in list(cls._handlers):  # snapshot for thread safety
+            try:
+                handler(event_type, payload)
+            except Exception:
+                pass  # Don't let observer errors kill the pipeline
+
+    @classmethod
+    def clear(cls) -> None:
+        """Remove all handlers."""
+        cls._handlers.clear()
+
+    @classmethod
+    def get_handlers(cls) -> list[Callable[[str, dict], None]]:
+        """Return registered handlers."""
+        return cls._handlers

lexical-graph/src/graphrag_toolkit/core/compat.py

@@ -0,0 +1,88 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Type aliases and relationship key utilities."""
+
+from __future__ import annotations
+
+from graphrag_toolkit.core.types import Node, NodeRef
+
+# Type aliases
+TextNode = Node
+BaseNode = Node
+RelatedNodeInfo = NodeRef
+
+# Module-level lazy cache for LlamaIndex enum map
+_LI_ENUM_MAP = None
+
+
+def _get_li_enum_map():
+    global _LI_ENUM_MAP
+    if _LI_ENUM_MAP is None:
+        try:
+            from llama_index.core.schema import NodeRelationship as LI_NR
+            _LI_ENUM_MAP = {"source": LI_NR.SOURCE, "previous": LI_NR.PREVIOUS,
+                           "next": LI_NR.NEXT, "parent": LI_NR.PARENT, "child": LI_NR.CHILD}
+        except ImportError:
+            _LI_ENUM_MAP = {}
+    return _LI_ENUM_MAP
+
+
+class NodeRelationship:
+    """String constants for node relationship types."""
+
+    SOURCE = "source"
+    PREVIOUS = "previous"
+    NEXT = "next"
+    PARENT = "parent"
+    CHILD = "child"
+
+    # Enum value mapping for nodes created by optional chunking dependencies
+    _LLAMA_INDEX_KEYS = {
+        "source": "1",
+        "previous": "2",
+        "next": "3",
+        "parent": "4",
+        "child": "5",
+    }
+
+    @classmethod
+    def get_relationship(cls, relationships: dict, rel_type: str, default=None):
+        """Get a relationship from a node's relationships dict, handling
+        string keys, enum value strings, and enum object keys."""
+        # Try our string key first (e.g. "source")
+        result = relationships.get(rel_type)
+        if result is not None:
+            return result
+        # Try enum value string (e.g. "1")
+        enum_val = cls._LLAMA_INDEX_KEYS.get(rel_type)
+        if enum_val:
+            result = relationships.get(enum_val)
+            if result is not None:
+                return result
+        # Try enum object as key (SentenceSplitter uses actual enum instances)
+        enum_map = _get_li_enum_map()
+        if enum_map:
+            enum_key = enum_map.get(rel_type)
+            if enum_key:
+                result = relationships.get(enum_key)
+                if result is not None:
+                    return result
+        return default
+
+
+from pydantic import BaseModel
+
+from graphrag_toolkit.core.transform import Transform
+
+
+class BaseComponent(BaseModel):
+    """Base class for pipeline components."""
+
+
+class TransformComponent(BaseComponent, Transform):
+    """Base class for callable transform pipeline components."""
+
+    def __call__(self, nodes: list, **kwargs) -> list:
+        """Default no-op implementation."""
+        return nodes