From 1941a9c36c9adab24bb5c5546cbe3454a60e7235 Mon Sep 17 00:00:00 2001
From: owenguoo <88258850+owenguoo@users.noreply.github.com>
Date: Sat, 13 Sep 2025 20:47:25 -0400
Subject: [PATCH 1/8] python server and endpoint

---
 backend/agents/README.md        | 106 ++++++++
 backend/agents/api/index.py     | 419 ++++++++++++++++++++++++++++++++
 backend/agents/database.py      |  37 +++
 backend/agents/main.py          | 391 +++++++++++++++++++++++++++++
 backend/agents/requirements.txt |   7 +-
 backend/agents/runner.py        |  56 ++++-
 backend/agents/vercel.json      |  18 ++
 7 files changed, 1027 insertions(+), 7 deletions(-)
 create mode 100644 backend/agents/README.md
 create mode 100644 backend/agents/api/index.py
 create mode 100644 backend/agents/main.py
 create mode 100644 backend/agents/vercel.json

diff --git a/backend/agents/README.md b/backend/agents/README.md
new file mode 100644
index 0000000..cfe8097
--- /dev/null
+++ b/backend/agents/README.md
@@ -0,0 +1,106 @@
+# QAI Agent Runner API
+
+FastAPI server for hosting QAI autonomous testing agents, deployable on Vercel.
+
+## Features
+
+- Full REST API matching the QAI API specification
+- Database integration with Supabase
+- Autonomous agent execution capabilities
+- CICD pipeline integration
+- Vercel-ready deployment configuration
+
+## API Endpoints
+
+### Database Management
+- `POST /results` - Create new test result
+- `PATCH /results/{id}` - Update result status
+- `GET /results` - Get all results
+- `POST /suites` - Create new test suite
+- `PATCH /suites/{id}` - Update suite status
+- `GET /results/{id}/suites` - Get suites for result
+- `POST /tests` - Create new test
+- `PATCH /tests/{id}` - Update test status
+- `GET /suites/{id}/tests` - Get tests for suite
+
+### Agent Execution
+- `POST /run-suite` - Run test suite by ID (main CICD endpoint)
+- `POST /run-agent` - Run single agent (legacy)
+- `POST /run-agents` - Run multiple agents (legacy)
+
+### Utility
+- `GET /health` - Health check and database status
+- `GET /` - API documentation
+
+## Environment Variables
+
+Required environment variables:
+
+```bash
+# Database
+SUPABASE_URL=your_supabase_url
+SUPABASE_KEY=your_supabase_key
+
+# Agent Configuration
+CUA_MODEL=anthropic/claude-3-5-sonnet-20241022
+CUA_API_KEY=your_cua_api_key
+CUA_CONTAINER_NAME=your_container_name
+
+# Optional
+PORT=8000
+```
+
+## Local Development
+
+1. Install dependencies:
+```bash
+pip install -r requirements.txt
+```
+
+2. Set up environment variables in `.env` file
+
+3. Run the server:
+```bash
+python main.py
+# or
+uvicorn main:app --reload --host 0.0.0.0 --port 8000
+```
+
+## Vercel Deployment
+
+1. Install Vercel CLI:
+```bash
+npm i -g vercel
+```
+
+2. Deploy:
+```bash
+vercel --prod
+```
+
+3. Set environment variables in Vercel dashboard
+
+The API will be available at your Vercel deployment URL.
+
+## CICD Integration
+
+The server integrates with the QAI pipeline via the `/run-suite` endpoint:
+
+1. Pipeline creates database records (results, suites, tests)
+2. Pipeline calls `POST /run-suite` with suite ID
+3. Agent runs tests and updates database with results
+4. Pipeline verifies completion via database queries
+
+## Database Schema
+
+See `API.md` for complete database schema documentation.
+
+## Files Structure
+
+- `main.py` - Local FastAPI server
+- `api/index.py` - Vercel deployment handler
+- `runner.py` - Agent execution logic
+- `database.py` - Database operations
+- `run_suite.py` - Command-line suite runner
+- `vercel.json` - Vercel deployment configuration
+- `requirements.txt` - Python dependencies
\ No newline at end of file
diff --git a/backend/agents/api/index.py b/backend/agents/api/index.py
new file mode 100644
index 0000000..9ccf053
--- /dev/null
+++ b/backend/agents/api/index.py
@@ -0,0 +1,419 @@
+from fastapi import FastAPI, HTTPException
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel
+from typing import List, Dict, Any, Optional
+import asyncio
+import os
+import sys
+from pathlib import Path
+
+# Add the parent directory to Python path for imports
+current_dir = Path(__file__).parent
+parent_dir = current_dir.parent
+sys.path.append(str(parent_dir))
+
+from dotenv import load_dotenv
+load_dotenv()
+
+try:
+    from runner import run_single_agent, run_agents, run_qai_tests
+    from database import (
+        get_or_create_test,
+        create_result,
+        set_suite_result_id,
+        get_suite_with_tests,
+        update_test_fields,
+        append_test_step,
+        _has_client
+    )
+except ImportError as e:
+    # Fallback for when running on Vercel with different import paths
+    print(f"Import error: {e}")
+    # Define stub functions to prevent startup errors
+    async def run_qai_tests(suite_id): 
+        return {"agent_result": {"status": "failed", "error": "Agent runner not available"}}
+    async def run_single_agent(spec): 
+        return []
+    async def run_agents(specs, pr_name, pr_link): 
+        return {}
+    def _has_client(): 
+        return False
+
+app = FastAPI(
+    title="QAI Agent Runner API", 
+    version="1.0.0",
+    description="FastAPI server for QAI autonomous testing agents"
+)
+
+# Configure CORS
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Pydantic models matching API.md specification
+
+class CreateResultRequest(BaseModel):
+    prLink: str
+    prName: str
+
+class UpdateResultRequest(BaseModel):
+    resSuccess: bool
+
+class CreateSuiteRequest(BaseModel):
+    resultId: int
+    name: str
+    s3Link: Optional[str] = None
+    suitesSuccess: Optional[bool] = False
+
+class UpdateSuiteRequest(BaseModel):
+    suitesSuccess: Optional[bool] = None
+    s3Link: Optional[str] = None
+
+class CreateTestRequest(BaseModel):
+    suiteId: int
+    name: str
+    summary: Optional[str] = None
+    testSuccess: Optional[bool] = False
+
+class UpdateTestRequest(BaseModel):
+    testSuccess: Optional[bool] = None
+    summary: Optional[str] = None
+
+class RunSuiteRequest(BaseModel):
+    suite_id: int
+
+class AgentRunRequest(BaseModel):
+    spec: Dict[str, Any]
+
+class MultiAgentRunRequest(BaseModel):
+    test_specs: List[Dict[str, Any]]
+    pr_name: str
+    pr_link: str
+
+@app.get("/health")
+async def health_check():
+    """Health check endpoint"""
+    return {
+        "status": "healthy",
+        "database_connected": _has_client(),
+        "version": "1.0.0",
+        "environment": "vercel" if os.getenv("VERCEL") else "local"
+    }
+
+# Results endpoints (matching API.md)
+
+@app.post("/results")
+async def create_result_endpoint(request: CreateResultRequest):
+    """Create a new result (PR test run)"""
+    try:
+        if not _has_client():
+            raise HTTPException(status_code=500, detail="Database not configured")
+        
+        result_id = await create_result(request.prName, request.prLink, {}, "PENDING")
+        if result_id is None:
+            raise HTTPException(status_code=500, detail="Failed to create result")
+        
+        return {
+            "success": True,
+            "message": "Result created successfully",
+            "data": {
+                "id": result_id,
+                "pr-link": request.prLink,
+                "pr-name": request.prName,
+                "res-success": False
+            }
+        }
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to create result: {str(e)}")
+
+@app.patch("/results/{result_id}")
+async def update_result_endpoint(result_id: int, request: UpdateResultRequest):
+    """Update result success status"""
+    try:
+        if not _has_client():
+            raise HTTPException(status_code=500, detail="Database not configured")
+        
+        # Import here to handle Vercel deployment issues
+        from supabase import create_client
+        supabase = create_client(os.getenv('SUPABASE_URL'), os.getenv('SUPABASE_KEY'))
+        
+        response = supabase.table('results').update({
+            'res-success': request.resSuccess
+        }).eq('id', result_id).execute()
+        
+        if not response.data:
+            raise HTTPException(status_code=404, detail="Result not found")
+        
+        return {
+            "success": True,
+            "message": "Result updated successfully",
+            "data": response.data[0]
+        }
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to update result: {str(e)}")
+
+@app.get("/results")
+async def get_all_results():
+    """Get all results"""
+    try:
+        if not _has_client():
+            raise HTTPException(status_code=500, detail="Database not configured")
+        
+        from supabase import create_client
+        supabase = create_client(os.getenv('SUPABASE_URL'), os.getenv('SUPABASE_KEY'))
+        
+        response = supabase.table('results').select('*').order('created_at', desc=True).execute()
+        
+        return {
+            "success": True,
+            "data": response.data or []
+        }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to fetch results: {str(e)}")
+
+# Suite endpoints
+
+@app.post("/suites")
+async def create_suite_endpoint(request: CreateSuiteRequest):
+    """Create a new test suite"""
+    try:
+        if not _has_client():
+            raise HTTPException(status_code=500, detail="Database not configured")
+        
+        from supabase import create_client
+        supabase = create_client(os.getenv('SUPABASE_URL'), os.getenv('SUPABASE_KEY'))
+        
+        suite_data = {
+            'result_id': request.resultId,
+            'name': request.name,
+            'suites-success': request.suitesSuccess
+        }
+        if request.s3Link:
+            suite_data['s3-link'] = request.s3Link
+        
+        response = supabase.table('suites').insert([suite_data]).execute()
+        
+        if not response.data:
+            raise HTTPException(status_code=500, detail="Failed to create suite")
+        
+        return {
+            "success": True,
+            "message": "Suite created successfully",
+            "data": response.data[0]
+        }
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to create suite: {str(e)}")
+
+@app.patch("/suites/{suite_id}")
+async def update_suite_endpoint(suite_id: int, request: UpdateSuiteRequest):
+    """Update suite success status and/or S3 link"""
+    try:
+        if not _has_client():
+            raise HTTPException(status_code=500, detail="Database not configured")
+        
+        from supabase import create_client
+        supabase = create_client(os.getenv('SUPABASE_URL'), os.getenv('SUPABASE_KEY'))
+        
+        update_data = {}
+        if request.suitesSuccess is not None:
+            update_data['suites-success'] = request.suitesSuccess
+        if request.s3Link is not None:
+            update_data['s3-link'] = request.s3Link
+        
+        if not update_data:
+            raise HTTPException(status_code=400, detail="No update data provided")
+        
+        response = supabase.table('suites').update(update_data).eq('id', suite_id).execute()
+        
+        if not response.data:
+            raise HTTPException(status_code=404, detail="Suite not found")
+        
+        return {
+            "success": True,
+            "message": "Suite updated successfully",
+            "data": response.data[0]
+        }
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to update suite: {str(e)}")
+
+@app.get("/results/{result_id}/suites")
+async def get_suites_for_result(result_id: int):
+    """Get suites for a specific result"""
+    try:
+        if not _has_client():
+            raise HTTPException(status_code=500, detail="Database not configured")
+        
+        from supabase import create_client
+        supabase = create_client(os.getenv('SUPABASE_URL'), os.getenv('SUPABASE_KEY'))
+        
+        response = supabase.table('suites').select('*').eq('result_id', result_id).order('created_at', desc=True).execute()
+        
+        return {
+            "success": True,
+            "data": response.data or []
+        }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to fetch suites: {str(e)}")
+
+# Test endpoints
+
+@app.post("/tests")
+async def create_test_endpoint(request: CreateTestRequest):
+    """Create a new individual test"""
+    try:
+        if not _has_client():
+            raise HTTPException(status_code=500, detail="Database not configured")
+        
+        from supabase import create_client
+        supabase = create_client(os.getenv('SUPABASE_URL'), os.getenv('SUPABASE_KEY'))
+        
+        test_data = {
+            'suite_id': request.suiteId,
+            'name': request.name,
+            'summary': request.summary,
+            'test-success': request.testSuccess
+        }
+        
+        response = supabase.table('tests').insert([test_data]).execute()
+        
+        if not response.data:
+            raise HTTPException(status_code=500, detail="Failed to create test")
+        
+        return {
+            "success": True,
+            "message": "Test created successfully",
+            "data": response.data[0]
+        }
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to create test: {str(e)}")
+
+@app.patch("/tests/{test_id}")
+async def update_test_endpoint(test_id: int, request: UpdateTestRequest):
+    """Update test success status and/or summary"""
+    try:
+        if not _has_client():
+            raise HTTPException(status_code=500, detail="Database not configured")
+        
+        from supabase import create_client
+        supabase = create_client(os.getenv('SUPABASE_URL'), os.getenv('SUPABASE_KEY'))
+        
+        update_data = {}
+        if request.testSuccess is not None:
+            update_data['test-success'] = request.testSuccess
+        if request.summary is not None:
+            update_data['summary'] = request.summary
+        
+        if not update_data:
+            raise HTTPException(status_code=400, detail="No update data provided")
+        
+        response = supabase.table('tests').update(update_data).eq('id', test_id).execute()
+        
+        if not response.data:
+            raise HTTPException(status_code=404, detail="Test not found")
+        
+        return {
+            "success": True,
+            "message": "Test updated successfully",
+            "data": response.data[0]
+        }
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to update test: {str(e)}")
+
+@app.get("/suites/{suite_id}/tests")
+async def get_tests_for_suite(suite_id: int):
+    """Get tests for a specific suite"""
+    try:
+        if not _has_client():
+            raise HTTPException(status_code=500, detail="Database not configured")
+        
+        from supabase import create_client
+        supabase = create_client(os.getenv('SUPABASE_URL'), os.getenv('SUPABASE_KEY'))
+        
+        response = supabase.table('tests').select('*').eq('suite_id', suite_id).order('created_at', desc=True).execute()
+        
+        return {
+            "success": True,
+            "data": response.data or []
+        }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to fetch tests: {str(e)}")
+
+# Agent execution endpoints
+
+@app.post("/run-suite")
+async def run_suite_endpoint(request: RunSuiteRequest):
+    """
+    Run a test suite by ID - This is the main endpoint called by the CICD pipeline
+    """
+    try:
+        result = await run_qai_tests(request.suite_id)
+        
+        if result['agent_result']['status'] == 'success':
+            return {
+                "status": "success",
+                "message": f"Suite {request.suite_id} executed successfully",
+                "data": result
+            }
+        else:
+            raise HTTPException(
+                status_code=500,
+                detail=f"Suite execution failed: {result['agent_result'].get('error', 'Unknown error')}"
+            )
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Suite execution failed: {str(e)}")
+
+@app.post("/run-agent")
+async def run_agent_endpoint(request: AgentRunRequest):
+    """Run a single agent test (legacy endpoint)"""
+    try:
+        result = await run_single_agent(request.spec)
+        return {"status": "success", "result": result}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Agent execution failed: {str(e)}")
+
+@app.post("/run-agents")
+async def run_agents_endpoint(request: MultiAgentRunRequest):
+    """Run multiple agent tests (legacy endpoint)"""
+    try:
+        result = await run_agents(request.test_specs, request.pr_name, request.pr_link)
+        return {"status": "success", "result": result}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Agents execution failed: {str(e)}")
+
+@app.get("/")
+async def root():
+    """Root endpoint"""
+    return {
+        "message": "QAI Agent Runner API",
+        "version": "1.0.0",
+        "deployment": "vercel" if os.getenv("VERCEL") else "local",
+        "endpoints": {
+            "health": "/health",
+            "results": "/results",
+            "suites": "/suites", 
+            "tests": "/tests",
+            "run_suite": "/run-suite"
+        }
+    }
+
+# Export the FastAPI app for Vercel
+handler = app
+app_handler = app
\ No newline at end of file
diff --git a/backend/agents/database.py b/backend/agents/database.py
index 2e49f3b..fd9eb37 100644
--- a/backend/agents/database.py
+++ b/backend/agents/database.py
@@ -99,6 +99,43 @@ async def update_test_fields(test_id: int, fields: Dict[str, Any]) -> None:
 		print(f"[db] ❌ update_test_fields error: {str(e)}")
 
 
+async def get_suite_with_tests(suite_id: int) -> Optional[Dict[str, Any]]:
+	"""Fetch a suite with all its tests from the database."""
+	try:
+		if not _has_client():
+			print(f"[db] Skipping get_suite_with_tests for suite {suite_id}: no client")
+			return None
+		
+		# Get suite info
+		suite_resp = supabase.table('suites').select('*').eq('id', suite_id).single().execute()
+		if not suite_resp.data:
+			print(f"[db] Suite {suite_id} not found")
+			return None
+		
+		suite_data = suite_resp.data
+		
+		# Get associated tests
+		tests_resp = supabase.table('tests').select('*').eq('suite_id', suite_id).execute()
+		tests = tests_resp.data or []
+		
+		# Convert tests to the format expected by agents
+		formatted_tests = []
+		for test in tests:
+			formatted_tests.append({
+				'name': test.get('name', 'Untitled Test'),
+				'instructions': test.get('summary', '').split('\n') if test.get('summary') else ['Run basic test'],
+			})
+		
+		return {
+			'id': suite_data['id'],
+			'name': suite_data.get('name', 'Untitled Suite'),
+			'tests': formatted_tests
+		}
+	except Exception as e:
+		print(f"[db] ❌ get_suite_with_tests error: {str(e)}")
+		return None
+
+
 async def get_result_id_for_suite(suite_id: int) -> Optional[int]:
 	"""Return result_id for a given suite_id from suites table."""
 	try:
diff --git a/backend/agents/main.py b/backend/agents/main.py
new file mode 100644
index 0000000..a716318
--- /dev/null
+++ b/backend/agents/main.py
@@ -0,0 +1,391 @@
+from fastapi import FastAPI, HTTPException
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel
+from typing import List, Dict, Any, Optional
+import asyncio
+import os
+from dotenv import load_dotenv
+
+from runner import run_single_agent, run_agents, run_qai_tests
+from database import (
+    get_or_create_test,
+    create_result,
+    set_suite_result_id,
+    get_suite_with_tests,
+    update_test_fields,
+    append_test_step,
+    _has_client
+)
+
+load_dotenv()
+
+app = FastAPI(
+    title="QAI Agent Runner API", 
+    version="1.0.0",
+    description="FastAPI server for QAI autonomous testing agents"
+)
+
+# Configure CORS
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Pydantic models matching API.md specification
+
+class CreateResultRequest(BaseModel):
+    prLink: str
+    prName: str
+
+class UpdateResultRequest(BaseModel):
+    resSuccess: bool
+
+class CreateSuiteRequest(BaseModel):
+    resultId: int
+    name: str
+    s3Link: Optional[str] = None
+    suitesSuccess: Optional[bool] = False
+
+class UpdateSuiteRequest(BaseModel):
+    suitesSuccess: Optional[bool] = None
+    s3Link: Optional[str] = None
+
+class CreateTestRequest(BaseModel):
+    suiteId: int
+    name: str
+    summary: Optional[str] = None
+    testSuccess: Optional[bool] = False
+
+class UpdateTestRequest(BaseModel):
+    testSuccess: Optional[bool] = None
+    summary: Optional[str] = None
+
+class RunSuiteRequest(BaseModel):
+    suite_id: int
+
+class AgentRunRequest(BaseModel):
+    spec: Dict[str, Any]
+
+class MultiAgentRunRequest(BaseModel):
+    test_specs: List[Dict[str, Any]]
+    pr_name: str
+    pr_link: str
+
+@app.get("/health")
+async def health_check():
+    """Health check endpoint"""
+    return {
+        "status": "healthy",
+        "database_connected": _has_client(),
+        "version": "1.0.0"
+    }
+
+# Results endpoints (matching API.md)
+
+@app.post("/results")
+async def create_result_endpoint(request: CreateResultRequest):
+    """Create a new result (PR test run)"""
+    try:
+        if not _has_client():
+            raise HTTPException(status_code=500, detail="Database not configured")
+        
+        # Use the database function to create result
+        result_id = await create_result(request.prName, request.prLink, {}, "PENDING")
+        if result_id is None:
+            raise HTTPException(status_code=500, detail="Failed to create result")
+        
+        # Return response matching API.md format
+        return {
+            "success": True,
+            "message": "Result created successfully",
+            "data": {
+                "id": result_id,
+                "pr-link": request.prLink,
+                "pr-name": request.prName,
+                "res-success": False
+            }
+        }
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to create result: {str(e)}")
+
+@app.patch("/results/{result_id}")
+async def update_result_endpoint(result_id: int, request: UpdateResultRequest):
+    """Update result success status"""
+    try:
+        if not _has_client():
+            raise HTTPException(status_code=500, detail="Database not configured")
+        
+        from database import supabase
+        
+        response = supabase.table('results').update({
+            'res-success': request.resSuccess
+        }).eq('id', result_id).execute()
+        
+        if not response.data:
+            raise HTTPException(status_code=404, detail="Result not found")
+        
+        return {
+            "success": True,
+            "message": "Result updated successfully",
+            "data": response.data[0]
+        }
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to update result: {str(e)}")
+
+@app.get("/results")
+async def get_all_results():
+    """Get all results"""
+    try:
+        if not _has_client():
+            raise HTTPException(status_code=500, detail="Database not configured")
+        
+        from database import supabase
+        
+        response = supabase.table('results').select('*').order('created_at', desc=True).execute()
+        
+        return {
+            "success": True,
+            "data": response.data or []
+        }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to fetch results: {str(e)}")
+
+# Suite endpoints
+
+@app.post("/suites")
+async def create_suite_endpoint(request: CreateSuiteRequest):
+    """Create a new test suite"""
+    try:
+        if not _has_client():
+            raise HTTPException(status_code=500, detail="Database not configured")
+        
+        from database import supabase
+        
+        suite_data = {
+            'result_id': request.resultId,
+            'name': request.name,
+            'suites-success': request.suitesSuccess
+        }
+        if request.s3Link:
+            suite_data['s3-link'] = request.s3Link
+        
+        response = supabase.table('suites').insert([suite_data]).execute()
+        
+        if not response.data:
+            raise HTTPException(status_code=500, detail="Failed to create suite")
+        
+        return {
+            "success": True,
+            "message": "Suite created successfully",
+            "data": response.data[0]
+        }
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to create suite: {str(e)}")
+
+@app.patch("/suites/{suite_id}")
+async def update_suite_endpoint(suite_id: int, request: UpdateSuiteRequest):
+    """Update suite success status and/or S3 link"""
+    try:
+        if not _has_client():
+            raise HTTPException(status_code=500, detail="Database not configured")
+        
+        from database import supabase
+        
+        update_data = {}
+        if request.suitesSuccess is not None:
+            update_data['suites-success'] = request.suitesSuccess
+        if request.s3Link is not None:
+            update_data['s3-link'] = request.s3Link
+        
+        if not update_data:
+            raise HTTPException(status_code=400, detail="No update data provided")
+        
+        response = supabase.table('suites').update(update_data).eq('id', suite_id).execute()
+        
+        if not response.data:
+            raise HTTPException(status_code=404, detail="Suite not found")
+        
+        return {
+            "success": True,
+            "message": "Suite updated successfully",
+            "data": response.data[0]
+        }
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to update suite: {str(e)}")
+
+@app.get("/results/{result_id}/suites")
+async def get_suites_for_result(result_id: int):
+    """Get suites for a specific result"""
+    try:
+        if not _has_client():
+            raise HTTPException(status_code=500, detail="Database not configured")
+        
+        from database import supabase
+        
+        response = supabase.table('suites').select('*').eq('result_id', result_id).order('created_at', desc=True).execute()
+        
+        return {
+            "success": True,
+            "data": response.data or []
+        }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to fetch suites: {str(e)}")
+
+# Test endpoints
+
+@app.post("/tests")
+async def create_test_endpoint(request: CreateTestRequest):
+    """Create a new individual test"""
+    try:
+        if not _has_client():
+            raise HTTPException(status_code=500, detail="Database not configured")
+        
+        from database import supabase
+        
+        test_data = {
+            'suite_id': request.suiteId,
+            'name': request.name,
+            'summary': request.summary,
+            'test-success': request.testSuccess
+        }
+        
+        response = supabase.table('tests').insert([test_data]).execute()
+        
+        if not response.data:
+            raise HTTPException(status_code=500, detail="Failed to create test")
+        
+        return {
+            "success": True,
+            "message": "Test created successfully",
+            "data": response.data[0]
+        }
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to create test: {str(e)}")
+
+@app.patch("/tests/{test_id}")
+async def update_test_endpoint(test_id: int, request: UpdateTestRequest):
+    """Update test success status and/or summary"""
+    try:
+        if not _has_client():
+            raise HTTPException(status_code=500, detail="Database not configured")
+        
+        from database import supabase
+        
+        update_data = {}
+        if request.testSuccess is not None:
+            update_data['test-success'] = request.testSuccess
+        if request.summary is not None:
+            update_data['summary'] = request.summary
+        
+        if not update_data:
+            raise HTTPException(status_code=400, detail="No update data provided")
+        
+        response = supabase.table('tests').update(update_data).eq('id', test_id).execute()
+        
+        if not response.data:
+            raise HTTPException(status_code=404, detail="Test not found")
+        
+        return {
+            "success": True,
+            "message": "Test updated successfully",
+            "data": response.data[0]
+        }
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to update test: {str(e)}")
+
+@app.get("/suites/{suite_id}/tests")
+async def get_tests_for_suite(suite_id: int):
+    """Get tests for a specific suite"""
+    try:
+        if not _has_client():
+            raise HTTPException(status_code=500, detail="Database not configured")
+        
+        from database import supabase
+        
+        response = supabase.table('tests').select('*').eq('suite_id', suite_id).order('created_at', desc=True).execute()
+        
+        return {
+            "success": True,
+            "data": response.data or []
+        }
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to fetch tests: {str(e)}")
+
+# Agent execution endpoints
+
+@app.post("/run-suite")
+async def run_suite_endpoint(request: RunSuiteRequest):
+    """
+    Run a test suite by ID - This is the main endpoint called by the CICD pipeline
+    """
+    try:
+        result = await run_qai_tests(request.suite_id)
+        
+        if result['agent_result']['status'] == 'success':
+            return {
+                "status": "success",
+                "message": f"Suite {request.suite_id} executed successfully",
+                "data": result
+            }
+        else:
+            raise HTTPException(
+                status_code=500,
+                detail=f"Suite execution failed: {result['agent_result'].get('error', 'Unknown error')}"
+            )
+    except HTTPException:
+        raise
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Suite execution failed: {str(e)}")
+
+@app.post("/run-agent")
+async def run_agent_endpoint(request: AgentRunRequest):
+    """Run a single agent test (legacy endpoint)"""
+    try:
+        result = await run_single_agent(request.spec)
+        return {"status": "success", "result": result}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Agent execution failed: {str(e)}")
+
+@app.post("/run-agents")
+async def run_agents_endpoint(request: MultiAgentRunRequest):
+    """Run multiple agent tests (legacy endpoint)"""
+    try:
+        result = await run_agents(request.test_specs, request.pr_name, request.pr_link)
+        return {"status": "success", "result": result}
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Agents execution failed: {str(e)}")
+
+@app.get("/")
+async def root():
+    """Root endpoint"""
+    return {
+        "message": "QAI Agent Runner API",
+        "version": "1.0.0",
+        "endpoints": {
+            "health": "/health",
+            "results": "/results",
+            "suites": "/suites", 
+            "tests": "/tests",
+            "run_suite": "/run-suite"
+        }
+    }
+
+# For Vercel deployment
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=int(os.getenv("PORT", "8000")))
\ No newline at end of file
diff --git a/backend/agents/requirements.txt b/backend/agents/requirements.txt
index 346ee94..0817eeb 100644
--- a/backend/agents/requirements.txt
+++ b/backend/agents/requirements.txt
@@ -1,5 +1,6 @@
 supabase==2.0.2
 python-dotenv==1.0.0
-asyncio
-pathlib
-typing
\ No newline at end of file
+fastapi==0.104.1
+uvicorn==0.24.0
+pydantic==2.5.0
+typing-extensions==4.8.0
\ No newline at end of file
diff --git a/backend/agents/runner.py b/backend/agents/runner.py
index 3f83c51..31f4975 100644
--- a/backend/agents/runner.py
+++ b/backend/agents/runner.py
@@ -7,16 +7,16 @@
 from dotenv import load_dotenv
 from enum import Enum
 
-from .database import (
+from database import (
     get_or_create_test,
     append_test_step,
     update_test_fields,
     create_result,
     set_suite_result_id,
 )
-from .prompts import build_agent_instructions
-from .utils import normalize_tests, make_remote_recording_dir, process_item
-from .record import start_recording, stop_recording
+from prompts import build_agent_instructions
+from utils import normalize_tests, make_remote_recording_dir, process_item
+from record import start_recording, stop_recording
 
 class RunStatus(Enum):
     QUEUED = "QUEUED"
@@ -177,6 +177,54 @@ def _prepare_step_for_storage(item: Dict[str, Any]):
     
     return await _execute()
 
+async def run_qai_tests(suite_id: int) -> Dict[str, Any]:
+    """
+    Run tests for a specific suite ID from the database
+    This function is called by run_suite.py and qai-pipeline.js
+    """
+    from database import get_suite_with_tests
+    
+    try:
+        # Fetch suite and test data from database
+        suite_data = await get_suite_with_tests(suite_id)
+        if not suite_data:
+            return {
+                'agent_result': {
+                    'status': 'failed',
+                    'error': f'Suite {suite_id} not found'
+                }
+            }
+        
+        # Convert database format to agent spec format
+        spec = {
+            'suite_id': suite_id,
+            'model': os.getenv("CUA_MODEL", "anthropic/claude-3-5-sonnet-20241022"),
+            'budget': 5.0,
+            'container_name': os.getenv("CUA_CONTAINER_NAME"),
+            'tests': suite_data.get('tests', [])
+        }
+        
+        # Run the agent
+        result = await run_single_agent(spec)
+        
+        return {
+            'agent_result': {
+                'status': 'success',
+                'suite_id': suite_id,
+                'tests_run': len(result),
+                'results': result
+            }
+        }
+    except Exception as e:
+        print(f"Error running QAI tests for suite {suite_id}: {e}")
+        return {
+            'agent_result': {
+                'status': 'failed',
+                'error': str(e),
+                'suite_id': suite_id
+            }
+        }
+
 async def run_agents(test_specs: List[Dict[str, Any]], pr_name: str, pr_link: str) -> Dict[str, Any]:
     tasks = [run_single_agent(spec) for spec in test_specs]
     results: List[Dict[str, Any]] = await asyncio.gather(*tasks, return_exceptions=True)
diff --git a/backend/agents/vercel.json b/backend/agents/vercel.json
new file mode 100644
index 0000000..2dc2994
--- /dev/null
+++ b/backend/agents/vercel.json
@@ -0,0 +1,18 @@
+{
+  "version": 2,
+  "builds": [
+    {
+      "src": "api/index.py",
+      "use": "@vercel/python"
+    }
+  ],
+  "routes": [
+    {
+      "src": "/(.*)",
+      "dest": "api/index.py"
+    }
+  ],
+  "env": {
+    "PYTHONPATH": "."
+  }
+}
\ No newline at end of file

From 96896a9f5b2313894de50ada35bfd64acf5ec55d Mon Sep 17 00:00:00 2001
From: owenguoo <88258850+owenguoo@users.noreply.github.com>
Date: Sat, 13 Sep 2025 20:48:59 -0400
Subject: [PATCH 2/8] duplciate name fix

---
 backend/cicd/qai-pipeline.js | 1 -
 1 file changed, 1 deletion(-)

diff --git a/backend/cicd/qai-pipeline.js b/backend/cicd/qai-pipeline.js
index 59a816f..7786786 100644
--- a/backend/cicd/qai-pipeline.js
+++ b/backend/cicd/qai-pipeline.js
@@ -218,7 +218,6 @@ Generate focused test scenarios for autonomous agents.`
       const finalSuccess = await this.verifyFinalResults();
       
       console.log(`::set-output name=success::${finalSuccess}`);
-      const finalSuccess = await this.verifyFinalResults();
       return finalSuccess;
     } catch (error) {
       console.error(`❌ Agent execution failed: ${error.message}`);

From 885db0d266f25ad82ed557751cae56a24ba18779 Mon Sep 17 00:00:00 2001
From: owenguoo <88258850+owenguoo@users.noreply.github.com>
Date: Sat, 13 Sep 2025 20:52:39 -0400
Subject: [PATCH 3/8] updates to match new schema

---
 backend/agents/database.py   |  3 +-
 backend/cicd/qai-pipeline.js | 68 +++++++++++++++++++-----------------
 2 files changed, 38 insertions(+), 33 deletions(-)

diff --git a/backend/agents/database.py b/backend/agents/database.py
index fd9eb37..474fb7b 100644
--- a/backend/agents/database.py
+++ b/backend/agents/database.py
@@ -21,7 +21,7 @@ async def create_result(pr_name: str, pr_link: str, overall_result: Dict[str, An
 			return None
 		payload = {
 			"pr_name": pr_name,
-			"pr_link": pr_link,
+			"pr-link": pr_link,
 			"overall_result": overall_result,
 			"run_status": run_status,
 		}
@@ -62,6 +62,7 @@ async def get_or_create_test(suite_id: int, name: str) -> Optional[int]:
 			"name": name,
 			"steps": [],
 			"run_status": "RUNNING",
+			"test_success": None,
 		}
 		ins = supabase.table('tests').insert(payload).execute()
 		row = (ins.data or [{}])[0]
diff --git a/backend/cicd/qai-pipeline.js b/backend/cicd/qai-pipeline.js
index 7786786..471c0df 100644
--- a/backend/cicd/qai-pipeline.js
+++ b/backend/cicd/qai-pipeline.js
@@ -226,7 +226,7 @@ Generate focused test scenarios for autonomous agents.`
       if (this.resultId) {
         await this.supabase
           .from('results')
-          .update({ 'res-success': false })
+          .update({ 'run_status': 'FAILED' })
           .eq('id', this.resultId);
       }
       
@@ -239,8 +239,9 @@ Generate focused test scenarios for autonomous agents.`
       // First, create the results record (PR level)
       const prData = {
         'pr-link': `https://github.com/${this.repo.join('/')}/pull/${this.prNumber}`,
-        'pr-name': `PR #${this.prNumber}`,
-        'res-success': false // Will be updated after tests complete
+        'pr_name': `PR #${this.prNumber}`,
+        'overall_result': {},
+        'run_status': 'PENDING'
       };
 
       const { data: resultData, error: resultError } = await this.supabase
@@ -270,8 +271,7 @@ Generate focused test scenarios for autonomous agents.`
       for (const [persona, personaScenarios] of Object.entries(personaGroups)) {
         const suiteRecord = {
           result_id: this.resultId, // Foreign key to results table
-          name: `${persona} Agent Suite`,
-          'suites-success': null // Will be updated after tests
+          name: `${persona} Agent Suite`
         };
 
         const { data: suiteData, error: suiteError } = await this.supabase
@@ -291,8 +291,9 @@ Generate focused test scenarios for autonomous agents.`
         const testRecords = personaScenarios.map(scenario => ({
           suite_id: suiteData.id, // Foreign key to suites table
           name: scenario.description,
-          summary: `${scenario.type} test with ${scenario.priority} priority`,
-          'test-success': null
+          test_success: null,
+          run_status: 'PENDING',
+          steps: []
         }));
 
         const { error: testsError } = await this.supabase
@@ -317,8 +318,8 @@ Generate focused test scenarios for autonomous agents.`
         const { error } = await this.supabase
           .from('tests')
           .update({
-            'test-success': result.success,
-            summary: result.error || `Test completed in ${result.duration}ms`
+            'test_success': result.success,
+            'run_status': result.success ? 'PASSED' : 'FAILED'
           })
           .eq('name', result.scenario.description);
 
@@ -336,33 +337,33 @@ Generate focused test scenarios for autonomous agents.`
       for (const suite of suites || []) {
         const { data: suiteTests } = await this.supabase
           .from('tests')
-          .select('test-success')
+          .select('test_success')
           .eq('suite_id', suite.id);
 
-        const allTestsComplete = suiteTests?.every(test => test['test-success'] !== null);
-        const allTestsPassed = suiteTests?.every(test => test['test-success'] === true);
+        const allTestsComplete = suiteTests?.every(test => test.test_success !== null);
+        const allTestsPassed = suiteTests?.every(test => test.test_success === true);
 
-        if (allTestsComplete) {
-          await this.supabase
-            .from('suites')
-            .update({ 'suites-success': allTestsPassed })
-            .eq('id', suite.id);
-        }
+        // Note: suites table doesn't have success column in the provided schema
+        // Tests success will be tracked at the result level
       }
 
-      // Update overall PR result
-      const { data: allSuites } = await this.supabase
-        .from('suites')
-        .select('suites-success')
-        .eq('result_id', this.resultId);
+      // Update overall PR result based on all tests
+      const { data: allTests } = await this.supabase
+        .from('tests')
+        .select('test_success')
+        .in('suite_id', suites?.map(s => s.id) || []);
 
-      const allSuitesComplete = allSuites?.every(suite => suite['suites-success'] !== null);
-      const allSuitesPassed = allSuites?.every(suite => suite['suites-success'] === true);
+      const totalTests = allTests?.length || 0;
+      const passedTests = allTests?.filter(test => test.test_success === true).length || 0;
+      const allTestsPassed = totalTests > 0 && passedTests === totalTests;
 
-      if (allSuitesComplete) {
+      if (totalTests > 0) {
         await this.supabase
           .from('results')
-          .update({ 'res-success': allSuitesPassed })
+          .update({ 
+            'run_status': allTestsPassed ? 'PASSED' : 'FAILED',
+            'overall_result': { passed: passedTests, total: totalTests }
+          })
           .eq('id', this.resultId);
       }
 
@@ -436,17 +437,20 @@ Your goal is to identify bugs and issues that human testers might miss through a
       
       if (result) {
         const totalSuites = result.suites?.length || 0;
-        const passedSuites = result.suites?.filter(s => s['suites-success'] === true).length || 0;
         const totalTests = result.suites?.reduce((acc, s) => acc + (s.tests?.length || 0), 0) || 0;
         const passedTests = result.suites?.reduce((acc, s) => 
-          acc + (s.tests?.filter(t => t['test-success'] === true).length || 0), 0) || 0;
+          acc + (s.tests?.filter(t => t.test_success === true).length || 0), 0) || 0;
+        
+        // Calculate success based on run_status and test results
+        const overallSuccess = result.run_status === 'PASSED' || (passedTests === totalTests && totalTests > 0);
         
         console.log(`📊 Final Results Summary:`);
-        console.log(`   • Overall Success: ${result['res-success'] ? '✅' : '❌'}`);
-        console.log(`   • Suites: ${passedSuites}/${totalSuites} passed`);
+        console.log(`   • Overall Status: ${result.run_status}`);
+        console.log(`   • Overall Success: ${overallSuccess ? '✅' : '❌'}`);
+        console.log(`   • Suites: ${totalSuites} total`);
         console.log(`   • Tests: ${passedTests}/${totalTests} passed`);
         
-        return result['res-success'];
+        return overallSuccess;
       }
     } catch (error) {
       console.error(`❌ Failed to verify results: ${error.message}`);

From b50a667e60351d989b613d3980129c83785d7cd3 Mon Sep 17 00:00:00 2001
From: owenguoo <88258850+owenguoo@users.noreply.github.com>
Date: Sat, 13 Sep 2025 20:55:12 -0400
Subject: [PATCH 4/8] update enum values

---
 backend/agents/database.py   |  2 +-
 backend/agents/runner.py     |  2 +-
 backend/cicd/qai-pipeline.js | 14 +++++++-------
 3 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/backend/agents/database.py b/backend/agents/database.py
index 474fb7b..1996490 100644
--- a/backend/agents/database.py
+++ b/backend/agents/database.py
@@ -61,7 +61,7 @@ async def get_or_create_test(suite_id: int, name: str) -> Optional[int]:
 			"suite_id": suite_id,
 			"name": name,
 			"steps": [],
-			"run_status": "RUNNING",
+			"run_status": "QUEUED",
 			"test_success": None,
 		}
 		ins = supabase.table('tests').insert(payload).execute()
diff --git a/backend/agents/runner.py b/backend/agents/runner.py
index 31f4975..f9f75ab 100644
--- a/backend/agents/runner.py
+++ b/backend/agents/runner.py
@@ -20,7 +20,7 @@
 
 class RunStatus(Enum):
     QUEUED = "QUEUED"
-    RUNNING = "RUNNING"
+    RUNNING = "RUNNING" 
     PASSED = "PASSED"
     FAILED = "FAILED"
 
diff --git a/backend/cicd/qai-pipeline.js b/backend/cicd/qai-pipeline.js
index 471c0df..afad594 100644
--- a/backend/cicd/qai-pipeline.js
+++ b/backend/cicd/qai-pipeline.js
@@ -226,7 +226,7 @@ Generate focused test scenarios for autonomous agents.`
       if (this.resultId) {
         await this.supabase
           .from('results')
-          .update({ 'run_status': 'FAILED' })
+          .update({ run_status: 'FAILED' })
           .eq('id', this.resultId);
       }
       
@@ -241,7 +241,7 @@ Generate focused test scenarios for autonomous agents.`
         'pr-link': `https://github.com/${this.repo.join('/')}/pull/${this.prNumber}`,
         'pr_name': `PR #${this.prNumber}`,
         'overall_result': {},
-        'run_status': 'PENDING'
+        'run_status': 'QUEUED'
       };
 
       const { data: resultData, error: resultError } = await this.supabase
@@ -292,7 +292,7 @@ Generate focused test scenarios for autonomous agents.`
           suite_id: suiteData.id, // Foreign key to suites table
           name: scenario.description,
           test_success: null,
-          run_status: 'PENDING',
+          run_status: 'QUEUED',
           steps: []
         }));
 
@@ -318,8 +318,8 @@ Generate focused test scenarios for autonomous agents.`
         const { error } = await this.supabase
           .from('tests')
           .update({
-            'test_success': result.success,
-            'run_status': result.success ? 'PASSED' : 'FAILED'
+            test_success: result.success,
+            run_status: result.success ? 'PASSED' : 'FAILED'
           })
           .eq('name', result.scenario.description);
 
@@ -361,8 +361,8 @@ Generate focused test scenarios for autonomous agents.`
         await this.supabase
           .from('results')
           .update({ 
-            'run_status': allTestsPassed ? 'PASSED' : 'FAILED',
-            'overall_result': { passed: passedTests, total: totalTests }
+            run_status: allTestsPassed ? 'PASSED' : 'FAILED',
+            overall_result: { passed: passedTests, total: totalTests }
           })
           .eq('id', this.resultId);
       }

From e58adc0754ca9df7c85379d0b9bf3c28e896d35c Mon Sep 17 00:00:00 2001
From: owenguoo <88258850+owenguoo@users.noreply.github.com>
Date: Sat, 13 Sep 2025 20:59:36 -0400
Subject: [PATCH 5/8] fix dependies

---
 .github/workflows/qai-test.yml | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/qai-test.yml b/.github/workflows/qai-test.yml
index af6363e..23a7fc8 100644
--- a/.github/workflows/qai-test.yml
+++ b/.github/workflows/qai-test.yml
@@ -21,7 +21,19 @@ jobs:
         cache: 'npm'
         cache-dependency-path: 'backend/cicd/package-lock.json'
 
-    - name: Install dependencies
+    - name: Setup Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.11'
+        cache: 'pip'
+
+    - name: Install Python dependencies
+      run: |
+        cd backend
+        pip install -r requirements.txt
+        pip install python-dotenv supabase
+
+    - name: Install Node dependencies
       run: |
         cd backend/cicd
         npm install
@@ -31,10 +43,15 @@ jobs:
         cd backend/cicd
         npm run pipeline
       env:
+        PYTHONPATH: ${{ github.workspace }}/backend/agents
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         QAI_ENDPOINT: ${{ secrets.QAI_ENDPOINT }}
         SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
         SUPABASE_KEY: ${{ secrets.SUPABASE_KEY }}
         DEPLOYMENT_URL: ${{ secrets.DEPLOYMENT_URL }}
-        AGENT_TIMEOUT: ${{ secrets.AGENT_TIMEOUT }}
\ No newline at end of file
+        AGENT_TIMEOUT: ${{ secrets.AGENT_TIMEOUT }}
+        CUA_API_KEY: ${{ secrets.CUA_API_KEY }}
+        CUA_CONTAINER_NAME: ${{ secrets.CUA_CONTAINER_NAME }}
+        ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+        CUA_MODEL: anthropic/claude-3-5-sonnet-20241022
\ No newline at end of file

From 8dd5cee1052a410bce169f898339f9a89f860d9b Mon Sep 17 00:00:00 2001
From: owenguoo <88258850+owenguoo@users.noreply.github.com>
Date: Sat, 13 Sep 2025 21:07:00 -0400
Subject: [PATCH 6/8] l

---
 .github/workflows/qai-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/qai-test.yml b/.github/workflows/qai-test.yml
index 23a7fc8..79bf910 100644
--- a/.github/workflows/qai-test.yml
+++ b/.github/workflows/qai-test.yml
@@ -43,7 +43,7 @@ jobs:
         cd backend/cicd
         npm run pipeline
       env:
-        PYTHONPATH: ${{ github.workspace }}/backend/agents
+        PYTHONPATH: ${{ github.workspace }}/backend:${{ github.workspace }}/backend/agents
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         QAI_ENDPOINT: ${{ secrets.QAI_ENDPOINT }}

From d6877be0f45d49c2ff385da39606965053826d5c Mon Sep 17 00:00:00 2001
From: owenguoo <88258850+owenguoo@users.noreply.github.com>
Date: Sat, 13 Sep 2025 21:07:37 -0400
Subject: [PATCH 7/8] g

---
 .github/workflows/qai-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/qai-test.yml b/.github/workflows/qai-test.yml
index 79bf910..d871d71 100644
--- a/.github/workflows/qai-test.yml
+++ b/.github/workflows/qai-test.yml
@@ -24,7 +24,7 @@ jobs:
     - name: Setup Python
       uses: actions/setup-python@v5
       with:
-        python-version: '3.11'
+        python-version: '3.12'
         cache: 'pip'
 
     - name: Install Python dependencies

From 6ea385fe4d497280ae7daf8e71845a17490ada15 Mon Sep 17 00:00:00 2001
From: owenguoo <88258850+owenguoo@users.noreply.github.com>
Date: Sat, 13 Sep 2025 21:15:23 -0400
Subject: [PATCH 8/8] pkjlkn

---
 .github/workflows/qai-test.yml |  18 +-----
 backend/cicd/DEPLOYMENT.md     |   6 +-
 backend/cicd/qai-pipeline.js   | 106 +++++++++++++--------------------
 3 files changed, 47 insertions(+), 83 deletions(-)

diff --git a/.github/workflows/qai-test.yml b/.github/workflows/qai-test.yml
index d871d71..53a8732 100644
--- a/.github/workflows/qai-test.yml
+++ b/.github/workflows/qai-test.yml
@@ -21,17 +21,6 @@ jobs:
         cache: 'npm'
         cache-dependency-path: 'backend/cicd/package-lock.json'
 
-    - name: Setup Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.12'
-        cache: 'pip'
-
-    - name: Install Python dependencies
-      run: |
-        cd backend
-        pip install -r requirements.txt
-        pip install python-dotenv supabase
 
     - name: Install Node dependencies
       run: |
@@ -43,15 +32,10 @@ jobs:
         cd backend/cicd
         npm run pipeline
       env:
-        PYTHONPATH: ${{ github.workspace }}/backend:${{ github.workspace }}/backend/agents
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
         QAI_ENDPOINT: ${{ secrets.QAI_ENDPOINT }}
         SUPABASE_URL: ${{ secrets.SUPABASE_URL }}
         SUPABASE_KEY: ${{ secrets.SUPABASE_KEY }}
         DEPLOYMENT_URL: ${{ secrets.DEPLOYMENT_URL }}
-        AGENT_TIMEOUT: ${{ secrets.AGENT_TIMEOUT }}
-        CUA_API_KEY: ${{ secrets.CUA_API_KEY }}
-        CUA_CONTAINER_NAME: ${{ secrets.CUA_CONTAINER_NAME }}
-        ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
-        CUA_MODEL: anthropic/claude-3-5-sonnet-20241022
\ No newline at end of file
+        AGENT_TIMEOUT: ${{ secrets.AGENT_TIMEOUT }}
\ No newline at end of file
diff --git a/backend/cicd/DEPLOYMENT.md b/backend/cicd/DEPLOYMENT.md
index 2381d6e..9587e85 100644
--- a/backend/cicd/DEPLOYMENT.md
+++ b/backend/cicd/DEPLOYMENT.md
@@ -89,10 +89,12 @@ Your teammate's agent endpoint should:
 The workflow will:
 - ✅ Analyze your PR changes
 - ✅ Generate relevant test scenarios using LLM
-- ✅ Send scenarios to your agent endpoint
-- ✅ Pass/fail the CI based on test results
+- ✅ Upload scenarios to database and call QAI API endpoint
+- ✅ Pass/fail the CI based on test results from the API
 - ✅ Update codebase summary if tests pass
 
+**Note:** The pipeline now calls the QAI API endpoint (`/run-suite`) instead of running agents locally in GitHub Actions. This means GitHub Actions only needs the `QAI_ENDPOINT` URL and doesn't require `CUA_API_KEY` or other agent-specific credentials.
+
 ## Structured Output Benefits
 
 ✅ **No JSON parsing failures** - Uses OpenAI's structured output with schema validation
diff --git a/backend/cicd/qai-pipeline.js b/backend/cicd/qai-pipeline.js
index afad594..d8f80db 100644
--- a/backend/cicd/qai-pipeline.js
+++ b/backend/cicd/qai-pipeline.js
@@ -101,8 +101,13 @@ Generate focused test scenarios for autonomous agents.`
     this.createAgentFiles(scenarios);
     
     try {
-      // Use our database-integrated agent runner instead of external endpoint
-      console.log(`🤖 Running tests through integrated agent system...`);
+      // Use QAI API endpoint instead of running agents locally
+      console.log(`🤖 Running tests through QAI API endpoint...`);
+      
+      // Check if QAI_ENDPOINT is configured
+      if (!process.env.QAI_ENDPOINT) {
+        throw new Error('QAI_ENDPOINT environment variable is required');
+      }
       
       // Get the result_id from the database upload step
       if (!this.resultId) {
@@ -119,91 +124,64 @@ Generate focused test scenarios for autonomous agents.`
         throw new Error('No suites found for this result');
       }
       
-      console.log(`🏃 Running ${suites.length} agent suites...`);
-      
-      // Import and run our agent system for each suite
-      const { spawn } = require('child_process');
-      const path = require('path');
+      console.log(`🏃 Running ${suites.length} agent suites via API...`);
       
       const results = [];
+      const agentTimeout = parseInt(process.env.AGENT_TIMEOUT || '600000');
       
       for (const suite of suites) {
-        console.log(`🤖 Running suite: ${suite.name} (ID: ${suite.id})`);
+        console.log(`🤖 Calling API for suite: ${suite.name} (ID: ${suite.id})`);
         
         try {
-          // Run Python agent runner with suite_id
-          const pythonProcess = spawn('python3', [
-            path.join(__dirname, '../agents/run_suite.py'),
-            suite.id.toString()
-          ], {
-            env: { 
-              ...process.env,
-              SUITE_ID: suite.id.toString(),
-              DEPLOYMENT_URL: process.env.DEPLOYMENT_URL || 'https://staging.example.com'
+          // Call the QAI API endpoint to run the suite
+          const response = await axios.post(
+            `${process.env.QAI_ENDPOINT}/run-suite`,
+            { suite_id: suite.id },
+            { 
+              timeout: agentTimeout + 60000, // API timeout + 1 minute buffer
+              headers: { 'Content-Type': 'application/json' }
             }
-          });
-          
-          let output = '';
-          let errorOutput = '';
+          );
           
-          pythonProcess.stdout.on('data', (data) => {
-            output += data.toString();
-            console.log(`[Suite ${suite.id}] ${data.toString().trim()}`);
-          });
-          
-          pythonProcess.stderr.on('data', (data) => {
-            errorOutput += data.toString();
-            console.error(`[Suite ${suite.id} ERROR] ${data.toString().trim()}`);
-          });
-          
-          await new Promise((resolve, reject) => {
-            pythonProcess.on('close', (code) => {
-              if (code === 0) {
-                resolve();
-              } else {
-                reject(new Error(`Agent process exited with code ${code}: ${errorOutput}`));
-              }
-            });
-            
-            pythonProcess.on('error', (error) => {
-              reject(new Error(`Failed to start agent process: ${error.message}`));
+          if (response.data.status === 'success') {
+            console.log(`✅ Suite ${suite.id} completed successfully via API`);
+            results.push({
+              suite_id: suite.id,
+              suite_name: suite.name,
+              success: true,
+              api_response: response.data
             });
-            
-            // Set timeout for agent execution
-            setTimeout(() => {
-              pythonProcess.kill();
-              reject(new Error('Agent execution timed out'));
-            }, parseInt(process.env.AGENT_TIMEOUT || '600000'));
-          });
-          
-          // Parse agent result from output (agents save to database)
-          results.push({
-            suite_id: suite.id,
-            suite_name: suite.name,
-            success: true,
-            output: output
-          });
+          } else {
+            throw new Error(`API returned non-success status: ${response.data.status}`);
+          }
           
         } catch (error) {
-          console.error(`❌ Suite ${suite.id} failed: ${error.message}`);
+          console.error(`❌ Suite ${suite.id} API call failed: ${error.message}`);
+          
+          let errorMessage = error.message;
+          if (error.response) {
+            errorMessage = `HTTP ${error.response.status}: ${error.response.data?.detail || error.response.statusText}`;
+          } else if (error.code === 'ECONNREFUSED') {
+            errorMessage = 'Cannot connect to QAI API endpoint';
+          }
+          
           results.push({
             suite_id: suite.id,
             suite_name: suite.name,
             success: false,
-            error: error.message
+            error: errorMessage
           });
         }
       }
       
-      console.log(`✅ Completed ${results.length} agent suite runs`);
-      let results_legacy = results;
+      console.log(`✅ Completed ${results.length} agent suite API calls`);
       
       this.saveFile('test-results.json', results);
 
       const passed = results.filter(r => r.success).length;
       const failed = results.length - passed;
       
-      console.log(`📊 Test Results: ${passed}/${results.length} suites passed`);
+      console.log(`📊 API Results: ${passed}/${results.length} suites passed`);
       
       if (failed > 0) {
         console.log(`❌ Failed suites:`);
@@ -212,7 +190,7 @@ Generate focused test scenarios for autonomous agents.`
         });
       }
 
-      console.log(`💾 Database results already updated by agent system`);
+      console.log(`💾 Database results updated by QAI API system`);
       
       // Verify final database state
       const finalSuccess = await this.verifyFinalResults();
@@ -220,7 +198,7 @@ Generate focused test scenarios for autonomous agents.`
       console.log(`::set-output name=success::${finalSuccess}`);
       return finalSuccess;
     } catch (error) {
-      console.error(`❌ Agent execution failed: ${error.message}`);
+      console.error(`❌ QAI API execution failed: ${error.message}`);
       
       // Update database with failure status
       if (this.resultId) {