apache · bobbai00 · May 28, 2026
diff --git a/bin/k8s/templates/access-control-service-deployment.yaml b/bin/k8s/templates/access-control-service-deployment.yaml
@@ -50,6 +50,18 @@ spec:
               value: {{ .Values.workflowComputingUnitPool.name }}
             - name: KUBERNETES_COMPUTE_UNIT_POOL_NAMESPACE
               value: {{ .Values.workflowComputingUnitPool.namespace }}
+            {{- if .Values.litellm.enabled }}
+            # LLM gateway used to serve /api/chat and /api/models to the agent service.
+            - name: LITELLM_BASE_URL
+              value: http://{{ .Release.Name }}-{{ .Values.litellm.name }}-svc:{{ .Values.litellm.service.port }}
+            - name: LITELLM_MASTER_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Release.Name }}-agent-service-secret
+                  key: litellm-master-key
+            - name: GUI_WORKFLOW_WORKSPACE_COPILOT_ENABLED
+              value: "true"
+            {{- end }}
             {{- range .Values.texeraEnvVars }}
             - name: {{ .name }}
               value: "{{ .value }}"

diff --git a/bin/k8s/templates/agent-service-backend-traffic-policy.yaml b/bin/k8s/templates/agent-service-backend-traffic-policy.yaml
@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Agents are stateful and held in memory per pod, so all requests for a given
+# workflow must reach the same replica. The client stamps X-Agent-Workflow-Id on
+# every agent request; pin routing to it with a consistent hash.
+{{- if .Values.agentService.enabled }}
+apiVersion: gateway.envoyproxy.io/v1alpha1
+kind: BackendTrafficPolicy
+metadata:
+  name: {{ .Release.Name }}-agent-service-traffic-policy
+  namespace: {{ .Release.Namespace }}
+spec:
+  targetRefs:
+    - group: gateway.networking.k8s.io
+      kind: HTTPRoute
+      name: {{ .Release.Name }}-agent-service-route
+  loadBalancer:
+    type: ConsistentHash
+    consistentHash:
+      type: Header
+      header:
+        name: X-Agent-Workflow-Id
+{{- end }}
diff --git a/bin/k8s/templates/agent-service-deployment.yaml b/bin/k8s/templates/agent-service-deployment.yaml
@@ -0,0 +1,79 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+{{- if .Values.agentService.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Release.Name }}-{{ .Values.agentService.name }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ .Release.Name }}-{{ .Values.agentService.name }}
+spec:
+  replicas: {{ .Values.agentService.numOfPods | default 1 }}
+  selector:
+    matchLabels:
+      app: {{ .Release.Name }}-{{ .Values.agentService.name }}
+  template:
+    metadata:
+      labels:
+        app: {{ .Release.Name }}-{{ .Values.agentService.name }}
+    spec:
+      containers:
+        - name: {{ .Values.agentService.name }}
+          image: {{ .Values.texera.imageRegistry }}/{{ .Values.agentService.imageName }}:{{ .Values.texera.imageTag }}
+          imagePullPolicy: {{ .Values.texeraImages.pullPolicy }}
+          ports:
+            - containerPort: {{ .Values.agentService.service.port }}
+          # Env names match the agent service's config schema (agent-service/src/config/env.ts).
+          env:
+            - name: PORT
+              value: "{{ .Values.agentService.service.port }}"
+            # Dashboard service: workflow CRUD + operator metadata.
+            - name: TEXERA_DASHBOARD_SERVICE_ENDPOINT
+              value: http://{{ .Values.webserver.name }}-svc:{{ .Values.webserver.service.port }}
+            # LLM gateway: access-control-service serves /api/chat and /api/models,
+            # forwarding to LiteLLM (mirrors the single-node nginx routing).
+            - name: LLM_ENDPOINT
+              value: http://{{ .Release.Name }}-{{ .Values.accessControlService.name }}-svc:{{ .Values.accessControlService.service.port }}
+            - name: WORKFLOW_COMPILING_SERVICE_ENDPOINT
+              value: http://{{ .Values.workflowCompilingService.name }}-svc:{{ .Values.workflowCompilingService.service.port }}
+            # Per-computing-unit execution endpoint; "{cuid}" is substituted with the
+            # computing unit id at request time.
+            - name: EXECUTION_ENDPOINT_TEMPLATE
+              value: http://computing-unit-{cuid}.{{ .Values.workflowComputingUnitPool.name }}-svc.{{ .Values.workflowComputingUnitPool.namespace }}.svc.cluster.local:{{ .Values.workflowComputingUnitPool.service.port }}
+            - name: LLM_API_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Release.Name }}-agent-service-secret
+                  key: llm-api-key
+          # The service loads operator metadata from the dashboard service on
+          # startup, so gate readiness on its health endpoint before the gateway
+          # routes traffic here. /api/healthcheck needs no auth.
+          readinessProbe:
+            httpGet:
+              path: /api/healthcheck
+              port: {{ .Values.agentService.service.port }}
+            initialDelaySeconds: 5
+            periodSeconds: 5
+          livenessProbe:
+            httpGet:
+              path: /api/healthcheck
+              port: {{ .Values.agentService.service.port }}
+            initialDelaySeconds: 15
+            periodSeconds: 10
+{{- end }}
diff --git a/bin/k8s/templates/agent-service-secret.yaml b/bin/k8s/templates/agent-service-secret.yaml
@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Shared secret for the agent service and LiteLLM. Holds the agent's gateway
+# key, LiteLLM's master key, and the upstream provider API keys. Provide real
+# values via `--set` or a values override file; do not commit them.
+{{- if or .Values.agentService.enabled .Values.litellm.enabled }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Release.Name }}-agent-service-secret
+  namespace: {{ .Release.Namespace }}
+type: Opaque
+stringData:
+  llm-api-key: "{{ .Values.agentService.env.llmApiKey }}"
+  litellm-master-key: "{{ .Values.litellm.masterKey }}"
+  {{- range $key, $value := .Values.litellm.providerApiKeys }}
+  {{ $key }}: "{{ $value }}"
+  {{- end }}
+{{- end }}
diff --git a/bin/k8s/templates/agent-service-service.yaml b/bin/k8s/templates/agent-service-service.yaml
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+{{- if .Values.agentService.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Release.Name }}-{{ .Values.agentService.name }}-svc
+  namespace: {{ .Release.Namespace }}
+spec:
+  type: {{ .Values.agentService.service.type }}
+  selector:
+    app: {{ .Release.Name }}-{{ .Values.agentService.name }}
+  ports:
+    - protocol: TCP
+      port: {{ .Values.agentService.service.port }}
+      targetPort: {{ .Values.agentService.service.port }}
+{{- end }}
diff --git a/bin/k8s/templates/gateway-routes.yaml b/bin/k8s/templates/gateway-routes.yaml
@@ -119,6 +119,33 @@ spec:
           kind: Backend
           name: texera-dynamic-backend
 ---
+# Agent Service Route (a separate HTTPRoute so the BackendTrafficPolicy can
+# target it for consistent-hash routing). Covers REST and the
+# /api/agents/:id/react WebSocket. Longest-prefix matching gives /api/agents
+# priority over the /api catch-all in the static routes above.
+{{- if .Values.agentService.enabled }}
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: {{ .Release.Name }}-agent-service-route
+  namespace: {{ .Release.Namespace }}
+spec:
+  parentRefs:
+    - name: {{ .Release.Name }}-gateway
+  {{- if and .Values.gatewayConfig .Values.gatewayConfig.hostname }}
+  hostnames:
+    - {{ .Values.gatewayConfig.hostname }}
+  {{- end }}
+  rules:
+    - matches:
+        - path:
+            type: PathPrefix
+            value: /api/agents
+      backendRefs:
+        - name: {{ .Release.Name }}-{{ .Values.agentService.name }}-svc
+          port: {{ .Values.agentService.service.port }}
+{{- end }}
+---
 # MinIO Route
 {{- if .Values.minio.gateway.enabled }}
 apiVersion: gateway.networking.k8s.io/v1

diff --git a/bin/k8s/templates/litellm-config.yaml b/bin/k8s/templates/litellm-config.yaml
@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+{{- if .Values.litellm.enabled }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Release.Name }}-litellm-config
+  namespace: {{ .Release.Namespace }}
+data:
+  litellm-config.yaml: |
+{{ .Values.litellm.config | indent 4 }}
+{{- end }}
diff --git a/bin/k8s/templates/litellm-deployment.yaml b/bin/k8s/templates/litellm-deployment.yaml
@@ -0,0 +1,84 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+{{- if .Values.litellm.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Release.Name }}-{{ .Values.litellm.name }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: {{ .Release.Name }}-{{ .Values.litellm.name }}
+spec:
+  replicas: {{ .Values.litellm.numOfPods | default 1 }}
+  selector:
+    matchLabels:
+      app: {{ .Release.Name }}-{{ .Values.litellm.name }}
+  template:
+    metadata:
+      labels:
+        app: {{ .Release.Name }}-{{ .Values.litellm.name }}
+    spec:
+      containers:
+        - name: {{ .Values.litellm.name }}
+          image: {{ .Values.litellm.image }}
+          imagePullPolicy: Always
+          args:
+            - "--config"
+            - "/app/config/litellm-config.yaml"
+          ports:
+            - containerPort: {{ .Values.litellm.service.port }}
+          env:
+            - name: LITELLM_MASTER_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: {{ .Release.Name }}-agent-service-secret
+                  key: litellm-master-key
+            # Postgres persistence: LiteLLM runs its Prisma migrations against this
+            # database on startup and stores keys, spend, and (with STORE_MODEL_IN_DB)
+            # model config there, so state survives pod restarts.
+            - name: DATABASE_URL
+              value: postgresql://postgres:{{ .Values.postgresql.auth.postgresPassword }}@{{ .Release.Name }}-postgresql:5432/{{ .Values.litellm.databaseName }}
+            - name: STORE_MODEL_IN_DB
+              value: "{{ .Values.litellm.storeModelInDb }}"
+            {{- range $key, $value := .Values.litellm.providerApiKeys }}
+            - name: {{ $key }}
+              valueFrom:
+                secretKeyRef:
+                  name: {{ $.Release.Name }}-agent-service-secret
+                  key: {{ $key }}
+            {{- end }}
+          volumeMounts:
+            - name: litellm-config
+              mountPath: /app/config
+          livenessProbe:
+            httpGet:
+              path: /health/liveliness
+              port: {{ .Values.litellm.service.port }}
+            initialDelaySeconds: 30
+            periodSeconds: 10
+          readinessProbe:
+            httpGet:
+              path: /health/liveliness
+              port: {{ .Values.litellm.service.port }}
+            initialDelaySeconds: 15
+            periodSeconds: 5
+      volumes:
+        - name: litellm-config
+          configMap:
+            name: {{ .Release.Name }}-litellm-config
+{{- end }}
diff --git a/bin/k8s/templates/litellm-service.yaml b/bin/k8s/templates/litellm-service.yaml
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+{{- if .Values.litellm.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ .Release.Name }}-{{ .Values.litellm.name }}-svc
+  namespace: {{ .Release.Namespace }}
+spec:
+  type: {{ .Values.litellm.service.type }}
+  selector:
+    app: {{ .Release.Name }}-{{ .Values.litellm.name }}
+  ports:
+    - protocol: TCP
+      port: {{ .Values.litellm.service.port }}
+      targetPort: {{ .Values.litellm.service.port }}
+{{- end }}
diff --git a/bin/k8s/templates/postgresql-init-script-config.yaml b/bin/k8s/templates/postgresql-init-script-config.yaml
@@ -52,5 +52,12 @@ data:
 {{ .Files.Get "files/texera_ddl.sql" | indent 6 }}
     EOF
     psql -U postgres -f /tmp/texera_ddl.sql
+{{- if .Values.litellm.enabled }}
+
+    echo "Initializing LiteLLM database..."
+    # LiteLLM (Prisma) creates its own tables on startup but the database must exist.
+    psql -U postgres -tc "SELECT 1 FROM pg_database WHERE datname = '{{ .Values.litellm.databaseName }}'" | grep -q 1 \
+      || psql -U postgres -c "CREATE DATABASE {{ .Values.litellm.databaseName }}"
+{{- end }}
 
     echo "Schema initialization complete."