Fix — Trigger sync via Admin UI or API

Go to LiteMaaS Frontend → Admin → Models and verify the model appears. If missing, add it via the UI or use the API:

export ADMIN_KEY=sk-1234567890abcdef1234 && export LITELLM_URL=https://litellm-prod.apps.maas.redhatworkshops.io

curl -sk -X POST "$LITELLM_URL/model/new" -H "Authorization: Bearer $ADMIN_KEY" -H "Content-Type: application/json" -d '{"model_name":"my-model","litellm_params":{"model":"openai/my-model","api_base":"http://my-model-predictor.llm-hosting.svc.cluster.local/v1","custom_llm_provider":"openai"}}'

Fix — Via Admin API Sync Endpoint

# If the backend exposes a sync endpoint:
ADMIN_KEY=$(oc get secret backend-secret -n $NS \
  -o jsonpath='{.data.ADMIN_API_KEY}' | base64 -d)
BACKEND_URL=$(oc get route litellm-prod-admin -n $NS \
  -o jsonpath='https://{.spec.host}')

curl -X POST "${BACKEND_URL}/api/admin/sync-models" \
  -H "Authorization: Bearer ${ADMIN_KEY}" \
  -H "Content-Type: application/json"

Fix — Enable Auto-Sync for Ongoing Changes

oc set env deployment/litellm-backend LITELLM_AUTO_SYNC=true -n $NS
oc rollout restart deployment/litellm-backend -n $NS

Verify

# Model should now appear in backend DB
oc exec -n $NS litellm-postgres-0 -- \
  psql -U litellm -d litellm -c "SELECT id, name FROM models;" | grep your-model-name

Diagnose

# Check if model exists in LiteMaaS models table
NS=litellm-rhpds
MODEL_ID=your-model-name

oc exec -n $NS litellm-postgres-0 -- \
  psql -U litellm -d litellm -c \
  "SELECT id, name, provider FROM models WHERE id = '${MODEL_ID}';"
# If 0 rows returned, the model is not in LiteMaaS DB

# Check LiteLLM has it
LITELLM_KEY=$(oc get secret litellm-secret -n $NS \
  -o jsonpath='{.data.LITELLM_MASTER_KEY}' | base64 -d)
ROUTE=$(oc get route litellm-prod -n $NS -o jsonpath='{.spec.host}')
curl -s "https://$ROUTE/model/info" \
  -H "Authorization: Bearer $LITELLM_KEY" | \
  jq ".data[] | select(.model_name == \"${MODEL_ID}\")"

Fix — Add model via Admin UI or API

Go to LiteMaaS Frontend → Admin → Models → Add Model, or use the API:

Diagnose

# Check both the main container and init container image tags
NS=litellm-rhpds

oc get deployment/litellm-frontend -n $NS \
  -o jsonpath='{.spec.template.spec.containers[*].image}'
echo ""
oc get deployment/litellm-frontend -n $NS \
  -o jsonpath='{.spec.template.spec.initContainers[*].image}'
echo ""

# They must match for the correct version to be served

Fix — Update Both Container Images Together

NS=litellm-rhpds
NEW_TAG=0.5.0
IMAGE=quay.io/rh-aiservices-bu/litemaas-frontend

# Update the main container
oc set image deployment/litellm-frontend \
  frontend=${IMAGE}:${NEW_TAG} \
  -n $NS

# Update the init container (if present)
oc set image deployment/litellm-frontend \
  init-frontend=${IMAGE}:${NEW_TAG} \
  -n $NS

# Force a rollout to apply changes
oc rollout restart deployment/litellm-frontend -n $NS
oc rollout status deployment/litellm-frontend -n $NS

# Verify running pod is using new image
oc get pods -n $NS -l app=litellm-frontend \
  -o jsonpath='{.items[0].spec.initContainers[*].image}'

Prevention

Always update init container and main container to the same tag simultaneously. When using Ansible, the role handles both; when patching manually, update both in a single oc set image call or via a single deployment patch.

Diagnose

# Check LiteLLM startup logs for migration output
NS=litellm-rhpds
LITELLM_POD=$(oc get pods -n $NS -l app=litellm \
  -o jsonpath='{.items[0].metadata.name}')

oc logs -n $NS $LITELLM_POD | grep -iE "migrat|prisma|error|schema"

# Check the PostgreSQL migration history table
oc exec -n $NS litellm-postgres-0 -- \
  psql -U litellm -d litellm -c \
  "SELECT * FROM \"_prisma_migrations\" ORDER BY finished_at DESC LIMIT 10;"

Fix — Trigger Manual Migration

# Scale down LiteLLM to 1 replica to avoid concurrent migration attempts
oc scale deployment/litellm --replicas=1 -n $NS

# Wait for the single pod to start
oc rollout status deployment/litellm -n $NS

# Check if migration ran successfully
oc logs -n $NS deployment/litellm --tail=100 | grep -i migrat

# If migration still fails, exec into the pod and run it manually
LITELLM_POD=$(oc get pods -n $NS -l app=litellm \
  -o jsonpath='{.items[0].metadata.name}')

oc exec -n $NS $LITELLM_POD -- \
  python -c "
import litellm
from litellm.proxy.proxy_server import ProxyStartupEvent
import asyncio
asyncio.run(ProxyStartupEvent.run_migrations())
"

# Scale back to 3 replicas after migration succeeds
oc scale deployment/litellm --replicas=3 -n $NS

Fix — Reset Migration State (Last Resort)

# Mark a failed migration as applied (replace with actual migration name from logs)
oc exec -n $NS litellm-postgres-0 -- \
  psql -U litellm -d litellm -c \
  "UPDATE \"_prisma_migrations\" \
   SET finished_at = NOW(), applied_steps_count = 1 \
   WHERE migration_name = '20240101000000_add_some_column' \
   AND finished_at IS NULL;"

# Restart LiteLLM
oc rollout restart deployment/litellm -n $NS

Diagnose — Find Keys with Sync Errors

NS=litellm-rhpds

# List all keys with sync errors
oc exec -n $NS litellm-postgres-0 -- \
  psql -U litellm -d litellm -c \
  "SELECT id, user_id, litellm_key_alias, is_active, sync_status, sync_error
   FROM api_keys
   WHERE sync_status = 'error' OR sync_status = 'pending'
   ORDER BY updated_at DESC
   LIMIT 20;"

# Find LiteMaaS-active keys that don't exist in LiteLLM
oc exec -n $NS litellm-postgres-0 -- \
  psql -U litellm -d litellm -c \
  "SELECT ak.id, ak.litellm_key_alias, ak.is_active
   FROM api_keys ak
   WHERE ak.is_active = true
     AND ak.litellm_key_alias IS NOT NULL
     AND NOT EXISTS (
       SELECT 1 FROM \"LiteLLM_VerificationToken\" lv
       WHERE lv.key_alias = ak.litellm_key_alias
     );"

Fix — Mark Orphaned LiteMaaS Keys as Inactive

# Mark all LiteMaaS keys inactive if they have no LiteLLM counterpart
oc exec -n $NS litellm-postgres-0 -- \
  psql -U litellm -d litellm -c \
  "UPDATE api_keys
   SET is_active = false,
       revoked_at = NOW(),
       sync_status = 'error',
       sync_error = 'Key not found in LiteLLM - manual cleanup',
       updated_at = NOW()
   WHERE is_active = true
     AND litellm_key_alias IS NOT NULL
     AND NOT EXISTS (
       SELECT 1 FROM \"LiteLLM_VerificationToken\" lv
       WHERE lv.key_alias = api_keys.litellm_key_alias
     );"

Fix — Re-sync a Specific Key

# If a key exists in LiteLLM but not tracked in LiteMaaS,
# the user should revoke the old key and create a new one via the portal.
# The new key creation will go through the proper sync flow.

# Alternatively, delete the orphaned LiteLLM key:
LITELLM_KEY=$(oc get secret litellm-secret -n $NS \
  -o jsonpath='{.data.LITELLM_MASTER_KEY}' | base64 -d)
ROUTE=$(oc get route litellm-prod -n $NS -o jsonpath='{.spec.host}')

curl -X POST "https://$ROUTE/key/delete" \
  -H "Authorization: Bearer $LITELLM_KEY" \
  -H "Content-Type: application/json" \
  -d '{"keys": ["sk-the-orphaned-key-token"]}'

Fix — Run the Key Cleanup Cronjob Immediately

# The cleanup cron handles orphaned keys automatically
# Run it manually for immediate effect
sudo /usr/local/bin/cleanup-litemaas-keys-litellm-rhpds.sh
sudo tail -50 /var/log/litemaas-key-cleanup.log

Diagnose

NS=litellm-rhpds

# Check Redis is running
oc get pods -n $NS -l app=litellm-redis

# Check Redis memory usage and key count
REDIS_POD=$(oc get pods -n $NS -l app=litellm-redis -o name | head -1)
oc exec -n $NS $REDIS_POD -- redis-cli INFO memory | grep used_memory_human
oc exec -n $NS $REDIS_POD -- redis-cli DBSIZE

# Check if LiteLLM pods can reach Redis
LITELLM_POD=$(oc get pods -n $NS -l app=litellm \
  -o jsonpath='{.items[0].metadata.name}')
oc exec -n $NS $LITELLM_POD -- \
  sh -c 'echo REDIS_HOST=$REDIS_HOST REDIS_PORT=$REDIS_PORT'

Fix — Flush the Redis Cache

# Flush all Redis keys (forces LiteLLM pods to reload from DB)
# WARNING: This briefly increases DB load as all pods reload their caches
REDIS_POD=$(oc get pods -n $NS -l app=litellm-redis -o name | head -1)
oc exec -n $NS $REDIS_POD -- redis-cli FLUSHALL

# Verify flush
oc exec -n $NS $REDIS_POD -- redis-cli DBSIZE
# Should return: 0

Fix — Restart LiteLLM Pods (Forces Cache Reload)

# Rolling restart — no downtime
oc rollout restart deployment/litellm -n $NS
oc rollout status deployment/litellm -n $NS

Fix — Redis Pod Down (Restart Redis)

# If Redis pod is in error state
oc delete pod -n $NS -l app=litellm-redis
# The deployment controller will create a new pod automatically

# If Redis deployment is misconfigured, check logs
oc logs -n $NS deployment/litellm-redis --tail=50

Diagnose — Check OAuthClient Redirect URIs

NS=litellm-rhpds

# Get the OAuthClient configuration
oc get oauthclient $NS -o yaml | grep -A 20 redirectURIs

# Get current route hostnames
oc get routes -n $NS -o jsonpath='{range .items[*]}{.spec.host}{"\n"}{end}'

# The OAuthClient redirectURIs must include:
# https://<api-route>/api/auth/callback
# https://<frontend-route>/api/auth/callback

Fix — Update OAuthClient Redirect URIs

NS=litellm-rhpds
API_ROUTE=$(oc get route litellm-prod -n $NS -o jsonpath='{.spec.host}')
FRONTEND_ROUTE=$(oc get route litellm-prod-frontend -n $NS -o jsonpath='{.spec.host}')

# Patch the OAuthClient
oc patch oauthclient $NS --type=merge -p \
  "{\"redirectURIs\": [
    \"https://${API_ROUTE}/api/auth/callback\",
    \"https://${FRONTEND_ROUTE}/api/auth/callback\"
  ]}"

# Verify
oc get oauthclient $NS -o jsonpath='{.redirectURIs}'

Fix — User oauth_id Mismatch (after migration or manual insert)

# LiteMaaS v0.2.1+ has email fallback: it looks up by email if oauth_id doesn't match
# and auto-updates the oauth_id. No manual fix needed for v0.2.1+.
# For older versions, update oauth_id manually:

# Get the OpenShift user's UID (this is the oauth_id)
oc get user user@redhat.com -o jsonpath='{.metadata.uid}'

# Update the oauth_id in LiteMaaS DB
oc exec -n $NS litellm-postgres-0 -- \
  psql -U litellm -d litellm -c \
  "UPDATE users SET oauth_id = 'the-openshift-user-uid-here' \
   WHERE email = 'user@redhat.com';"

Fix — Set HAProxy Timeout on Routes

NS=litellm-rhpds

# Add 600-second timeout annotation to the LiteLLM API route
oc annotate route litellm-prod -n $NS \
  haproxy.router.openshift.io/timeout=600s --overwrite

# Also apply to backend and frontend routes if needed
oc annotate route litellm-prod-admin -n $NS \
  haproxy.router.openshift.io/timeout=600s --overwrite
oc annotate route litellm-prod-frontend -n $NS \
  haproxy.router.openshift.io/timeout=600s --overwrite

# Verify
oc get route litellm-prod -n $NS \
  -o jsonpath='{.metadata.annotations.haproxy\.router\.openshift\.io/timeout}'

Diagnose

NS=litellm-rhpds
LITELLM_KEY=$(oc get secret litellm-secret -n $NS \
  -o jsonpath='{.data.LITELLM_MASTER_KEY}' | base64 -d)
ROUTE=$(oc get route litellm-prod -n $NS -o jsonpath='{.spec.host}')

# Check if the key exists in LiteLLM
curl "https://$ROUTE/key/info?key=sk-the-user-key-here" \
  -H "Authorization: Bearer $LITELLM_KEY" | jq '.'

# Check key in LiteMaaS DB
oc exec -n $NS litellm-postgres-0 -- \
  psql -U litellm -d litellm -c \
  "SELECT id, is_active, sync_status, revoked_at, litellm_key_alias
   FROM api_keys
   WHERE litellm_key_alias = 'key-alias-here';"

Fix — Key Budget Exhausted

# Check if the key hit its budget limit
curl "https://$ROUTE/key/info?key=sk-the-user-key-here" \
  -H "Authorization: Bearer $LITELLM_KEY" | \
  jq '.info | {max_budget, spend, budget_reset_at}'

# If spent >= max_budget, reset the spend or increase the budget
curl -X POST "https://$ROUTE/key/update" \
  -H "Authorization: Bearer $LITELLM_KEY" \
  -H "Content-Type: application/json" \
  -d '{"key": "sk-the-user-key-here", "max_budget": 200}'

Fix — Redis Cache Has Stale Key State

# Flush Redis so LiteLLM reloads all key data from PostgreSQL
REDIS_POD=$(oc get pods -n $NS -l app=litellm-redis -o name | head -1)
oc exec -n $NS $REDIS_POD -- redis-cli FLUSHALL

Diagnose

NS=litellm-rhpds

# Check OOM history
oc describe pods -n $NS -l app=litellm | grep -A 5 "Last State\|OOMKilled"

# Check current resource limits
oc get deployment/litellm -n $NS \
  -o jsonpath='{.spec.template.spec.containers[0].resources}' | jq .

Fix — Increase Memory Limits

# Increase LiteLLM memory limit to 4Gi
oc set resources deployment/litellm \
  --limits=memory=4Gi,cpu=2000m \
  --requests=memory=1Gi,cpu=500m \
  -n $NS

# Or patch directly
oc patch deployment/litellm -n $NS --type=json -p='[
  {
    "op": "replace",
    "path": "/spec/template/spec/containers/0/resources/limits/memory",
    "value": "4Gi"
  }
]'

oc rollout status deployment/litellm -n $NS

Root Cause

LiteLLM requires the DATABASE_URL environment variable to persist state to PostgreSQL. Without it, LiteLLM uses in-memory storage — all models, keys, and configuration exist only in the running pod and are lost the moment the pod restarts. This was a common issue in deployments before v0.4.0 where DATABASE_URL was not set automatically.

Diagnose

# Check if DATABASE_URL is set on the LiteLLM pod
oc exec -n litellm-rhpds deployment/litellm -- sh -c "echo \$DATABASE_URL"

# If empty or not set — this is the cause
# Models and keys are stored in memory only and lost on restart

Fix — Set DATABASE_URL

# Get the PostgreSQL connection string from the DB secret
DB_URL=$(oc get secret litemaas-db -n litellm-rhpds \
  -o jsonpath='{.data.DATABASE_URL}' | base64 -d)

# Set it on the LiteLLM deployment
oc set env deployment/litellm -n litellm-rhpds DATABASE_URL="$DB_URL"

oc rollout status deployment/litellm -n litellm-rhpds

After fixing — re-register models

The model-serving ArgoCD app will re-create InferenceServices, but LiteLLM model registrations must be re-added via the Admin UI → Models → Add Model or the API. Then restart the backend to trigger sync:

oc rollout restart deployment/litellm-backend -n litellm-rhpds

Diagnose

# Find pending predictor pods
oc get pods -n llm-hosting --field-selector=status.phase=Pending

# Check the scheduling error
oc describe pod <pending-pod-name> -n llm-hosting | grep -A5 "Events:"

# Check GPU capacity vs allocation per node
oc get nodes -l 'node-role.kubernetes.io/worker-gpu' \
  -o custom-columns="NAME:.metadata.name,INSTANCE:.metadata.labels['node\.kubernetes\.io/instance-type'],GPU:.status.capacity['nvidia\.com/gpu']"

# See which pods are using GPUs on a specific node
oc describe node <node-name> | grep -A20 "Allocated resources"

Options

# Option 1: Scale down another model on the same node type to free a GPU slot
oc patch inferenceservice <other-model> -n llm-hosting \
  --type merge -p '{"spec":{"predictor":{"maxReplicas":0,"minReplicas":0}}}'

# Option 2: Change the node selector to a different instance type with free GPUs
# Edit the InferenceService in the models-aas GitOps repo and update nodeSelector

# Option 3: Add more GPU nodes (requires cluster admin / AWS console)

Diagnose

MASTER_KEY=$(oc get secret litellm-secret -n litellm-rhpds \
  -o jsonpath='{.data.LITELLM_MASTER_KEY}' | base64 -d)

# List all models and spot duplicates
curl -s https://litellm-prod.apps.maas.redhatworkshops.io/model/info \
  -H "Authorization: Bearer $MASTER_KEY" | \
  python3 -c "
import sys,json
d=json.load(sys.stdin)
names=[m.get('model_name') for m in d.get('data',[])]
dups=[n for n in names if names.count(n)>1]
print('Duplicates:', set(dups))
for m in d.get('data',[]):
    if m.get('model_name') in dups:
        print(m.get('model_name'), m.get('model_info',{}).get('id'))
"

Fix

# Delete the duplicate by its model_info.id
curl -X DELETE https://litellm-prod.apps.maas.redhatworkshops.io/model/delete \
  -H "Authorization: Bearer $MASTER_KEY" \
  -H "Content-Type: application/json" \
  -d '{"id": "<model-info-id-to-remove>"}'

# Verify it is removed
curl -s https://litellm-prod.apps.maas.redhatworkshops.io/model/info \
  -H "Authorization: Bearer $MASTER_KEY" | \
  python3 -c "import sys,json; [print(m['model_name']) for m in json.load(sys.stdin).get('data',[])]"