{
  "id": 4409321175638016,
  "panels": [
    {
      "datasource": {
        "type": "prometheus",
        "uid": "PBFA97CFB590B2093"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "reqps"
        }
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 0
      },
      "id": 1,
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "PBFA97CFB590B2093"
          },
          "expr": "rate(litellm_proxy_total_requests_metric_total[5m])",
          "legendFormat": "{{model}} {{api_provider}}",
          "refId": "A"
        }
      ],
      "title": "Requests / sec",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "PBFA97CFB590B2093"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "fixedColor": "red",
            "mode": "fixed"
          },
          "unit": "reqps"
        }
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 0
      },
      "id": 2,
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "PBFA97CFB590B2093"
          },
          "expr": "rate(litellm_proxy_failed_requests_metric_total[5m])",
          "legendFormat": "{{model}}",
          "refId": "A"
        }
      ],
      "title": "Error Rate",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "PBFA97CFB590B2093"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "s"
        }
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 8
      },
      "id": 3,
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "PBFA97CFB590B2093"
          },
          "expr": "histogram_quantile(0.95, rate(litellm_request_total_latency_metric_bucket[5m]))",
          "legendFormat": "p95 {{model}}",
          "refId": "A"
        }
      ],
      "title": "Request Latency p95 (s)",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "PBFA97CFB590B2093"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "s"
        }
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 8
      },
      "id": 4,
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "PBFA97CFB590B2093"
          },
          "expr": "histogram_quantile(0.95, rate(litellm_llm_api_latency_metric_bucket[5m]))",
          "legendFormat": "p95 {{model}}",
          "refId": "A"
        }
      ],
      "title": "LLM API Latency p95 (s)",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "PBFA97CFB590B2093"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "short"
        }
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 16
      },
      "id": 5,
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "PBFA97CFB590B2093"
          },
          "expr": "rate(litellm_input_tokens_metric_total[5m])",
          "legendFormat": "input {{model}}",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "PBFA97CFB590B2093"
          },
          "expr": "rate(litellm_output_tokens_metric_total[5m])",
          "legendFormat": "output {{model}}",
          "refId": "B"
        }
      ],
      "title": "Tokens / sec",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "PBFA97CFB590B2093"
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 16
      },
      "id": 6,
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "PBFA97CFB590B2093"
          },
          "expr": "sum by (model) (litellm_total_tokens_metric_total)",
          "instant": true,
          "legendFormat": "{{model}}",
          "refId": "A"
        }
      ],
      "title": "Total Tokens by Model",
      "type": "piechart"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "PBFA97CFB590B2093"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "currencyUSD"
        }
      },
      "gridPos": {
        "h": 4,
        "w": 6,
        "x": 0,
        "y": 24
      },
      "id": 7,
      "options": {
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "PBFA97CFB590B2093"
          },
          "expr": "sum(litellm_spend_metric_total)",
          "instant": true,
          "refId": "A"
        }
      ],
      "title": "Cumulative Spend ($)",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "PBFA97CFB590B2093"
      },
      "fieldConfig": {
        "defaults": {
          "unit": "s"
        }
      },
      "gridPos": {
        "h": 4,
        "w": 6,
        "x": 6,
        "y": 24
      },
      "id": 8,
      "options": {
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ]
        }
      },
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "PBFA97CFB590B2093"
          },
          "expr": "histogram_quantile(0.95, rate(litellm_llm_api_time_to_first_token_metric_bucket[5m]))",
          "instant": true,
          "refId": "A"
        }
      ],
      "title": "Time to First Token p95 (s)",
      "type": "stat"
    }
  ],
  "refresh": "1m",
  "schemaVersion": 38,
  "tags": [
    "litellm",
    "ai"
  ],
  "timezone": "browser",
  "title": "LiteLLM",
  "uid": "d99330ba-28ab-4544-9a9a-0170d87aecc8",
  "version": 1
}