groups:
  - name: Keystone Health Checks
    rules:
      - alert: Database Connection
        expr: keystone_health_check_status{namespace="keystone", check="Database"} != 200
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Keystone database connectivity failure"
          description: "Keystone has been unable to reach its database for over 5 minutes."

      - alert: Redis Connection
        expr: keystone_health_check_status{namespace="keystone", check="Redis"} != 200
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Keystone Redis connectivity failure"
          description: "Keystone has been unable to reach its Redis cache for over 5 minutes."

      - alert: File Storage Connection
        expr: keystone_health_check_status{namespace="keystone", check="Storage"} != 200
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Keystone file storage access failure"
          description: "Keystone has been unable to access its file storage for over 5 minutes."

      - alert: SMTP Connection
        expr: keystone_health_check_status{namespace="keystone", check="SMTP"} != 200
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Keystone SMTP connectivity failure"
          description: "Keystone has been unable to reach the SMTP server for over 5 minutes."

      - alert: Celery Connection
        expr: keystone_health_check_status{namespace="keystone", check="Celery"} != 200
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Keystone Celery connectivity failure"
          description: "Keystone has been unable to reach the Celery scheduler for over 5 minutes."

      # Note: The ldap check will not be available and this alert will never fire if LDAP is disabled in the API.
      - alert: LDAP Connection
        expr: keystone_health_check_status{namespace="keystone", check="LDAP"} != 200
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Keystone LDAP connectivity failure"
          description: "Keystone has been unable to reach the LDAP server for over 5 minutes."

  - name: Keystone Database
    rules:
      - alert: High Database Connections
        expr: (sum(pg_stat_activity_count{namespace="keystone"}) / scalar(pg_settings_max_connections{namespace="keystone"})) > 0.8
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Keystone PostgreSQL connection usage is high"
          description: "Active PostgreSQL connections are exceeding 90% of the configured limit."

      - alert: Replication Lag
        expr: pg_replication_lag_seconds{namespace="keystone"} > 15
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Keystone PostgreSQL replication is lagging"
          description: "PostgreSQL replication lag has exceeded 15 seconds."

      - alert: Postgres Using Temporary Files
        expr: increase(pg_stat_database_temp_files{namespace="keystone", datname!~"template.*|postgres"}[5m]) > 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Keystone PostgreSQL is using temporary files"
          description: "PostgreSQL is creating temporary files, which may indicate inefficient work_mem settings."

      - alert: Too Many Locks Acquired
        expr: sum(pg_locks_count{namespace="keystone"}) / scalar(pg_settings_max_locks_per_transaction{namespace="keystone"} * pg_settings_max_connections{namespace="keystone"}) > 0.20
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Keystone Postgresql is allocating too many locks"
          description: "If this alert happens frequently, you may need to increase the max_locks_per_transaction setting."

  - name: Keystone Celery
    rules:
      - alert: High Queue Times
        expr: |
          avg(
            flower_task_prefetch_time_seconds{
              namespace="keystone-celery-metrics",
            }
          ) > 60
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High wait times in the Celery job queue"
          description: "The average Celery task queue time for worker {{ $labels.worker }} has exceeded 60 seconds for the over 5 minutes."

      - alert: Failing Celery Tasks
        expr: |
          sum(
            increase(
              flower_events_total{
                namespace="keystone-celery-metrics",
                type="task-failed"
              }[1d]
            )
          ) by (task) > 0
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "Failing Celery task {{ $labels.task }}"
          description: "Task '{{ $labels.task }}' has failed one or more times in the last 24 hours."