Improve observability

ansible/roles/monitoring/defaults/main.yml

@@ -9,6 +9,9 @@ monitoring_dir: /opt/guitar0-monitoring
 # Grafana
 monitoring_grafana_port: 3000
 
+# Docker group GID — needed for Prometheus Docker SD without running as root.
+docker_gid: "999"
+
 # Dashboards to deploy (filenames from observability/grafana/provisioning/dashboards/)
 monitoring_grafana_dashboards:
   - guitar0-backend.json

ansible/roles/monitoring/tasks/deploy.yml

@@ -3,6 +3,16 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
 ---
+- name: Stat docker.sock to get actual group GID
+  ansible.builtin.stat:
+    path: /var/run/docker.sock
+  register: docker_sock_stat
+
+- name: Set docker_gid from host docker.sock
+  ansible.builtin.set_fact:
+    docker_gid: "{{ docker_sock_stat.stat.gid }}"
+  when: docker_sock_stat.stat.exists
+
 - name: Render docker-compose for monitoring stack
   ansible.builtin.template:
     src: docker-compose.yml.j2

ansible/roles/monitoring/tasks/nginx.yml

@@ -8,6 +8,7 @@
     src: nginx-grafana.j2
     dest: /etc/nginx/sites-available/grafana
     mode: "0644"
+    force: false
   notify: Reload nginx
 
 - name: Enable Grafana nginx site

ansible/roles/monitoring/templates/docker-compose.yml.j2

@@ -11,7 +11,9 @@ services:
   prometheus:
     image: prom/prometheus:v3.1.0
     restart: unless-stopped
-    user: root
+    # Run as default nobody user (UID 65534), add to docker group for socket access.
+    group_add:
+      - "{{ docker_gid | default('999') }}"
     volumes:
       - {{ monitoring_dir }}/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
       - {{ monitoring_dir }}/prometheus/rules:/etc/prometheus/rules:ro

observability/.env.example

@@ -5,5 +5,12 @@
 # cp .env.example .env
 
 GRAFANA_ADMIN_PASSWORD=
+# Public URL of Grafana (used for share links and redirects after login)
+GRAFANA_ROOT_URL=http://localhost:3000
+
+# Docker group GID — run on the host to get the value:
+# stat -c '%g' /var/run/docker.sock
+DOCKER_GID=999
+
 TELEGRAM_BOT_TOKEN=
 TELEGRAM_CHAT_ID=

observability/alloy/config.alloy

@@ -5,7 +5,7 @@
 // Discover all running Docker containers
 discovery.docker "containers" {
   host             = "unix:///var/run/docker.sock"
-  refresh_interval = "5s"
+  refresh_interval = "30s"
 }
 
 // Filter to guitar0-backend project containers only and add labels

observability/docker-compose.monitoring.yml

@@ -5,14 +5,18 @@
 # Monitoring stack: Prometheus + Loki + Alloy + Grafana + Alertmanager
 # Requires external Docker network: docker network create guitar0-network
 # Requires .env file with: GRAFANA_ADMIN_PASSWORD, TELEGRAM_BOT_TOKEN, TELEGRAM_CHAT_ID
+# DOCKER_GID: run `stat -c '%g' /var/run/docker.sock` on the host to get the value
 
 name: guitar0-monitoring
 
 services:
   prometheus:
     image: prom/prometheus:v3.1.0
     restart: unless-stopped
-    user: root
+    # Run as default nobody user (UID 65534), add to docker group for socket access.
+    # Avoids running as root while retaining Docker SD capability.
+    group_add:
+      - "${DOCKER_GID:-999}"
     volumes:
       - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
       - ./prometheus/rules:/etc/prometheus/rules:ro
@@ -90,8 +94,9 @@ services:
     ports:
       - "127.0.0.1:3000:3000"
     environment:
+      GF_SECURITY_ADMIN_PASSWORD: "${GRAFANA_ADMIN_PASSWORD}"
       GF_USERS_ALLOW_SIGN_UP: "false"
-      GF_SERVER_ROOT_URL: http://localhost:3000
+      GF_SERVER_ROOT_URL: "${GRAFANA_ROOT_URL:-http://localhost:3000}"
       GF_ANALYTICS_REPORTING_ENABLED: "false"
       GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
     volumes:

observability/loki/config.yml

@@ -33,12 +33,12 @@ storage_config:
     directory: /loki/chunks
 
 limits_config:
-  retention_period: 720h  # 30 days
+  retention_period: 720h # 30 days
 
 compactor:
   working_directory: /loki/compactor
   compaction_interval: 10m
   retention_enabled: true
   retention_delete_delay: 2h
-  retention_delete_worker_count: 150
+  retention_delete_worker_count: 15
   delete_request_store: filesystem

observability/prometheus/rules/alerts.yml

@@ -24,11 +24,22 @@ groups:
           summary: "Guitar0 backend недоступен"
           description: "Backend не отвечает более 2 минут."
 
-      - alert: Guitar0High5xxErrorRate
-        expr: guitar0:http_error_rate_5xx:ratio5m > 0.05
+      - alert: Guitar0High5xxErrorRateProduction
+        expr: guitar0:http_error_rate_5xx:ratio5m{env="production"} > 0.05
         for: 5m
         labels:
           severity: critical
+          env: production
+        annotations:
+          summary: "Высокий уровень 5xx ошибок"
+          description: "5xx error rate: {{ $value | humanizePercentage }} (порог 5%)"
+
+      - alert: Guitar0High5xxErrorRateStaging
+        expr: guitar0:http_error_rate_5xx:ratio5m{env="staging"} > 0.05
+        for: 5m
+        labels:
+          severity: warning
+          env: staging
         annotations:
           summary: "Высокий уровень 5xx ошибок"
           description: "5xx error rate: {{ $value | humanizePercentage }} (порог 5%)"
@@ -53,11 +64,22 @@ groups:
           summary: "Высокий уровень 4xx ошибок"
           description: "4xx error rate: {{ $value | humanizePercentage }} (порог 90%)"
 
-      - alert: Guitar0HighLatency
-        expr: guitar0:http_request_duration_p99:5m > 1
+      - alert: Guitar0HighLatencyProduction
+        expr: guitar0:http_request_duration_p99:5m{env="production"} > 1
         for: 5m
         labels:
           severity: warning
+          env: production
+        annotations:
+          summary: "Высокая латентность P99"
+          description: "P99 latency: {{ humanizeDuration $value }} (порог 1s)"
+
+      - alert: Guitar0HighLatencyStaging
+        expr: guitar0:http_request_duration_p99:5m{env="staging"} > 1
+        for: 5m
+        labels:
+          severity: warning
+          env: staging
         annotations:
           summary: "Высокая латентность P99"
           description: "P99 latency: {{ humanizeDuration $value }} (порог 1s)"

observability/prometheus/rules/recording.yml

@@ -24,17 +24,17 @@ groups:
       - record: guitar0:http_error_rate:ratio5m
         expr: >-
           sum by (container, env) (rate(guitar0_backend_http_requests_total{status_code=~"[45].."}[5m]))
-          / sum by (container, env) (rate(guitar0_backend_http_requests_total[5m]))
+          / (sum by (container, env) (rate(guitar0_backend_http_requests_total[5m])) > 0)
 
       - record: guitar0:http_error_rate_4xx:ratio5m
         expr: >-
           sum by (container, env) (rate(guitar0_backend_http_requests_total{status_code=~"4.."}[5m]))
-          / sum by (container, env) (rate(guitar0_backend_http_requests_total[5m]))
+          / (sum by (container, env) (rate(guitar0_backend_http_requests_total[5m])) > 0)
 
       - record: guitar0:http_error_rate_5xx:ratio5m
         expr: >-
           sum by (container, env) (rate(guitar0_backend_http_requests_total{status_code=~"5.."}[5m]))
-          / sum by (container, env) (rate(guitar0_backend_http_requests_total[5m]))
+          / (sum by (container, env) (rate(guitar0_backend_http_requests_total[5m])) > 0)
 
       # ── Latency percentiles (histogram_quantile — самые тяжёлые) ─────────────