Monitoring Server

This server was set up to provide monitoring for many systems using Prometheus and Grafana.

Installation

1. Submitted the following ticket to have IS set up the VM

   - Request_Submitted_By: kgomes@mbari.org
    - VM_Name: monitoring
    - VM_Purpose: I am working with Travis on setting up a monitoring server 
      that will host Prometheus and Grafana. This server will be a central monitoring 
      service for various applications that are slated to be built as well as 
      upgrades to existing infrastructure
    - VM_Expiration: 5 years
    - VM_Support: IS_Supported
    - VM_Support_Alt_Adm: 
    - VM_OS: Ubuntu 24.04 LTS
    - CPU: 2
    - CPU_Addl_Reason: 
    - RAM: 4
    - RAM_Addl_Reason: 
    - GPU_Extra: 
    - GPU: NO
    - Disk_Extra: 
    - Network: (SHORE) Internal
    - Resource_Priority: Low
    - Resource_Priority_Reason: 
    - Conf_Firewall: 
    - Conf_Share_Access: 
    - Conf_vCenter_Access: 
    - Conf_Desktop_Extra: 
    - Conf_Desktop: NO
    - Conf_Logins: ad+nfshome
    - Conf_Docker: YES
    - Conf_Docker_Extra: 
    - Conf_WebServer: nginx-docker
    - Conf_ACME_SSLcert: YES
    - Conf_ACME_SSLcert_Reload: 
    - Conf_sudo: kgomes and tbaraki
    - VM_Comments: Hey Peter,
   
   I'm thinking I will use Docker compose for this and it would be good 
   to be able to run this stack as a shared account so Travis and I can 
   both control it (or whatever way works best that allows us both to 
   control it). So, I'm thinking I will just include Nginx in my docker 
   compose and then have your put SSL certs somewhere that I can mount 
   them in the containers.
   
   I don't know if there are any real heavy disk requirements yet, that 
   will be something we figure out as we go.
   
   I am sure I'm forgetting something, but let me know if you need something else.
   

2. And Peter set it up and responded:

   The VM has been created with all that you specified. Your and Travis's 
   users have full sudo permissions. The ACME SSL certs are located in `/etc/ssl/acme-certs/`.
   
   Take a look around and let me know if anything is missing.
   

3. I created the necessary directories using sudo mkdir -p /opt/monitoring/{nginx,prometheus/data,grafana/data}
4. Set ownership using sudo chown -R kgomes:docker /opt/monitoring
5. Set permissions usig sudo chmod -R 2775 /opt/monitoring
6. Next, I created the /opt/monitoring/docker-compose.yml file:

   services:
   
     prometheus:
       image: prom/prometheus:latest
       restart: unless-stopped
       command:
         - "--config.file=/etc/prometheus/prometheus.yml"
         - "--storage.tsdb.path=/prometheus"
         - "--storage.tsdb.retention.time=30d"
         - "--web.external-url=/prometheus"
         - "--web.route-prefix=/prometheus"
       volumes:
         - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
         - ./prometheus/data:/prometheus
       expose:
         - "9090"
       networks:
         - internal
   
     grafana:
       image: grafana/grafana:latest
       restart: unless-stopped
       environment:
         GF_SERVER_ROOT_URL: "https://monitoring.shore.mbari.org/grafana"
         GF_SERVER_SERVE_FROM_SUB_PATH: "true"
         GF_SECURITY_ADMIN_USER: admin
         GF_SECURITY_ADMIN_PASSWORD: xxxxxxxxxxxxx
         GF_USERS_ALLOW_SIGN_UP: "false"
       volumes:
         - ./grafana/data:/var/lib/grafana
       expose:
         - "3000"
       networks:
         - internal
   
     nginx:
       image: nginx:alpine
       restart: unless-stopped
       ports:
         - "80:80"
         - "443:443"
       volumes:
         - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro
         - /etc/ssl/acme-certs:/etc/ssl/acme-certs:ro
       depends_on:
         - prometheus
         - grafana
       networks:
         - internal
   
   networks:
     internal:
       driver: bridge
   

7. Then I created the /opt/monitoring/nginx/nginx.conf file:

   events {
       worker_connections 1024;
   }
   
   http {
   
       upstream prometheus {
           server prometheus:9090;
       }
   
       upstream grafana {
           server grafana:3000;
       }
   
       # Redirect HTTP → HTTPS
       server {
           listen 80;
           server_name monitoring.shore.mbari.org;
           return 301 https://$host$request_uri;
       }
   
       server {
           listen 443 ssl;
           server_name monitoring.shore.mbari.org;
   
           ssl_certificate     /etc/ssl/acme-certs/monitoring.shore.mbari.org-chain.crt;
           ssl_certificate_key /etc/ssl/acme-certs/monitoring.shore.mbari.org.key;
           ssl_protocols       TLSv1.2 TLSv1.3;
           ssl_ciphers         HIGH:!aNULL:!MD5;
           ssl_session_cache   shared:SSL:10m;
           ssl_session_timeout 10m;
   
           # Prometheus — available at /prometheus
           location /prometheus {
               proxy_pass         http://prometheus;
               proxy_set_header   Host              $host;
               proxy_set_header   X-Real-IP         $remote_addr;
               proxy_set_header   X-Forwarded-For   $proxy_add_x_forwarded_for;
               proxy_set_header   X-Forwarded-Proto $scheme;
           }
   
           # Grafana — available at /grafana
           location /grafana {
               proxy_pass         http://grafana;
               proxy_set_header   Host              $host;
               proxy_set_header   X-Real-IP         $remote_addr;
               proxy_set_header   X-Forwarded-For   $proxy_add_x_forwarded_for;
               proxy_set_header   X-Forwarded-Proto $scheme;
   
               # WebSocket support — Grafana Live uses it
               proxy_http_version 1.1;
               proxy_set_header   Upgrade    $http_upgrade;
               proxy_set_header   Connection "upgrade";
           }
   
           # Optional: redirect bare / to Grafana
           location = / {
               return 302 /grafana;
           }
       }
   }
   

8. Then, I create the /opt/monitoring/prometheus/prometheus.yml file:

   global:
     scrape_interval: 15s
     evaluation_interval: 15s
   
   scrape_configs:
   
     # Prometheus scraping itself
     - job_name: prometheus
       metrics_path: /prometheus/metrics
       static_configs:
         - targets: ["localhost:9090"]
   
     # Prefect Server metrics
     - job_name: prefect
       metrics_path: /api/metrics
       scheme: https
       tls_config:
         insecure_skip_verify: false   # set true only if certs are self-signed
       static_configs:
         - targets: ["prefect.shore.mbari.org"]
       # If you want to scrape more frequently for Prefect:
       scrape_interval: 30s
   

9. I created the /etc/systemd/system/monitoring.service file:

   [Unit]
   Description=Monitoring Stack (Prometheus + Grafana)
   After=docker.service
   Requires=docker.service
   
   [Service]
   Type=oneshot
   RemainAfterExit=yes
   User=kgomes
   Group=docker
   WorkingDirectory=/opt/monitoring
   
   ExecStart=/usr/bin/docker compose up -d
   ExecStop=/usr/bin/docker compose down
   
   [Install]
   WantedBy=multi-user.target
   

10. Then, enabled and started it all with:

    sudo systemctl daemon-reload
    sudo systemctl enable monitoring
    sudo systemctl start monitoring
    

11. It failed to start the first time and I had to adjust permission on the opt files using sudo chown -R 65534:65534 /opt/monitoring/prometheus/data and sudo chown -R 472:472 /opt/monitoring/grafana/data.
12. All is up and running and you can find Prometheus at https://monitoring.shore.mbari.org/prometheus and Grafana on https://monitoring.shore.mbari.org/grafana.
13. Lastly, I had to wire Prometheus up to Grafana:
    1. Logged into Granfana using the credentials from the docker-compose file
    2. Went to Connections → Data Sources → Add data source → Prometheus
    3. Set the URL to http://prometheus:9090/prometheus (used the internal Docker network name)
    4. Clicked Save & Test and got green check mark
    5. Then imported a dashboard:
    6. Dashboards → Import → ID 19105 (a solid general Prometheus dashboard)
       1. For Prefect specifically there isn't an official Grafana dashboard ID yet, but you can build panels querying metrics like:
          1. prefect_server_flow_runs_total — flow run counts by state
          2. prefect_server_subflow_runs_total
          3. prefect_server_work_queues_total
          4. prefect_server_deployments_total
    7. Run curl https://prefect.shore.mbari.org/api/metrics from the monitoring server to see the full list of available metrics once Prefect is being scraped.
    8. Just to get started, I imported the following JSON to get a basic Prefect dashboard running:

           {
         "__inputs": [
           {
             "name": "DS_PROMETHEUS",
             "label": "Prometheus",
             "description": "",
             "type": "datasource",
             "pluginId": "prometheus",
             "pluginName": "Prometheus"
           }
         ],
         "__requires": [
           {
             "type": "grafana",
             "id": "grafana",
             "name": "Grafana",
             "version": "10.0.0"
           },
           {
             "type": "datasource",
             "id": "prometheus",
             "name": "Prometheus",
             "version": "1.0.0"
           },
           {
             "type": "panel",
             "id": "stat",
             "name": "Stat",
             "version": ""
           },
           {
             "type": "panel",
             "id": "timeseries",
             "name": "Time series",
             "version": ""
           },
           {
             "type": "panel",
             "id": "gauge",
             "name": "Gauge",
             "version": ""
           }
         ],
         "annotations": {
           "list": []
         },
         "description": "Prefect Server Health Dashboard",
         "editable": true,
         "fiscalYearStartMonth": 0,
         "graphTooltip": 1,
         "id": null,
         "links": [],
         "panels": [
       
           {
             "collapsed": false,
             "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 },
             "id": 100,
             "title": "Process Health",
             "type": "row"
           },
       
           {
             "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
             "fieldConfig": {
               "defaults": {
                 "color": { "mode": "thresholds" },
                 "thresholds": {
                   "mode": "absolute",
                   "steps": [
                     { "color": "red", "value": null },
                     { "color": "green", "value": 1 }
                   ]
                 },
                 "mappings": [
                   { "options": { "0": { "text": "Down" }, "1": { "text": "Up" } }, "type": "value" }
                 ]
               }
             },
             "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 },
             "id": 1,
             "options": { "colorMode": "background", "graphMode": "none", "textMode": "auto" },
             "title": "Prefect Server Status",
             "type": "stat",
             "targets": [
               {
                 "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
                 "expr": "up{job=\"prefect\"}",
                 "legendFormat": "Status"
               }
             ]
           },
       
           {
             "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
             "fieldConfig": {
               "defaults": {
                 "color": { "mode": "thresholds" },
                 "unit": "bytes",
                 "thresholds": {
                   "mode": "absolute",
                   "steps": [
                     { "color": "green", "value": null },
                     { "color": "yellow", "value": 500000000 },
                     { "color": "red", "value": 1000000000 }
                   ]
                 }
               }
             },
             "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 },
             "id": 2,
             "options": { "colorMode": "value", "graphMode": "area", "textMode": "auto" },
             "title": "Resident Memory",
             "type": "stat",
             "targets": [
               {
                 "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
                 "expr": "process_resident_memory_bytes{job=\"prefect\"}",
                 "legendFormat": "RSS"
               }
             ]
           },
       
           {
             "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
             "fieldConfig": {
               "defaults": {
                 "color": { "mode": "thresholds" },
                 "unit": "bytes",
                 "thresholds": {
                   "mode": "absolute",
                   "steps": [
                     { "color": "green", "value": null },
                     { "color": "yellow", "value": 1000000000 },
                     { "color": "red", "value": 2000000000 }
                   ]
                 }
               }
             },
             "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 },
             "id": 3,
             "options": { "colorMode": "value", "graphMode": "area", "textMode": "auto" },
             "title": "Virtual Memory",
             "type": "stat",
             "targets": [
               {
                 "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
                 "expr": "process_virtual_memory_bytes{job=\"prefect\"}",
                 "legendFormat": "VSZ"
               }
             ]
           },
       
           {
             "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
             "fieldConfig": {
               "defaults": {
                 "color": { "mode": "thresholds" },
                 "thresholds": {
                   "mode": "absolute",
                   "steps": [
                     { "color": "green", "value": null },
                     { "color": "yellow", "value": 800 },
                     { "color": "red", "value": 950 }
                   ]
                 }
               }
             },
             "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 },
             "id": 4,
             "options": { "colorMode": "value", "graphMode": "none", "textMode": "auto" },
             "title": "Open File Descriptors",
             "type": "stat",
             "targets": [
               {
                 "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
                 "expr": "process_open_fds{job=\"prefect\"}",
                 "legendFormat": "Open FDs"
               }
             ]
           },
       
           {
             "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
             "fieldConfig": {
               "defaults": {
                 "color": { "mode": "thresholds" },
                 "unit": "percentunit",
                 "max": 1,
                 "min": 0,
                 "thresholds": {
                   "mode": "absolute",
                   "steps": [
                     { "color": "green", "value": null },
                     { "color": "yellow", "value": 0.7 },
                     { "color": "red", "value": 0.9 }
                   ]
                 }
               }
             },
             "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 },
             "id": 5,
             "options": { "colorMode": "value", "graphMode": "none", "textMode": "auto" },
             "title": "FD Usage %",
             "type": "stat",
             "targets": [
               {
                 "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
                 "expr": "process_open_fds{job=\"prefect\"} / process_max_fds{job=\"prefect\"}",
                 "legendFormat": "FD Usage"
               }
             ]
           },
       
           {
             "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
             "fieldConfig": {
               "defaults": {
                 "color": { "mode": "thresholds" },
                 "unit": "s",
                 "thresholds": {
                   "mode": "absolute",
                   "steps": [{ "color": "blue", "value": null }]
                 }
               }
             },
             "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 },
             "id": 6,
             "options": { "colorMode": "value", "graphMode": "none", "textMode": "auto" },
             "title": "Process Uptime",
             "type": "stat",
             "targets": [
               {
                 "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
                 "expr": "time() - process_start_time_seconds{job=\"prefect\"}",
                 "legendFormat": "Uptime"
               }
             ]
           },
       
           {
             "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
             "fieldConfig": {
               "defaults": {
                 "color": { "mode": "palette-classic" },
                 "unit": "bytes",
                 "custom": { "lineWidth": 2, "fillOpacity": 10 }
               },
               "overrides": []
             },
             "gridPos": { "h": 8, "w": 12, "x": 0, "y": 5 },
             "id": 7,
             "options": { "legend": { "calcs": ["last", "max"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } },
             "title": "Memory Usage Over Time",
             "type": "timeseries",
             "targets": [
               {
                 "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
                 "expr": "process_resident_memory_bytes{job=\"prefect\"}",
                 "legendFormat": "Resident (RSS)"
               },
               {
                 "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
                 "expr": "process_virtual_memory_bytes{job=\"prefect\"}",
                 "legendFormat": "Virtual (VSZ)"
               }
             ]
           },
       
           {
             "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
             "fieldConfig": {
               "defaults": {
                 "color": { "mode": "palette-classic" },
                 "unit": "percentunit",
                 "custom": { "lineWidth": 2, "fillOpacity": 10 }
               }
             },
             "gridPos": { "h": 8, "w": 12, "x": 12, "y": 5 },
             "id": 8,
             "options": { "legend": { "calcs": ["last", "mean"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } },
             "title": "CPU Usage Rate",
             "type": "timeseries",
             "targets": [
               {
                 "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
                 "expr": "rate(process_cpu_seconds_total{job=\"prefect\"}[5m])",
                 "legendFormat": "CPU %"
               }
             ]
           },
       
           {
             "collapsed": false,
             "gridPos": { "h": 1, "w": 24, "x": 0, "y": 13 },
             "id": 101,
             "title": "Prefect Events",
             "type": "row"
           },
       
           {
             "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
             "fieldConfig": {
               "defaults": {
                 "color": { "mode": "palette-classic" },
                 "unit": "short",
                 "custom": { "lineWidth": 2, "fillOpacity": 10 }
               }
             },
             "gridPos": { "h": 8, "w": 12, "x": 0, "y": 14 },
             "id": 9,
             "options": { "legend": { "calcs": ["last", "mean"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } },
             "title": "Events Emitted & Observed Rate",
             "type": "timeseries",
             "targets": [
               {
                 "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
                 "expr": "rate(prefect_events_emitted_total[5m])",
                 "legendFormat": "Emitted/s"
               },
               {
                 "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
                 "expr": "rate(prefect_events_observed_total[5m])",
                 "legendFormat": "Observed/s"
               }
             ]
           },
       
           {
             "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
             "fieldConfig": {
               "defaults": {
                 "color": { "mode": "palette-classic" },
                 "unit": "short",
                 "custom": { "lineWidth": 2, "fillOpacity": 10 }
               }
             },
             "gridPos": { "h": 8, "w": 12, "x": 12, "y": 14 },
             "id": 10,
             "options": { "legend": { "calcs": ["last", "mean"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } },
             "title": "Log Events Rate",
             "type": "timeseries",
             "targets": [
               {
                 "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
                 "expr": "rate(prefect_logs_observed_total[5m])",
                 "legendFormat": "Logs Observed/s"
               }
             ]
           },
       
           {
             "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
             "fieldConfig": {
               "defaults": {
                 "color": { "mode": "palette-classic" },
                 "unit": "short",
                 "custom": { "lineWidth": 2, "fillOpacity": 10 }
               }
             },
             "gridPos": { "h": 8, "w": 12, "x": 0, "y": 22 },
             "id": 11,
             "options": { "legend": { "calcs": ["last"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } },
             "title": "WebSocket Connections (Events)",
             "type": "timeseries",
             "targets": [
               {
                 "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
                 "expr": "rate(prefect_event_websocket_connections_total[5m])",
                 "legendFormat": "{{direction}} / {{connection}}"
               }
             ]
           },
       
           {
             "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
             "fieldConfig": {
               "defaults": {
                 "color": { "mode": "palette-classic" },
                 "unit": "short",
                 "custom": { "lineWidth": 2, "fillOpacity": 10 }
               }
             },
             "gridPos": { "h": 8, "w": 12, "x": 12, "y": 22 },
             "id": 12,
             "options": { "legend": { "calcs": ["last"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } },
             "title": "WebSocket Connections (Logs)",
             "type": "timeseries",
             "targets": [
               {
                 "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
                 "expr": "rate(prefect_log_websocket_connections_total[5m])",
                 "legendFormat": "{{direction}} / {{connection}}"
               }
             ]
           },
       
           {
             "collapsed": false,
             "gridPos": { "h": 1, "w": 24, "x": 0, "y": 30 },
             "id": 102,
             "title": "Python Runtime / GC",
             "type": "row"
           },
       
           {
             "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
             "fieldConfig": {
               "defaults": {
                 "color": { "mode": "palette-classic" },
                 "unit": "short",
                 "custom": { "lineWidth": 2, "fillOpacity": 10 }
               }
             },
             "gridPos": { "h": 8, "w": 12, "x": 0, "y": 31 },
             "id": 13,
             "options": { "legend": { "calcs": ["last", "mean"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } },
             "title": "GC Collections Rate (by generation)",
             "type": "timeseries",
             "targets": [
               {
                 "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
                 "expr": "rate(python_gc_collections_total{job=\"prefect\"}[5m])",
                 "legendFormat": "Gen {{generation}}"
               }
             ]
           },
       
           {
             "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
             "fieldConfig": {
               "defaults": {
                 "color": { "mode": "palette-classic" },
                 "unit": "short",
                 "custom": { "lineWidth": 2, "fillOpacity": 10 }
               }
             },
             "gridPos": { "h": 8, "w": 12, "x": 12, "y": 31 },
             "id": 14,
             "options": { "legend": { "calcs": ["last", "mean"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "multi" } },
             "title": "GC Objects Collected Rate (by generation)",
             "type": "timeseries",
             "targets": [
               {
                 "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" },
                 "expr": "rate(python_gc_objects_collected_total{job=\"prefect\"}[5m])",
                 "legendFormat": "Gen {{generation}}"
               }
             ]
           }
       
         ],
         "refresh": "30s",
         "schemaVersion": 38,
         "tags": ["prefect", "mbari"],
         "templating": { "list": [] },
         "time": { "from": "now-1h", "to": "now" },
         "timepicker": {},
         "timezone": "browser",
         "title": "Prefect Server",
         "uid": "prefect-server-mbari",
         "version": 1
       }