about summary refs log tree commit diff
path: root/modules/services/metrics.nix
blob: 764b65fca50955d155a7509994c7db17da668d9e (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
{ config, lib, ... }:

with lib;
let
  cfg = config.modules.services.metrics;
in
{
  options.modules.services.metrics = {
    enable = mkEnableOption "metrics";
    realHost = mkOption { type = types.str; };
    tls.acmeHost = mkOption { type = types.str; default = cfg.realHost; };
    secrets.adminPassword = mkOption { type = types.path; description = "path to the admin password"; };
  };

  config = mkIf cfg.enable {
    services.prometheus = {
      enable = true;
      port = 9001;

      exporters = {
        node = {
          enable = true;
          enabledCollectors = [ "systemd" ];
          port = 9002;
        };
      };

      scrapeConfigs = [
        {
          job_name = "node";
          static_configs = [{
            targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.node.port}" ];
          }];
        }
      ];
    };

    services.loki = {
      enable = true;
      configuration = {
        auth_enabled = false;
        server.http_listen_port = 3100;

        ingester = {
          lifecycler = {
            address = "127.0.0.1";
            ring.kvstore.store = "inmemory";
            ring.replication_factor = 1;
            final_sleep = "0s";
          };

          chunk_idle_period = "1h";
          max_chunk_age = "1h";
          chunk_target_size = 1048576; # 1.5M
          chunk_retain_period = "30s";
          wal = {
            enabled = true;
            dir = "/tmp/loki-wal";
          };
        };

        schema_config.configs = [
          {
            from = "2023-02-24";
            store = "tsdb";
            object_store = "filesystem";
            schema = "v13";
            index = {
              prefix = "index_";
              period = "24h";
            };
          }
        ];

        storage_config = {
          tsdb_shipper = {
            active_index_directory = "/var/lib/loki/tsdb-index";
            cache_location = "/var/lib/loki/tsdb-cache";
            cache_ttl = "24h";
          };

          filesystem.directory = "/var/lib/loki/chunks";
        };

        limits_config = {
          reject_old_samples = true;
          reject_old_samples_max_age = "168h";

          retention_period = "120h";
        };

        table_manager = {
          retention_deletes_enabled = true;
          retention_period = "120h";
        };

        compactor = {
          working_directory = "/var/lib/loki";
          delete_request_store = "filesystem";
          compactor_ring.kvstore.store = "inmemory";
        };
      };
    };

    services.promtail = {
      enable = true;
      configuration = {
        server = {
          http_listen_port = 3031;
          grpc_listen_port = 0;
        };
        positions.filename = "/tmp/positions.yaml";
        clients = [
          { url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push"; }
        ];
        scrape_configs = [
          {
            job_name = "journal";
            journal = {
              max_age = "12h";
              labels = {
                job = "systemd-journal";
                host = config.networking.hostName;
              };
            };
            relabel_configs = [
              {
                source_labels = [ "__journal__systemd_unit" ];
                target_label = "unit";
              }
            ];
          }
        ];
      };
    };

    services.grafana = {
      enable = true;

      settings.server.http_addr = "127.0.0.1";
      settings.server.http_port = 2342;
      settings.server.domain = cfg.realHost;
      settings.security.admin_password = "$__file{${cfg.secrets.adminPassword}}";
    };

    services.nginx.virtualHosts.${cfg.realHost} = {
      forceSSL = true;
      useACMEHost = cfg.tls.acmeHost;

      locations."/" = {
        proxyPass = "http://localhost:${toString config.services.grafana.settings.server.http_port}";
        proxyWebsockets = true;
        extraConfig = ''
          proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
          # proxy_set_header Host $host;
        '';
      };
    };

    modules.persistence.directories = [
      "/var/lib/prometheus2"
      "/var/lib/loki"
      "/var/lib/grafana"
    ];
  };
}