{
  pkgs,
  config,
  secrets,
  ...
}:
let
  lib = pkgs.lib;
in
{

  services.prometheus = {
    enable = true;
    globalConfig = {
      scrape_interval = "15s";
    };
    scrapeConfigs =
      map
        (
          { name, value }:
          {
            job_name = name;
            static_configs = [
              {
                targets = [
                  "${config.services.prometheus.exporters.${name}.listenAddress}:${
                    toString config.services.prometheus.exporters.${name}.port
                  }"
                ];
              }
            ];
          }
        )
        (
          builtins.filter (
            { value, ... }: (builtins.tryEval value).success && (builtins.isAttrs value) && value.enable
          ) (lib.attrsToList config.services.prometheus.exporters)
        );

  };

  services.prometheus.exporters = {
    node = {
      enable = true;
      port = 9001;
      listenAddress = "127.0.0.1";
      enabledCollectors = [
        "cpu"
        "filesystem"
        "loadavg"
        "systemd"
      ];
      disabledCollectors = [ "rapl" ];
      extraFlags = [
        "--collector.ethtool"
        "--collector.softirqs"
        "--collector.tcpstat"
      ];
    };
    process = {
      enable = true;
      listenAddress = "127.0.0.1";
      port = 9256;
    };
    smartctl = {
      enable = true;
      port = 9633;
      listenAddress = "127.0.0.1";
    };
    postgres = {
      enable = true;
      port = 9634;
      listenAddress = "127.0.0.1";
      runAsLocalSuperUser = true;
    };
  };

  services.loki = {
    enable = true;
    configuration = {
      auth_enabled = false;
      server = {
        http_listen_port = 3100;
        grpc_listen_port = 9096;
      };
      common = {
        instance_addr = "127.0.0.1";
        path_prefix = "/tmp/loki";
        storage = {
          filesystem = {
            chunks_directory = "/tmp/loki/chunks";
            rules_directory = "/tmp/loki/rules";
          };
        };
        replication_factor = 1;
        ring = {
          kvstore = {
            store = "inmemory";
          };
        };
      };
      query_range = {
        results_cache = {
          cache = {
            embedded_cache = {
              enabled = true;
              max_size_mb = 100;
            };
          };
        };
      };
      limits_config = {
        allow_structured_metadata = false;
        reject_old_samples = true;
        reject_old_samples_max_age = "120d";
        retention_period = "120d";
      };
      schema_config = {
        configs = [
          {
            from = "2020-10-24";
            store = "tsdb";
            object_store = "filesystem";
            schema = "v13";
            index = {
              prefix = "index_";
              period = "24h";
            };
          }
          {
            from = "2024-07-11";
            store = "tsdb";
            object_store = "filesystem";
            schema = "v13";
            index = {
              prefix = "index_";
              period = "24h";
            };
          }
        ];
      };
      ruler = {
        alertmanager_url = "http://localhost:9093";
      };
      analytics.reporting_enabled = false;
    };
  };

  systemd.services.promtail.serviceConfig.User = lib.mkForce "nginx";
  services.promtail = {
    enable = true;
    configuration = {
      server = {
        http_listen_port = 3031;
        grpc_listen_port = 0;
      };
      positions = {
        filename = "/tmp/positions.yaml";
      };
      clients = [
        {
          url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
        }
      ];
      scrape_configs = [
        {
          job_name = "nginx";
          static_configs = [
            {
              targets = [ "localhost" ];
              labels = {
                job = "nginx";
                host = "fili";
                __path__ = "/var/log/nginx/json_access.log";
              };
            }
          ];
          pipeline_stages = [
            {
              json = {
                expressions = {
                  msec = "msec";
                  connection = "connection";
                  connection_requests = "connection_requests";
                  pid = "pid";
                  request_id = "request_id";
                  request_length = "request_length";
                  remote_addr = "remote_addr";
                  remote_user = "remote_user";
                  remote_port = "remote_port";
                  time_local = "time_local";
                  time_iso8601 = "time_iso8601";
                  request = "request";
                  request_uri = "request_uri";
                  args = "args";
                  status = "status";
                  body_bytes_sent = "body_bytes_sent";
                  bytes_sent = "bytes_sent";
                  http_referer = "http_referer";
                  http_user_agent = "http_user_agent";
                  http_x_forwarded_for = "http_x_forwarded_for";
                  http_host = "http_host";
                  server_name = "server_name";
                  request_time = "request_time";
                  upstream = "upstream";
                  upstream_connect_time = "upstream_connect_time";
                  upstream_header_time = "upstream_header_time";
                  upstream_response_time = "upstream_response_time";
                  upstream_response_length = "upstream_response_length";
                  upstream_cache_status = "upstream_cache_status";
                  ssl_protocol = "ssl_protocol";
                  ssl_cipher = "ssl_cipher";
                  scheme = "scheme";
                  request_method = "request_method";
                  server_protocol = "server_protocol";
                  pipe = "pipe";
                  gzip_ratio = "gzip_ratio";
                  http_cf_ray = "http_cf_ray";
                };
              };
            }
            {
              timestamp = {
                source = "msec";
                format = "Unix";
              };
            }
            # {
            #   geoip = {
            #     db = "/var/lib/geoip-databases/GeoLite2-City.mmdb";
            #     source = "remote_addr";
            #     db_type = "city";
            #   };
            # }
            # {
            #   pack = {
            #     labels = [ "geoip_country_name" ];
            #   };
            # }
          ];
        }
        {
          job_name = "journal";
          journal = {
            max_age = "12h";
            labels = {
              job = "systemd-journal";
              host = "fili";
            };
          };
          relabel_configs = [
            {
              source_labels = [ "__journal__systemd_unit" ];
              target_label = "unit";
            }
          ];
        }
      ];
    };
  };
  services.grafana = {
    enable = true;

    settings = {
      server = {
        domain = "metrics.donsz.nl";
        http_port = 2342;
        http_addr = "127.0.0.1";
      };
      "auth.anonymous".enabled' = true;
    };

    provision = {
      enable = true;
      datasources = {
        settings.datasources = [
          {
            type = "prometheus";
            name = "Prometheus";
            url = "http://${config.services.prometheus.listenAddress}:${toString config.services.prometheus.port}";
          }
          {
            name = "Loki";
            type = "loki";
            url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}";
          }
        ];
      };
    };
  };

  sops.secrets.geoip = {
    sopsFile = "${secrets}/geoip.yaml";
    key = "key";
    format = "yaml";
  };
  services.geoipupdate = {
    enable = true;
    settings = {
      LicenseKey = config.sops.secrets.geoip.path;
      AccountID = 1247225;
      DatabaseDirectory = "/var/lib/geoip-databases";
      EditionIDs = [
        "GeoLite2-ASN"
        "GeoLite2-City"
        "GeoLite2-Country"
      ];
    };
  };

  services.nginx.virtualHosts.${config.services.grafana.settings.server.domain} = {
    forceSSL = true;
    enableACME = true;
    locations."/".proxyPass =
      "http://${config.services.grafana.settings.server.http_addr}:${toString config.services.grafana.settings.server.http_port}";
  };

  services.nginx.package = pkgs.nginxStable.override (oldAttrs: {
    modules = with pkgs.nginxModules; [ geoip2 ];
    buildInputs = oldAttrs.buildInputs ++ [ pkgs.libmaxminddb ];
  });

  services.nginx.commonHttpConfig = ''
    autoindex_localtime on;
    geoip2 /var/lib/geoip-databases/GeoLite2-Country.mmdb {
      $geoip2_data_country_iso_code country iso_code;
    }

    log_format json_analytics escape=json '{'
      '"msec": "$msec", ' # request unixtime in seconds with a milliseconds resolution
      '"connection": "$connection", ' # connection serial number
      '"connection_requests": "$connection_requests", ' # number of requests made in connection
      '"pid": "$pid", ' # process pid
      '"request_id": "$request_id", ' # the unique request id
      '"request_length": "$request_length", ' # request length (including headers and body)
      '"remote_addr": "$remote_addr", ' # client IP
      '"remote_user": "$remote_user", ' # client HTTP username
      '"remote_port": "$remote_port", ' # client port
      '"time_local": "$time_local", '
      '"time_iso8601": "$time_iso8601", ' # local time in the ISO 8601 standard format
      '"request": "$request", ' # full path no arguments if the request
      '"request_uri": "$request_uri", ' # full path and arguments if the request
      '"args": "$args", ' # args
      '"status": "$status", ' # response status code
      '"body_bytes_sent": "$body_bytes_sent", ' # the number of body bytes exclude headers sent to a client
      '"bytes_sent": "$bytes_sent", ' # the number of bytes sent to a client
      '"http_referer": "$http_referer", ' # HTTP referer
      '"http_user_agent": "$http_user_agent", ' # user agent
      '"http_x_forwarded_for": "$http_x_forwarded_for", ' # http_x_forwarded_for
      '"http_host": "$http_host", ' # the request Host: header
      '"server_name": "$server_name", ' # the name of the vhost serving the request
      '"request_time": "$request_time", ' # request processing time in seconds with msec resolution
      '"upstream": "$upstream_addr", ' # upstream backend server for proxied requests
      '"upstream_connect_time": "$upstream_connect_time", ' # upstream handshake time incl. TLS
      '"upstream_header_time": "$upstream_header_time", ' # time spent receiving upstream headers
      '"upstream_response_time": "$upstream_response_time", ' # time spend receiving upstream body
      '"upstream_response_length": "$upstream_response_length", ' # upstream response length
      '"upstream_cache_status": "$upstream_cache_status", ' # cache HIT/MISS where applicable
      '"ssl_protocol": "$ssl_protocol", ' # TLS protocol
      '"ssl_cipher": "$ssl_cipher", ' # TLS cipher
      '"scheme": "$scheme", ' # http or https
      '"request_method": "$request_method", ' # request method
      '"server_protocol": "$server_protocol", ' # request protocol, like HTTP/1.1 or HTTP/2.0
      '"pipe": "$pipe", ' # "p" if request was pipelined, "." otherwise
      '"gzip_ratio": "$gzip_ratio", '
      '"http_cf_ray": "$http_cf_ray", '
      '"geoip_country_code": "$geoip2_data_country_iso_code"'
    '}';
    access_log /var/log/nginx/json_access.log json_analytics;
  '';
}