feat: prometheus nixos module & test-container

for testing grafana in container
export QEMU_NET_OPTS="hostfwd=tcp::3000-:3000"
 nixos-rebuild build-vm --flake .#test-container
and then run the container, login is test/test

and the grafana port is forwarded to host machine, yay

.gitignore

@@ -4,3 +4,4 @@
 /result
 /routes/static/out.css
 /nixos.qcow2
+/data/

dev-conf/prometheus.yml

@@ -0,0 +1,7 @@
+global:
+  scrape_interval: 15s
+
+scrape_configs:
+- job_name: go
+  static_configs:
+  - targets: ['localhost:8081']

flake.nix

@@ -22,17 +22,20 @@
             pkgs.nodePackages.prettier
             pkgs.gnumake
             pkgs.redis
+            pkgs.prometheus
+            pkgs.grafana
           ];
           shellHook = ''
             export GOPATH=$PWD/.go
             export PATH=$GOPATH/bin:$PATH
+            export GRAPHANA_PATH=${pkgs.grafana}
           '';
         };
         packages = rec {
           some-automoderation = pkgs.buildGoModule {
             inherit pname version;
             src = pkgs.nix-gitignore.gitignoreSource [ ] ./.;
-            vendorHash = "sha256-zc4n5UxsmW8Nt52kS57i1W61Gy/J8T0RJPlwJnYJjHI=";
+            vendorHash = "sha256-FOIDFJNBViicvcpDG3T1KYABmI5Xyrv+IdQvT2Elhjg=";
 
             preBuild = ''
               ${pkgs.nodePackages.tailwindcss}/bin/tailwindcss -i routes/in.css -o routes/static/out.css
@@ -54,7 +57,7 @@
                   '';
                 };
                 networking.firewall.enable = false;
-                users.groups.test = {};
+                users.groups.test = { };
                 users.mutableUsers = false;
                 users.users.test = {
                   isNormalUser = true;
@@ -67,15 +70,33 @@
                   host = "some-automoderation.sunshine.industries";
                   useNginx = false;
                   port = 9090;
+                  metricsPort = 9091;
                   redisPort = 9999;
+                  enablePrometheus = true;
+                };
+                services.prometheus = {
+                  enable = true;
+                  port = 9998;
+                  # scrape config will be set up by the module
+                };
+                services.grafana = {
+                  enable = true;
+                  settings.server.http_port = 3000;
+                  settings.server.http_addr = "0.0.0.0";
+                  provision.datasources = {
+                    settings.datasources = [{
+                      name = "local-prometheus";
+                      type = "prometheus";
+                      url = "http://localhost:9998";
+                    }];
+                  };
                 };
               })
             ];
           };
         };
         nixosModules.some-automoderation-module = { lib, pkgs, config, ... }:
-          let
+          let cfg = config.services.${pname};
-            cfg = config.services.${pname};
           in {
             options.services.${pname} = {
               enable =
@@ -88,8 +109,7 @@
               useNginx = lib.mkOption {
                 type = lib.types.bool;
                 default = true;
-                description =
+                description = "Whether to set up nginx reverse proxy";
-                  "Whether to set up nginx reverse proxy";
               };
               port = lib.mkOption {
                 type = lib.types.int;
@@ -102,6 +122,16 @@
                 default = 7777;
                 description = "Port on which to connect to redis database.";
               };
+              metricsPort = lib.mkOption {
+                type = lib.types.int;
+                default = 8091;
+                description = "Port on which server exposes metrics.";
+              };
+              enablePrometheus = lib.mkOption {
+                type = lib.types.bool;
+                default = false;
+                description = "Whether to add scrape rules to prometheus";
+              };
               useHostTls = lib.mkOption {
                 type = lib.types.bool;
                 default = false;
@@ -109,12 +139,10 @@
                   "Whether virtual host should enable NixOS ACME certs";
               };
             };
-            config =
+            config = let
-              let
               username = "${pname}";
               groupname = "${pname}";
-            in
+            in lib.mkIf cfg.enable {
-              lib.mkIf cfg.enable {
               users.groups."${groupname}" = { };
               users.users."${username}" = {
                 isNormalUser = true; # needed to allow for home dir
@@ -128,8 +156,9 @@
                 startLimitBurst = 10;
                 serviceConfig = {
                   ExecStart = let
-                    serveCliArg =
+                    serveCliArg = "--port ${toString cfg.port} --redisPort ${
-                      "--port ${toString cfg.port} --redisPort ${toString cfg.redisPort}";
+                        toString cfg.redisPort
+                      } --metricsPort ${toString cfg.metricsPort}";
                   in "${packages.some-automoderation}/bin/${pname} ${serveCliArg}";
                   Restart = "on-failure";
                   User = "${username}";
@@ -154,10 +183,15 @@
                 enable = true;
                 user = "${username}";
                 port = cfg.redisPort;
-                settings = {
+                settings = { notify-keyspace-events = "KEA"; };
-                    notify-keyspace-events = "KEA";
+              };
-                  }
+              services.prometheus = lib.mkIf cfg.enablePrometheus {
-                ;
+                scrapeConfigs = [{
+                  job_name = "some-automoderation";
+                  static_configs = [{
+                    targets = [ "localhost:${toString cfg.metricsPort}" ];
+                  }];
+                }];
               };
             };
           };

go.mod

@@ -4,13 +4,22 @@ go 1.20
 
 require (
 	github.com/kr/pretty v0.3.1
+	github.com/prometheus/client_golang v1.17.0
 	github.com/redis/go-redis/v9 v9.2.1
 	golang.org/x/crypto v0.15.0
 )
 
 require (
+	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/kr/text v0.2.0 // indirect
-	github.com/rogpeppe/go-internal v1.9.0 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
+	github.com/prometheus/common v0.44.0 // indirect
+	github.com/prometheus/procfs v0.11.1 // indirect
+	github.com/rogpeppe/go-internal v1.10.0 // indirect
+	golang.org/x/sys v0.14.0 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
 )

go.sum

@@ -1,18 +1,46 @@
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
 github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
+github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
+github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
+github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q=
+github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY=
+github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM=
+github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
+github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
+github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
+github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI=
+github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY=
 github.com/redis/go-redis/v9 v9.2.1 h1:WlYJg71ODF0dVspZZCpYmoF1+U1Jjk9Rwd7pq6QmlCg=
 github.com/redis/go-redis/v9 v9.2.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
-github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
 github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
+github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA=
 golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
+golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
+google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=

main.go

@@ -7,6 +7,7 @@ import (
 	"log"
 	"net/http"
 
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/redis/go-redis/v9"
 
 	"sunshine.industries/some-automoderation/rooms"
@@ -19,10 +20,20 @@ var ctx = context.Background()
 func main() {
 	var port int
 	flag.IntVar(&port, "port", 8080, "Port on which the server should start")
+	var metricsPort int
+	flag.IntVar(&metricsPort, "metricsPort", 8081, "Port on which the server expose metrics")
 	var redisPort int
 	flag.IntVar(&redisPort, "redisPort", 7777, "Port on which server should connect to redis db")
 	flag.Parse()
 
+	promHandler := promhttp.Handler()
+	promMux := http.NewServeMux()
+	promMux.Handle("/metrics", promHandler)
+
+	go func() {
+		log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", metricsPort), promMux))
+	}()
+
 	rdb := redis.NewClient(&redis.Options{
 		Addr: fmt.Sprintf("localhost:%d", redisPort),
 		Password: "",
@@ -32,7 +43,7 @@ func main() {
 	roomsM := rooms.RedisRM { Rdb: rdb, }
 	sessions := sessions.RedisSM{ Rdb: rdb, }
 
-	log.Printf("Server will start on port: %d; listening to redis on: %d\n", port, redisPort)
+	log.Printf("Server will start on port: %d; /metrics on %d; listening to redis on: %d\n", port, metricsPort, redisPort)
 	routes.RegisterRoutes(sessions, roomsM)
 
 	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", port), nil))