victoria metrics
change tkxywrnxxsoqpttqywyqqusxuxwpypzk
commit 8a10cbf3ae2c1dfc6604933546ef7de646addae3
author Alpha Chen <alpha@kejadlen.dev>
date
parent ktrmulvk
diff --git a/lotus-land-story/grafana.yml b/lotus-land-story/grafana.yml
index 747dde5..baca2f1 100644
--- a/lotus-land-story/grafana.yml
+++ b/lotus-land-story/grafana.yml
@@ -52,6 +52,9 @@
           user = apikey
           password = {{ grafana.smtp_password }}
           from_address = grafana@kejadlen.dev
+
+          [plugins]
+          plugin_admin_enabled = true
         mode: "0600"
         owner: "{{ grafana_user.name }}"
 
@@ -74,6 +77,32 @@
                 prometheusVersion: 2.37.0
         mode: "0644"
 
+    - name: Provision VictoriaMetrics
+      ansible.builtin.copy:
+        dest: /mnt/lotus-land-story/grafana/provisioning/datasources/victoria-metrics.yml
+        content: |
+          apiVersion: 1
+
+          # List of data sources to insert/update depending on what's
+          # available in the database.
+          datasources:
+            # <string, required> Name of the VictoriaMetrics datasource
+            # displayed in Grafana panels and queries.
+            - name: VictoriaMetrics
+              # <string, required> Sets the data source type.
+              type: victoriametrics-metrics-datasource
+                # <string, required> Sets the access mode, either
+                # proxy or direct (Server or Browser in the UI).
+                # Some data sources are incompatible with any setting
+              # but proxy (Server).
+              access: proxy
+              # <string> Sets default URL of the single node version of VictoriaMetrics
+              url: http://victoria-metrics:8428
+              # <string> Sets the pre-selected datasource for new panels.
+              # You can set only one default data source per organization.
+              isDefault: true
+        mode: "0644"
+
     - name: Create Grafana volume
       community.docker.docker_volume:
         name: grafana
@@ -87,7 +116,7 @@
           - /mnt/lotus-land-story/grafana/provisioning:/etc/grafana/provisioning
           - grafana:/var/lib/grafana
         env:
-          # GF_INSTALL_PLUGINS: https://storage.googleapis.com/integration-artifacts/grafana-lokiexplore-app/grafana-lokiexplore-app-latest.zip;grafana-lokiexplore-app
+          GF_INSTALL_PLUGINS=victoriametrics-metrics-datasource
         restart_policy: unless-stopped
         networks:
           - name: lotus_land_story
diff --git a/lotus-land-story/metrics.yml b/lotus-land-story/metrics.yml
new file mode 100644
index 0000000..83e7cc0
--- /dev/null
+++ b/lotus-land-story/metrics.yml
@@ -0,0 +1,38 @@
+- name: VictoriaMetrics
+  hosts: lotus-land-story
+  vars_files:
+    - vars.yml
+  vars:
+    version: v1.119.0
+  tasks:
+
+    - name: Make directories
+      ansible.builtin.file:
+        path: /mnt/lotus-land-story/{{ item }}
+        state: directory
+        mode: "0755"
+      loop:
+        - victoria-metrics
+
+    - name: Get docker network
+      community.docker.docker_network:
+        name: lotus_land_story
+      register: docker_network
+
+    - name: Run VictoriaMetrics
+      community.docker.docker_container:
+        restart: true
+        name: victoria-metrics
+        image: victoriametrics/victoria-metrics:{{ version }}
+        command: --selfScrapeInterval=5s -storageDataPath=victoria-metrics-data
+        restart_policy: unless-stopped
+        volumes:
+          - /mnt/lotus-land-story/victoria-metrics:/victoria-metrics-data
+        networks:
+          - name: lotus_land_story
+
+  handlers:
+    - name: Import restarts
+      ansible.builtin.import_tasks: restarts.yml
+
+# vim: ft=yaml.ansible
diff --git a/lotus-land-story/security.yml b/lotus-land-story/security.yml
index e27c2cc..e4d1742 100644
--- a/lotus-land-story/security.yml
+++ b/lotus-land-story/security.yml
@@ -63,7 +63,7 @@
           [sshd]
           enabled = true
           mode = aggressive
-          bantime = 24h
+          bantime = 1w
           findtime = 12h
         mode: "0600"
 
@@ -87,7 +87,7 @@
               filter = gitea
               logpath = /var/log/gitea.log
               maxretry = 3
-              bantime = 24h
+              bantime = 1w
               findtime = 12h
               action = iptables-allports[chain="FORWARD"]
             mode: "0600"
diff --git a/lotus-land-story/templates/Caddyfile b/lotus-land-story/templates/Caddyfile
index ff7ce69..291ec9e 100644
--- a/lotus-land-story/templates/Caddyfile
+++ b/lotus-land-story/templates/Caddyfile
@@ -75,6 +75,13 @@ logs {
 	reverse_proxy victoria-logs:9428
 }
 
+# need scheme here since `metrics` is a directive
+https://metrics {
+ 	tls internal
+ 	bind tailscale/metrics
+ 	reverse_proxy victoria-metrics:8428
+}
+
 prometheus {
 	tls internal
 	bind tailscale/prometheus
@@ -134,6 +141,15 @@ docs {
 	}
 }
 
+# homeassistant {
+# 	tls internal
+# 	bind tailscale/homeassistant
+# 	tailscale_auth
+# 	reverse_proxy calibre-web:8083 {
+# 		header_up Remote-User {http.auth.user.tailscale_user}
+# 	}
+# }
+
 ### External
 
 git.kejadlen.dev {
diff --git a/lotus-land-story/templates/vector.yml b/lotus-land-story/templates/vector.yml
index 0e9e79a..edfdd67 100644
--- a/lotus-land-story/templates/vector.yml
+++ b/lotus-land-story/templates/vector.yml
@@ -14,11 +14,19 @@ transforms:
     source: |
       ._time = del(.timestamp)
 
-      if .container_name == "caddy" {
+      if .container_name == "anubis" {
+        parsed, err = parse_json(.message)
+        if err == null {
+          . = merge!(., parsed)
+          ._time = del(.time)
+          ._msg = del(.msg)
+        }
+      } else if .container_name == "caddy" {
         . = merge!(., parse_json!(.message))
         ._time = del(.ts)
         ._msg = del(.msg)
       } else if .container_name == "gitea" {
+        # TODO handle router logs: https://github.com/go-gitea/gitea/pull/33814
         # gitea's common log doesn't match the vector common log because of the referer and user agent
         router, router_err = parse_regex(
             .message,
@@ -32,8 +40,6 @@ transforms:
 
             ._time = parse_timestamp!(del(.datetime), format: "%d/%b/%Y:%H:%M:%S %z")
             ._msg = del(.request)
-        } else {
-          ._msg = .message
         }
       } else if .container_name == "gotosocial" {
         . = merge!(., parse_key_value!(.message))
@@ -50,20 +56,30 @@ transforms:
       } else if .container_name == "miniflux" {
         . = merge!(., parse_json!(.message))
         ._msg = del(.msg)
-      } else {
+      } else if .container_name == "prometheus" {
+        . = merge!(., parse_json!(.message))
+        ._time = del(.time)
+        ._msg = del(.msg)
+      }
+
+      if ._msg == null {
         ._msg = .message
       }
       del(.message)
 
-      ._stream_id = .container_name
-      ._stream = { "container_name": .container_name }
+      ._stream_id = .container_id
 
   varlogs_parsed:
     type: remap
     inputs:
       - varlogs
     source: |
-      ._time = del(.timestamp)
+      parsed = parse_regex!(del(.message), r'^(?P<date>.*?) (?P<time>.*?) (?P<component>.*?) *\[(?P<proc_id>.*?)\]: (?P<level>.*?) +(?P<message>.*?)$')
+      . = merge(., parsed)
+
+      t = join!([del(.date), del(.time)], separator: " ")
+      ._time = parse_timestamp!(t, format: "%Y-%m-%d %H:%M:%S,%3f")
+
       ._msg = del(.message)
       ._stream_id = .file
       ._stream = { "file": .file }
@@ -103,7 +119,7 @@ sinks:
       - docker_parsed
       - varlogs_parsed
     type: http
-    uri: http://victoria-logs:9428/insert/jsonline
+    uri: http://victoria-logs:9428/insert/jsonline?_stream_fields=container_name
     compression: gzip
     encoding:
       codec: json