diff --git a/lotus-land-story/grafana.yml b/lotus-land-story/grafana.yml
index 747dde5..baca2f1 100644
--- a/lotus-land-story/grafana.yml
+++ b/lotus-land-story/grafana.yml
@@ -52,6 +52,9 @@
user = apikey
password = {{ grafana.smtp_password }}
from_address = grafana@kejadlen.dev
+
+ [plugins]
+ plugin_admin_enabled = true
mode: "0600"
owner: "{{ grafana_user.name }}"
@@ -74,6 +77,32 @@
prometheusVersion: 2.37.0
mode: "0644"
+ - name: Provision VictoriaMetrics
+ ansible.builtin.copy:
+ dest: /mnt/lotus-land-story/grafana/provisioning/datasources/victoria-metrics.yml
+ content: |
+ apiVersion: 1
+
+ # List of data sources to insert/update depending on what's
+ # available in the database.
+ datasources:
+ # <string, required> Name of the VictoriaMetrics datasource
+ # displayed in Grafana panels and queries.
+ - name: VictoriaMetrics
+ # <string, required> Sets the data source type.
+ type: victoriametrics-metrics-datasource
+ # <string, required> Sets the access mode, either
+ # proxy or direct (Server or Browser in the UI).
+ # Some data sources are incompatible with any setting
+ # but proxy (Server).
+ access: proxy
+ # <string> Sets default URL of the single node version of VictoriaMetrics
+ url: http://victoria-metrics:8428
+ # <string> Sets the pre-selected datasource for new panels.
+ # You can set only one default data source per organization.
+ isDefault: true
+ mode: "0644"
+
- name: Create Grafana volume
community.docker.docker_volume:
name: grafana
@@ -87,7 +116,7 @@
- /mnt/lotus-land-story/grafana/provisioning:/etc/grafana/provisioning
- grafana:/var/lib/grafana
env:
- # GF_INSTALL_PLUGINS: https://storage.googleapis.com/integration-artifacts/grafana-lokiexplore-app/grafana-lokiexplore-app-latest.zip;grafana-lokiexplore-app
+ GF_INSTALL_PLUGINS=victoriametrics-metrics-datasource
restart_policy: unless-stopped
networks:
- name: lotus_land_story
diff --git a/lotus-land-story/metrics.yml b/lotus-land-story/metrics.yml
new file mode 100644
index 0000000..83e7cc0
--- /dev/null
+++ b/lotus-land-story/metrics.yml
@@ -0,0 +1,38 @@
+- name: VictoriaMetrics
+ hosts: lotus-land-story
+ vars_files:
+ - vars.yml
+ vars:
+ version: v1.119.0
+ tasks:
+
+ - name: Make directories
+ ansible.builtin.file:
+ path: /mnt/lotus-land-story/{{ item }}
+ state: directory
+ mode: "0755"
+ loop:
+ - victoria-metrics
+
+ - name: Get docker network
+ community.docker.docker_network:
+ name: lotus_land_story
+ register: docker_network
+
+ - name: Run VictoriaMetrics
+ community.docker.docker_container:
+ restart: true
+ name: victoria-metrics
+ image: victoriametrics/victoria-metrics:{{ version }}
+ command: --selfScrapeInterval=5s -storageDataPath=victoria-metrics-data
+ restart_policy: unless-stopped
+ volumes:
+ - /mnt/lotus-land-story/victoria-metrics:/victoria-metrics-data
+ networks:
+ - name: lotus_land_story
+
+ handlers:
+ - name: Import restarts
+ ansible.builtin.import_tasks: restarts.yml
+
+# vim: ft=yaml.ansible
diff --git a/lotus-land-story/security.yml b/lotus-land-story/security.yml
index e27c2cc..e4d1742 100644
--- a/lotus-land-story/security.yml
+++ b/lotus-land-story/security.yml
@@ -63,7 +63,7 @@
[sshd]
enabled = true
mode = aggressive
- bantime = 24h
+ bantime = 1w
findtime = 12h
mode: "0600"
@@ -87,7 +87,7 @@
filter = gitea
logpath = /var/log/gitea.log
maxretry = 3
- bantime = 24h
+ bantime = 1w
findtime = 12h
action = iptables-allports[chain="FORWARD"]
mode: "0600"
diff --git a/lotus-land-story/templates/Caddyfile b/lotus-land-story/templates/Caddyfile
index ff7ce69..291ec9e 100644
--- a/lotus-land-story/templates/Caddyfile
+++ b/lotus-land-story/templates/Caddyfile
@@ -75,6 +75,13 @@ logs {
reverse_proxy victoria-logs:9428
}
+# need scheme here since `metrics` is a directive
+https://metrics {
+ tls internal
+ bind tailscale/metrics
+ reverse_proxy victoria-metrics:8428
+}
+
prometheus {
tls internal
bind tailscale/prometheus
@@ -134,6 +141,15 @@ docs {
}
}
+# homeassistant {
+# tls internal
+# bind tailscale/homeassistant
+# tailscale_auth
+# reverse_proxy calibre-web:8083 {
+# header_up Remote-User {http.auth.user.tailscale_user}
+# }
+# }
+
### External
git.kejadlen.dev {
diff --git a/lotus-land-story/templates/vector.yml b/lotus-land-story/templates/vector.yml
index 0e9e79a..edfdd67 100644
--- a/lotus-land-story/templates/vector.yml
+++ b/lotus-land-story/templates/vector.yml
@@ -14,11 +14,19 @@ transforms:
source: |
._time = del(.timestamp)
- if .container_name == "caddy" {
+ if .container_name == "anubis" {
+ parsed, err = parse_json(.message)
+ if err == null {
+ . = merge!(., parsed)
+ ._time = del(.time)
+ ._msg = del(.msg)
+ }
+ } else if .container_name == "caddy" {
. = merge!(., parse_json!(.message))
._time = del(.ts)
._msg = del(.msg)
} else if .container_name == "gitea" {
+ # TODO handle router logs: https://github.com/go-gitea/gitea/pull/33814
# gitea's common log doesn't match the vector common log because of the referer and user agent
router, router_err = parse_regex(
.message,
@@ -32,8 +40,6 @@ transforms:
._time = parse_timestamp!(del(.datetime), format: "%d/%b/%Y:%H:%M:%S %z")
._msg = del(.request)
- } else {
- ._msg = .message
}
} else if .container_name == "gotosocial" {
. = merge!(., parse_key_value!(.message))
@@ -50,20 +56,30 @@ transforms:
} else if .container_name == "miniflux" {
. = merge!(., parse_json!(.message))
._msg = del(.msg)
- } else {
+ } else if .container_name == "prometheus" {
+ . = merge!(., parse_json!(.message))
+ ._time = del(.time)
+ ._msg = del(.msg)
+ }
+
+ if ._msg == null {
._msg = .message
}
del(.message)
- ._stream_id = .container_name
- ._stream = { "container_name": .container_name }
+ ._stream_id = .container_id
varlogs_parsed:
type: remap
inputs:
- varlogs
source: |
- ._time = del(.timestamp)
+ parsed = parse_regex!(del(.message), r'^(?P<date>.*?) (?P<time>.*?) (?P<component>.*?) *\[(?P<proc_id>.*?)\]: (?P<level>.*?) +(?P<message>.*?)$')
+ . = merge(., parsed)
+
+ t = join!([del(.date), del(.time)], separator: " ")
+ ._time = parse_timestamp!(t, format: "%Y-%m-%d %H:%M:%S,%3f")
+
._msg = del(.message)
._stream_id = .file
._stream = { "file": .file }
@@ -103,7 +119,7 @@ sinks:
- docker_parsed
- varlogs_parsed
type: http
- uri: http://victoria-logs:9428/insert/jsonline
+ uri: http://victoria-logs:9428/insert/jsonline?_stream_fields=container_name
compression: gzip
encoding:
codec: json