[logs] VictoriaLogs
diff --git a/lotus-land-story/logs.yml b/lotus-land-story/logs.yml
index e1cd9c8..087ecd7 100644
--- a/lotus-land-story/logs.yml
+++ b/lotus-land-story/logs.yml
@@ -1,130 +1,39 @@
-# https://www.parseable.com/docs/log-ingestion/agents/vector
-# https://vector.dev/docs/setup/installation/platforms/docker/
-
-- name: Set up Parseable
+- name: VictoriaLogs
hosts: lotus-land-story
vars_files:
- vars.yml
+ vars:
+ victoria_logs_version: 1.20.0
tasks:
- - name: Create directories for volume mounting
+ - name: Make directories
ansible.builtin.file:
- path: /mnt/lotus-land-story/parseable/{{ item }}
+ path: /mnt/lotus-land-story/{{ item }}
state: directory
mode: "0755"
loop:
- - data
- - staging
-
- - name: Configure Vector
- ansible.builtin.copy:
- dest: /mnt/lotus-land-story/parseable/vector.yml
- content: |
- sources:
- # vector_metrics:
- # type: internal_metrics
- raw_docker_logs:
- type: docker_logs
- transforms:
- docker_logs:
- type: remap
- inputs:
- - raw_docker_logs
- source: |
- if includes(["authelia", "caddy", "miniflux"], .container_name) {
- . |= object!(parse_json!(.message))
- } else if .container_name == "paperless" {
- # asctime has trailing milliseconds, which I can't figure out
- # how to parse, but I also don't care about it, so drop it
- parsed = parse_regex!(
- .message,
- r'\[(?P<asctime>.*?),\d*\] \[(?P<level>.*?)\] \[(?P<name>.*?)\] (?P<message>.*)',
- )
- .paperless_time = parse_timestamp!(del(parsed.asctime), format: "%F %T")
- . |= parsed
- } else if includes(["grafana", "loki"], .container_name) {
- . |= parse_key_value!(.message)
- }
- sinks:
- # console:
- # type: console
- # inputs:
- # - demo_logs
- # encoding:
- # codec: json
- parseable:
- type: http
- method: post
- batch:
- max_bytes: 10485760
- max_events: 1000
- timeout_secs: 10
- compression: gzip
- inputs:
- - docker_logs
- encoding:
- codec: json
- uri: http://parseable:8000/api/v1/ingest
- auth:
- strategy: basic
- user: alpha
- password: {{ parseable.password }}
- request:
- headers:
- X-P-Stream: vector
- healthcheck:
- enabled: true
- path: http://parseable/api/v1/liveness
- port: 8000
- # prometheus:
- # type: prometheus_remote_write
- # endpoint: http://prometheus:9090
- # inputs:
- # - vector_metrics
- mode: "0600"
+ - victoria-logs
- name: Get docker network
community.docker.docker_network:
name: lotus_land_story
+ register: docker_network
- # https://www.parseable.com/logstash/docker-compose.yaml
- - name: Run Parseable
+ - name: Run VictoriaLogs
community.docker.docker_container:
restart: true
- name: parseable
- image: parseable/parseable:v0.7.3
- command:
- - parseable
- - local-store
- env:
- P_FS_DIR: /parseable/data
- P_STAGING_DIR: /parseable/staging
- P_USERNAME: alpha
- P_PASSWORD: "{{ parseable.password }}"
- P_OIDC_CLIENT_ID: parseable
- P_OIDC_CLIENT_SECRET: "{{ parseable.oidc_secret }}"
- P_OIDC_ISSUER: https://auth.{{ domain }}
- P_ORIGIN_URI: https://logs.{{ domain }}
- # RUST_LOG: warning
- volumes:
- - ./data:/parseable/data
- - ./staging:/parseable/staging
+ name: victoria-logs
+ image: docker.io/victoriametrics/victoria-logs:v{{ victoria_logs_version }}-victorialogs
restart_policy: unless-stopped
- networks:
- - name: lotus_land_story
-
- - name: Run Vector
- community.docker.docker_container:
- restart: true
- name: vector
- image: timberio/vector:0.35.0-alpine
- env:
- # VECTOR_LOG: debug
volumes:
- - /mnt/lotus-land-story/parseable/vector.yml:/etc/vector/vector.yaml
- - /var/run/docker.sock:/var/run/docker.sock # for docker_logs
- restart_policy: unless-stopped
+ - /mnt/lotus-land-story/victoria-logs:/data
networks:
- name: lotus_land_story
+ # etc_hosts:
+ # host.docker.internal: host-gateway
+
+ handlers:
+ - name: Import restarts
+ ansible.builtin.import_tasks: restarts.yml
# vim: ft=yaml.ansible
diff --git a/lotus-land-story/loki.yml b/lotus-land-story/loki.yml
index 854a4b0..0a49342 100644
--- a/lotus-land-story/loki.yml
+++ b/lotus-land-story/loki.yml
@@ -127,71 +127,42 @@
dest: /mnt/lotus-land-story/vector/vector.yml
content: |
sources:
- docker_caddy:
- type: docker_logs
- include_containers:
- - caddy
- docker_gts:
- type: docker_logs
- include_containers:
- - gotosocial
- docker_miniflux:
- type: docker_logs
- include_containers:
- - miniflux
- docker_paperless:
- type: docker_logs
- include_containers:
- - paperless
- docker_rest:
- type: docker_logs
- exclude_containers:
- - caddy
- - gotosocial
- - miniflux
- - paperless
rsyslog:
type: syslog
address: 0.0.0.0:514
mode: tcp
+ docker:
+ type: docker_logs
transforms:
- remap_caddy:
+ docker_parsed:
type: remap
inputs:
- - docker_caddy
+ - docker
source: |
- . |= object!(parse_json!(.message))
- del(.message)
- .user_agent = .request.headers."User-Agent"[0]
+ ._time = del(.timestamp)
- remap_gts:
- type: remap
- inputs:
- - docker_gts
- source: |
- . |= parse_key_value!(.message)
- del(.message)
-
- remap_miniflux:
- type: remap
- inputs:
- - docker_miniflux
- source: |
- . |= object!(parse_json!(.message))
+ if .container_name == "caddy" {
+ . = merge!(., parse_json!(.message))
+ ._time = del(.ts)
+ ._msg = del(.msg)
+ } else if .container_name == "gotosocial" {
+ . = merge!(., parse_key_value!(.message))
+ ._time = parse_timestamp!(del(.timestamp), format: "%d/%m/%Y %H:%M:%S%.3f")
+ ._msg = del(.msg)
+ } else if .container_name == "loki" {
+ . = merge!(., parse_key_value!(.message))
+ ._time = del(.ts)
+ ._msg = del(.msg)
+ } else if .container_name == "miniflux" {
+ . = merge!(., parse_json!(.message))
+ ._msg = del(.msg)
+ } else {
+ ._msg = .message
+ }
del(.message)
- remap_paperless:
- type: remap
- inputs:
- - docker_paperless
- source: |
- . |= parse_regex!(.message, r'\[(?P<timestamp>.*?)\] \[(?P<level>.*?)\] \[(?P<name>.*?)\] (?P<message>.*)')
- .timestamp = parse_timestamp!(
- .timestamp,
- format: "%F %T,%3f",
- timezone: "America/Los_Angeles"
- )
+ ._stream_id = .container_name
sinks:
# console:
@@ -202,21 +173,6 @@
# encoding:
# codec: json
- loki_docker:
- type: loki
- inputs:
- - remap_caddy
- - remap_gts
- - remap_miniflux
- - remap_paperless
- - docker_rest
- endpoint: http://loki:3100
- encoding:
- codec: json
- labels:
- agent: vector
- container: "{{ '{{' }} container_name }}"
-
loki_rsyslog:
type: loki
inputs:
@@ -231,6 +187,19 @@
syslog_facility: "{{ '{{' }} facility }}"
syslog_identifier: "{{ '{{' }} appname }}"
+ victoria_logs:
+ inputs:
+ - docker_parsed
+ type: http
+ uri: http://victoria-logs:9428/insert/jsonline
+ compression: gzip
+ encoding:
+ codec: json
+ framing:
+ method: newline_delimited
+ healthcheck:
+ enabled: false
+
- name: Run Vector
community.docker.docker_container:
name: vector
diff --git a/lotus-land-story/templates/Caddyfile b/lotus-land-story/templates/Caddyfile
index 9ff35d5..ff7ce69 100644
--- a/lotus-land-story/templates/Caddyfile
+++ b/lotus-land-story/templates/Caddyfile
@@ -69,6 +69,12 @@ loki {
metrics /metrics
}
+logs {
+ tls internal
+ bind tailscale/logs
+ reverse_proxy victoria-logs:9428
+}
+
prometheus {
tls internal
bind tailscale/prometheus