[logs]
diff --git a/lotus-land-story/grafana.yml b/lotus-land-story/grafana.yml
index 4a275a7..cfdb3ec 100644
--- a/lotus-land-story/grafana.yml
+++ b/lotus-land-story/grafana.yml
@@ -87,7 +87,7 @@
- /mnt/lotus-land-story/grafana/provisioning:/etc/grafana/provisioning
- grafana:/var/lib/grafana
env:
- GF_INSTALL_PLUGINS: https://storage.googleapis.com/integration-artifacts/grafana-lokiexplore-app/grafana-lokiexplore-app-latest.zip;grafana-lokiexplore-app
+ # GF_INSTALL_PLUGINS: https://storage.googleapis.com/integration-artifacts/grafana-lokiexplore-app/grafana-lokiexplore-app-latest.zip;grafana-lokiexplore-app
restart_policy: unless-stopped
networks:
- name: lotus_land_story
diff --git a/lotus-land-story/logs.yml b/lotus-land-story/logs.yml
index 087ecd7..0b93bca 100644
--- a/lotus-land-story/logs.yml
+++ b/lotus-land-story/logs.yml
@@ -1,9 +1,41 @@
+- name: Set up Vector
+ tags: vector
+ hosts: lotus-land-story
+ vars:
+ version: 0.45.0-debian
+ tasks:
+
+ - name: Create Vector config dir
+ ansible.builtin.file:
+ path: /mnt/lotus-land-story/vector
+ state: directory
+ mode: "0700"
+
+ - name: Configure Vector
+ ansible.builtin.copy:
+ src: templates/vector.yml
+ dest: /mnt/lotus-land-story/vector/vector.yml
+
+ - name: Run Vector
+ community.docker.docker_container:
+ name: vector
+ image: timberio/vector:{{ version }}
+ restart: true
+ command: -c /etc/vector/*.yml
+ volumes:
+ - /mnt/lotus-land-story/vector:/etc/vector:ro
+ # - /var/log:/var/log
+ - /var/run/docker.sock:/var/run/docker.sock
+ restart_policy: unless-stopped
+ networks:
+ - name: lotus_land_story
+
- name: VictoriaLogs
hosts: lotus-land-story
vars_files:
- vars.yml
vars:
- victoria_logs_version: 1.20.0
+ version: 1.20.0
tasks:
- name: Make directories
@@ -23,14 +55,50 @@
community.docker.docker_container:
restart: true
name: victoria-logs
- image: docker.io/victoriametrics/victoria-logs:v{{ victoria_logs_version }}-victorialogs
+ image: docker.io/victoriametrics/victoria-logs:v{{ version }}-victorialogs
+ command: -storageDataPath=/data -syslog.listenAddr.tcp=:514
restart_policy: unless-stopped
+ ports:
+ - "514:514" # syslog
+ - "9428:9428" # journald
volumes:
- /mnt/lotus-land-story/victoria-logs:/data
networks:
- name: lotus_land_story
- # etc_hosts:
- # host.docker.internal: host-gateway
+
+ - name: Ship syslogs
+ ansible.builtin.copy:
+ content: |
+ *.* action(type="omfwd"
+ protocol="tcp"
+ target="127.0.0.1"
+ port="514"
+ Template="RSYSLOG_SyslogProtocol23Format"
+ TCP_Framing="octet-counted"
+ KeepAlive="on")
+ dest: /etc/rsyslog.d/50-promtail.conf
+ mode: "0644"
+ notify: Restart rsyslog
+
+ - name: Ship journald logs
+ block:
+
+ - name: Install systemd-journal-remote
+ ansible.builtin.apt:
+ name: systemd-journal-remote
+ state: present
+
+ - name: Set remote URL for uploading
+ ansible.builtin.lineinfile:
+ path: /etc/systemd/journal-upload.conf
+ regexp: '^URL='
+ insertafter: '^# URL='
+ line: URL=http://localhost:9428/insert/journald
+
+ - name: Run systemd-journal-remote
+ ansible.builtin.service:
+ name: systemd-journal-upload
+ state: started
handlers:
- name: Import restarts
diff --git a/lotus-land-story/loki.yml b/lotus-land-story/loki.yml
index 0a49342..1a1baaa 100644
--- a/lotus-land-story/loki.yml
+++ b/lotus-land-story/loki.yml
@@ -12,12 +12,10 @@
dest: /mnt/lotus-land-story/grafana/provisioning/datasources/loki.yml
content: |
apiVersion: 1
-
- datasources:
+ deleteDatasources:
- name: Loki
- type: loki
- access: proxy
- url: http://loki:3100
+ orgId: 1
+ prune: true
mode: "0644"
- name: Create Loki config dirs
@@ -94,6 +92,7 @@
- name: Run Loki
community.docker.docker_container:
+ state: absent
name: loki
image: grafana/loki:{{ version }}
restart: true
@@ -109,116 +108,6 @@
etc_hosts:
host.docker.internal: host-gateway
-- name: Set up Vector
- tags: vector
- hosts: lotus-land-story
- vars:
- version: 0.45.0-debian
- tasks:
-
- - name: Create Vector config dir
- ansible.builtin.file:
- path: /mnt/lotus-land-story/vector
- state: directory
- mode: "0700"
-
- - name: Configure Vector
- ansible.builtin.copy:
- dest: /mnt/lotus-land-story/vector/vector.yml
- content: |
- sources:
- rsyslog:
- type: syslog
- address: 0.0.0.0:514
- mode: tcp
- docker:
- type: docker_logs
-
- transforms:
- docker_parsed:
- type: remap
- inputs:
- - docker
- source: |
- ._time = del(.timestamp)
-
- if .container_name == "caddy" {
- . = merge!(., parse_json!(.message))
- ._time = del(.ts)
- ._msg = del(.msg)
- } else if .container_name == "gotosocial" {
- . = merge!(., parse_key_value!(.message))
- ._time = parse_timestamp!(del(.timestamp), format: "%d/%m/%Y %H:%M:%S%.3f")
- ._msg = del(.msg)
- } else if .container_name == "loki" {
- . = merge!(., parse_key_value!(.message))
- ._time = del(.ts)
- ._msg = del(.msg)
- } else if .container_name == "miniflux" {
- . = merge!(., parse_json!(.message))
- ._msg = del(.msg)
- } else {
- ._msg = .message
- }
- del(.message)
-
- ._stream_id = .container_name
-
- sinks:
- # console:
- # inputs:
- # - remap_caddy
- # target: stdout
- # type: console
- # encoding:
- # codec: json
-
- loki_rsyslog:
- type: loki
- inputs:
- - rsyslog
- endpoint: http://loki:3100
- encoding:
- codec: json
- labels:
- agent: vector
- host: "{{ '{{' }} host }}"
- level: "{{ '{{' }} severity }}"
- syslog_facility: "{{ '{{' }} facility }}"
- syslog_identifier: "{{ '{{' }} appname }}"
-
- victoria_logs:
- inputs:
- - docker_parsed
- type: http
- uri: http://victoria-logs:9428/insert/jsonline
- compression: gzip
- encoding:
- codec: json
- framing:
- method: newline_delimited
- healthcheck:
- enabled: false
-
- - name: Run Vector
- community.docker.docker_container:
- name: vector
- image: timberio/vector:{{ version }}
- restart: true
- command: -c /etc/vector/*.yml
- ports:
- - "514:514"
- volumes:
- - /mnt/lotus-land-story/vector:/etc/vector:ro
- # - /var/log:/var/log
- - /var/run/docker.sock:/var/run/docker.sock
- restart_policy: unless-stopped
- networks:
- - name: lotus_land_story
- # etc_hosts:
- # host.docker.internal: host-gateway
-
-
- name: Set up Promtail
tags: promtail
hosts: lotus-land-story
@@ -268,6 +157,7 @@
- name: Run Promtail
community.docker.docker_container:
+ state: absent
name: promtail
image: grafana/promtail:{{ version }}
restart: true
diff --git a/lotus-land-story/security.yml b/lotus-land-story/security.yml
index e69e151..39db116 100644
--- a/lotus-land-story/security.yml
+++ b/lotus-land-story/security.yml
@@ -1,4 +1,6 @@
-- hosts: lotus-land-story
+- name: Set up firewall
+ tags: ufw
+ hosts: lotus-land-story
tasks:
- name: Enable ufw
@@ -40,4 +42,27 @@
- name: Import restarts
ansible.builtin.import_tasks: restarts.yml
+- name: Set up Fail2Ban
+ tags: fail2ban
+ hosts: lotus-land-story
+ tasks:
+
+ - name: Install Fail2Ban
+ ansible.builtin.apt:
+ name: fail2ban
+ state: present
+
+ - name: Configure Fail2Ban
+ ansible.builtin.copy:
+ dest: /etc/jail.local
+ content: |
+ [sshd]
+ enabled = true
+ mode: "0600"
+
+ - name: Start Fail2Ban
+ ansible.builtin.service:
+ name: fail2ban
+ state: started
+
# vim: ft=yaml.ansible
diff --git a/lotus-land-story/templates/prometheus.yml b/lotus-land-story/templates/prometheus.yml
index b32024a..676ab88 100644
--- a/lotus-land-story/templates/prometheus.yml
+++ b/lotus-land-story/templates/prometheus.yml
@@ -37,10 +37,6 @@ scrape_configs:
static_configs:
- targets: ['postgres-exporter:9187']
- - job_name: promtail
- static_configs:
- - targets: ['promtail:9080']
-
- job_name: miniflux
static_configs:
- targets: ['miniflux:8080']
@@ -83,3 +79,7 @@ scrape_configs:
bearer_token: {{ gitea.metrics_token }}
static_configs:
- targets: ['gitea:3000']
+
+ - job_name: victoria-logs
+ static_configs:
+ - targets: ['victoria-logs:9428']
diff --git a/lotus-land-story/templates/vector.yml b/lotus-land-story/templates/vector.yml
new file mode 100644
index 0000000..af23dab
--- /dev/null
+++ b/lotus-land-story/templates/vector.yml
@@ -0,0 +1,77 @@
+sources:
+ docker:
+ type: docker_logs
+
+transforms:
+ docker_parsed:
+ type: remap
+ inputs:
+ - docker
+ source: |
+ ._time = del(.timestamp)
+
+ if .container_name == "caddy" {
+ . = merge!(., parse_json!(.message))
+ ._time = del(.ts)
+ ._msg = del(.msg)
+ } else if .container_name == "gitea" {
+ # gitea's common log doesn't match the vector common log because of the referer and user agent
+ router, router_err = parse_regex(
+ .message,
+ r'^(?P<remote_host>.*?) - (?P<identity>.*?) \[(?P<datetime>.*?)\] "(?P<request>.*?)" (?P<status>.*?) (?P<size>.*?) "(?P<referer>.*?)" "(?P<user_agent>.*?)"$',
+ )
+ if router_err == null {
+ . = merge(., router)
+
+ request = parse_regex!(.request, r'^(?P<method>.*?) (?P<request_uri>.*?) (?P<proto>.*?)$')
+ . = merge(., request)
+
+ ._time = parse_timestamp!(del(.datetime), format: "%d/%b/%Y:%H:%M:%S %z")
+ ._msg = del(.request)
+ } else {
+ ._msg = .message
+ }
+ } else if .container_name == "gotosocial" {
+ . = merge!(., parse_key_value!(.message))
+ ._time = parse_timestamp!(del(.timestamp), format: "%d/%m/%Y %H:%M:%S%.3f")
+ ._msg = del(.msg)
+ } else if .container_name == "grafana" {
+ . = merge!(., parse_key_value!(.message))
+ ._time = del(.t)
+ ._msg = del(.msg)
+ } else if .container_name == "loki" {
+ . = merge!(., parse_key_value!(.message))
+ ._time = del(.ts)
+ ._msg = del(.msg)
+ } else if .container_name == "miniflux" {
+ . = merge!(., parse_json!(.message))
+ ._msg = del(.msg)
+ } else {
+ ._msg = .message
+ }
+ del(.message)
+
+ ._stream_id = .container_name
+ ._stream = { "container_name": .container_name }
+
+sinks:
+ # console:
+ # inputs:
+ # - remap_caddy
+ # target: stdout
+ # type: console
+ # encoding:
+ # codec: json
+
+ victoria_logs:
+ inputs:
+ - docker_parsed
+ type: http
+ uri: http://victoria-logs:9428/insert/jsonline
+ compression: gzip
+ encoding:
+ codec: json
+ framing:
+ method: newline_delimited
+ healthcheck:
+ enabled: false