diff --git a/lotus-land-story/logs.yml b/lotus-land-story/logs.yml
index 00647c0..ee16d88 100644
--- a/lotus-land-story/logs.yml
+++ b/lotus-land-story/logs.yml
@@ -1,4 +1,5 @@
-# https://www.parseable.com/docs/log-ingestion/agents/logstash
+# https://www.parseable.com/docs/log-ingestion/agents/vector
+# https://vector.dev/docs/setup/installation/platforms/docker/
- name: Set up Parseable
hosts: lotus-land-story
@@ -15,30 +16,71 @@
- data
- staging
- # https://www.parseable.com/logstash/logstash.conf
- - name: Configure Logstash
+ - name: Configure Vector
ansible.builtin.copy:
- dest: /mnt/lotus-land-story/parseable/logstash.conf
+ dest: /mnt/lotus-land-story/parseable/vector.yml
content: |
- output {
- http {
- id => "parseable"
- format => "json_batch"
- codec => "json"
- url => "http://parseable:8000/api/v1/ingest"
- headers => {
- "Authorization" => "Basic YWRtaW46YWRtaW4="
- "x-p-stream" => "logstash"
- }
- http_method => "post"
- http_compression => false
- automatic_retries => 5
- retry_non_idempotent => true
- connect_timeout => 30
- keepalive => false
- content_type => "application/json"
- }
- }
+ sources:
+ # vector_metrics:
+ # type: internal_metrics
+ raw_docker_logs:
+ type: docker_logs
+ transforms:
+ docker_logs:
+ type: remap
+ inputs:
+ - raw_docker_logs
+ source: |
+ if includes(["authelia", "caddy"], .container_name) {
+ . |= object!(parse_json!(.message))
+ } else if .container_name == "paperless" {
+ # asctime has trailing milliseconds, which I can't figure out
+ # how to parse, but I also don't care about it, so drop it
+ parsed = parse_regex!(
+ .message,
+ r'\[(?P<asctime>.*?),\d*\] \[(?P<level>.*?)\] \[(?P<name>.*?)\] (?P<message>.*)',
+ )
+ .paperless_time = parse_timestamp!(del(parsed.asctime), format: "%F %T")
+ . |= parsed
+ } else if includes(["grafana", "loki"], .container_name) {
+ . |= parse_key_value!(.message)
+ }
+ sinks:
+ # console:
+ # type: console
+ # inputs:
+ # - demo_logs
+ # encoding:
+ # codec: json
+ parseable:
+ type: http
+ method: post
+ batch:
+ max_bytes: 10485760
+ max_events: 1000
+ timeout_secs: 10
+ compression: gzip
+ inputs:
+ - docker_logs
+ encoding:
+ codec: json
+ uri: http://parseable:8000/api/v1/ingest
+ auth:
+ strategy: basic
+ user: admin
+ password: admin
+ request:
+ headers:
+ X-P-Stream: vector
+ healthcheck:
+ enabled: true
+ path: http://parseable/api/v1/liveness
+ port: 8000
+ # prometheus:
+ # type: prometheus_remote_write
+ # endpoint: http://prometheus:9090
+ # inputs:
+ # - vector_metrics
mode: "0644"
- name: Get docker network
@@ -64,6 +106,7 @@
P_OIDC_CLIENT_SECRET: "{{ parseable.oidc_secret }}"
P_OIDC_ISSUER: https://auth.{{ domain }}
P_ORIGIN_URI: https://logs.{{ domain }}
+ # RUST_LOG: warning
volumes:
- ./data:/parseable/data
- ./staging:/parseable/staging
@@ -71,16 +114,16 @@
networks:
- name: lotus_land_story
- - name: Run Logstash
+ - name: Run Vector
community.docker.docker_container:
restart: true
- name: logstash
- image: docker.elastic.co/logstash/logstash-oss:8.12.0
- command: logstash -f /etc/logstash/conf.d/logstash.conf
+ name: vector
+ image: timberio/vector:0.35.0-alpine
env:
- LS_JAVA_OPTS: "-Xmx256m -Xms256m"
+ # VECTOR_LOG: debug
volumes:
- - /mnt/lotus-land-story/parseable/logstash.conf:/etc/logstash/conf.d/logstash.conf
+ - /mnt/lotus-land-story/parseable/vector.yml:/etc/vector/vector.yaml
+ - /var/run/docker.sock:/var/run/docker.sock # for docker_logs
restart_policy: unless-stopped
networks:
- name: lotus_land_story
diff --git a/lotus-land-story/loki.yml b/lotus-land-story/loki.yml
index d66cabc..7bc5b98 100644
--- a/lotus-land-story/loki.yml
+++ b/lotus-land-story/loki.yml
@@ -95,14 +95,14 @@
job: varlogs
__path__: /var/log/*.log
__path_exclude__: /var/log/syslog
- - job_name: docker
- docker_sd_configs:
- - host: unix:///var/run/docker.sock
- refresh_interval: 5s
- relabel_configs:
- - source_labels: ['__meta_docker_container_name']
- regex: '/(.*)'
- target_label: 'container'
+ # - job_name: docker
+ # docker_sd_configs:
+ # - host: unix:///var/run/docker.sock
+ # refresh_interval: 5s
+ # relabel_configs:
+ # - source_labels: ['__meta_docker_container_name']
+ # regex: '/(.*)'
+ # target_label: 'container'
- job_name: syslog
syslog:
listen_address: 0.0.0.0:514