1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
sources:
  docker:
    type: docker_logs
  varlogs:
    type: file
    include:
      - /var/log/fail2ban.log

transforms:
  docker_parsed:
    type: remap
    inputs:
      - docker
    source: |
      ._stream_id = .container_id

      ._time = del(.timestamp)

      if .container_name == "anubis" {
        parsed, err = parse_json(.message)
        if err == null {
          . = merge!(., parsed)
          ._time = del(.time)
          ._msg = del(.msg)
        }
      } else if .container_name == "caddy" {
        # Startup errors and panics go to stderr as plain text.
        parsed, err = parse_json(.message)
        if err == null && is_object(parsed) {
          . = merge!(., parsed)
          ._time = del(.ts)
          ._msg = del(.msg)
        }
      } else if .container_name == "gitea" {
        # TODO handle router logs: https://github.com/go-gitea/gitea/pull/33814
        # gitea's common log doesn't match the vector common log because of the referer and user agent
        router, router_err = parse_regex(
            .message,
            r'^(?P<remote_host>.*?) - (?P<identity>.*?) \[(?P<datetime>.*?)\] "(?P<request>.*?)" (?P<status>.*?) (?P<size>.*?) "(?P<referer>.*?)" "(?P<user_agent>.*?)"$',
        )
        if router_err == null {
            . = merge(., router)

            request = parse_regex!(.request, r'^(?P<method>.*?) (?P<request_uri>.*?) (?P<proto>.*?)$')
            . = merge(., request)

            ._time = parse_timestamp!(del(.datetime), format: "%d/%b/%Y:%H:%M:%S %z")
            ._msg = del(.request)
        }
      } else if .container_name == "gotosocial" {
        . = merge(., parse_key_value!(.message))
        ._time = parse_timestamp!(del(.timestamp), format: "%d/%m/%Y %H:%M:%S%.3f")
        ._msg = del(.msg)
      } else if .container_name == "grafana" {
        . = merge(., parse_key_value!(.message))
        ._time = del(.t)
        ._msg = del(.msg)
      } else if .container_name == "miniflux" {
        . = merge!(., parse_json!(.message))
        ._msg = del(.msg)
      }

      if ._msg == null {
        ._msg = .message
      }
      del(.message)

  varlogs_parsed:
    type: remap
    inputs:
      - varlogs
    source: |
      parsed = parse_regex!(del(.message), r'^(?P<date>.*?) (?P<time>.*?) (?P<component>.*?) *\[(?P<proc_id>.*?)\]: (?P<level>.*?) +(?P<message>.*?)$')
      . = merge(., parsed)

      ._msg = del(.message)

      t = join!([del(.date), del(.time)], separator: " ")
      ._time = parse_timestamp!(t, format: "%Y-%m-%d %H:%M:%S,%3f")

      ._stream_id = .file

  # A subset of gitea logs from docker that contain SSH information
  gitea_ssh:
    type: remap
    inputs:
      - docker
    source: |
      if .container_name == "gitea" && contains(string!(.message), "[preauth]") {
        . = join!(
          [
            format_timestamp!(now(), "%b %d %H:%M:%S"),
            .message,
          ],
          separator: " ",
        )
      } else {
        . = []
      }

sinks:
  # console:
  #   inputs:
  #     - gitea_ssh
  #   type: console
  #   encoding:
  #     codec: csv
  #     csv:
  #       fields:
  #         - message
  #       quote_style: never

  victoria_logs:
    inputs:
      - docker_parsed
      - varlogs_parsed
    type: http
    uri: http://victoria-logs:9428/insert/jsonline?_stream_fields=container_id,container_name,file
    compression: gzip
    encoding:
      codec: json
    framing:
      method: newline_delimited
    healthcheck:
      enabled: false

  # Since ssh logs from gitea only show up in docker logs and not gitea logs,
  # ship them to /var/log for use with fail2ban
  gitea_log:
    inputs:
      - gitea_ssh
    type: file
    path: /var/log/gitea.log
    encoding:
      # Surely there must be a better way to do unstructured logging than using
      # CSV rows with a single field?
      codec: csv
      csv:
        fields:
          - message
        quote_style: never