rm loki and prometheus
diff --git a/lotus-land-story/Rakefile b/lotus-land-story/Rakefile
index 1d64b47..470c3c7 100644
--- a/lotus-land-story/Rakefile
+++ b/lotus-land-story/Rakefile
@@ -42,7 +42,6 @@ task "vars.yml" do |t|
linode_volume = `terraform output -raw lotus_land_story_volume`
miniflux_db_password = `op read "op://Private/Miniflux/db password"`.strip
miniflux_password = `op read op://Private/Miniflux/password`.strip
- prometheus_linode_api_token = `op read "op://Private/lotus-land-story/prometheus/linode api token"`.strip
File.write(t.name, YAML.dump({
"domain" => DOMAIN,
diff --git a/lotus-land-story/gotosocial.yml b/lotus-land-story/gotosocial.yml
index 2873653..bf33feb 100644
--- a/lotus-land-story/gotosocial.yml
+++ b/lotus-land-story/gotosocial.yml
@@ -72,8 +72,8 @@
GTS_OIDC_ADMIN_GROUPS: admin
GTS_METRICS_ENABLED: "true"
GTS_METRICS_AUTH_ENABLED: "true"
- GTS_METRICS_AUTH_USERNAME: "{{ prometheus.gotosocial.username }}"
- GTS_METRICS_AUTH_PASSWORD: "{{ prometheus.gotosocial.password }}"
+ GTS_METRICS_AUTH_USERNAME: "{{ gotosocial.metrics.username }}"
+ GTS_METRICS_AUTH_PASSWORD: "{{ gotosocial.metrics.password }}"
# GTS_ADVANCED_RATE_LIMIT_REQUESTS: "0"
TZ: UTC
volumes:
diff --git a/lotus-land-story/grafana.yml b/lotus-land-story/grafana.yml
index baca2f1..a567cd5 100644
--- a/lotus-land-story/grafana.yml
+++ b/lotus-land-story/grafana.yml
@@ -64,18 +64,10 @@
content: |
apiVersion: 1
- datasources:
+ deleteDatasources:
- name: Prometheus
- type: prometheus
- # Access mode - proxy (server in the UI) or direct (browser in the UI).
- access: proxy
- url: http://prometheus:9090
- jsonData:
- httpMethod: POST
- manageAlerts: true
- prometheusType: Prometheus
- prometheusVersion: 2.37.0
mode: "0644"
+ state: absent
- name: Provision VictoriaMetrics
ansible.builtin.copy:
diff --git a/lotus-land-story/loki.yml b/lotus-land-story/loki.yml
deleted file mode 100644
index 1a1baaa..0000000
--- a/lotus-land-story/loki.yml
+++ /dev/null
@@ -1,197 +0,0 @@
-# https://raw.githubusercontent.com/grafana/loki/v2.8.0/production/docker-compose.yaml
-# https://grafana.com/docs/loki/latest/clients/docker-driver/
-
-- name: Set up Loki
- hosts: lotus-land-story
- vars:
- version: 3.2.0
- tasks:
-
- - name: Provision Grafana
- ansible.builtin.copy:
- dest: /mnt/lotus-land-story/grafana/provisioning/datasources/loki.yml
- content: |
- apiVersion: 1
- deleteDatasources:
- - name: Loki
- orgId: 1
- prune: true
- mode: "0644"
-
- - name: Create Loki config dirs
- ansible.builtin.file:
- path: /mnt/lotus-land-story/loki/{{ item }}
- state: directory
- owner: 10001
- group: 10001
- mode: "0755"
- loop:
- - config
- - data
-
- - name: Configure Loki
- ansible.builtin.copy:
- dest: /mnt/lotus-land-story/loki/config/loki.yml
- content: |
- auth_enabled: false
- server:
- http_listen_port: 3100
- common:
- path_prefix: /loki
- storage:
- filesystem:
- chunks_directory: /loki/chunks
- rules_directory: /loki/rules
- replication_factor: 1
- ring:
- kvstore:
- store: inmemory
- compactor:
- retention_enabled: true
- delete_request_store: filesystem
- limits_config:
- retention_period: 168h
- volume_enabled: true
- pattern_ingester: # for Explore Logs
- enabled: true
- schema_config:
- configs:
- - from: 2020-10-24
- store: boltdb-shipper
- object_store: filesystem
- schema: v11
- index:
- prefix: index_
- period: 24h
-
- # https://grafana.com/docs/loki/latest/operations/storage/tsdb/
- - from: 2023-07-17
- index:
- period: 24h
- prefix: index_
- object_store: filesystem
- schema: v12
- store: tsdb
-
- - from: 2024-09-29
- index:
- period: 24h
- prefix: index_
- object_store: filesystem
- schema: v13
- store: tsdb
-
- storage_config:
- filesystem:
- directory: /data
- ruler:
- alertmanager_url: http://localhost:9093
- owner: 10001
- group: 10001
- mode: "0644"
-
- - name: Run Loki
- community.docker.docker_container:
- state: absent
- name: loki
- image: grafana/loki:{{ version }}
- restart: true
- command: -config.file=/mnt/config/loki.yml
- ports:
- - "3100:3100"
- volumes:
- - /mnt/lotus-land-story/loki/config:/mnt/config
- - /mnt/lotus-land-story/loki/data:/data
- restart_policy: unless-stopped
- networks:
- - name: lotus_land_story
- etc_hosts:
- host.docker.internal: host-gateway
-
-- name: Set up Promtail
- tags: promtail
- hosts: lotus-land-story
- vars:
- version: 3.2.0
- tasks:
-
- # https://github.com/grafana/loki/issues/2361
- - name: Configure Promtail
- ansible.builtin.copy:
- dest: /mnt/lotus-land-story/loki/config/promtail.yml
- content: |
- server:
- http_listen_port: 9080
- grpc_listen_port: 0
- positions:
- filename: /tmp/positions.yaml
- clients:
- - url: http://loki:3100/loki/api/v1/push
- scrape_configs:
- - job_name: system
- static_configs:
- - targets:
- - localhost
- labels:
- job: varlogs
- __path__: /var/log/*.log
- __path_exclude__: /var/log/syslog
- # - job_name: syslog
- # syslog:
- # listen_address: 0.0.0.0:514
- # listen_protocol: tcp
- # idle_timeout: 60s
- # label_structured_data: yes
- # labels:
- # job: syslog
- # relabel_configs:
- # - source_labels: ['__syslog_message_hostname']
- # target_label: 'host'
- # - source_labels: ['__syslog_message_severity']
- # target_label: level
- # - source_labels: ['__syslog_message_facility']
- # target_label: syslog_facility
- # - source_labels: ['__syslog_message_app_name']
- # target_label: syslog_identifier
- mode: "0644"
-
- - name: Run Promtail
- community.docker.docker_container:
- state: absent
- name: promtail
- image: grafana/promtail:{{ version }}
- restart: true
- command: -config.file=/mnt/config/promtail.yml
- # ports:
- # - "514:514"
- volumes:
- - /var/log:/var/log
- - /var/run/docker.sock:/var/run/docker.sock
- - /mnt/lotus-land-story/loki/config:/mnt/config
- restart_policy: unless-stopped
- networks:
- - name: lotus_land_story
- etc_hosts:
- host.docker.internal: host-gateway
-
- - name: Ship to promtail using rsyslog
- ansible.builtin.copy:
- content: |
- *.* action(type="omfwd"
- protocol="tcp"
- target="127.0.0.1"
- port="514"
- Template="RSYSLOG_SyslogProtocol23Format"
- TCP_Framing="octet-counted"
- KeepAlive="on")
- dest: /etc/rsyslog.d/50-promtail.conf
- mode: "0644"
- notify: Restart rsyslog
-
- handlers:
- - name: Restart rsyslog
- ansible.builtin.service:
- name: rsyslog
- state: restarted
-
-# vim: ft=yaml.ansible
diff --git a/lotus-land-story/main.tf b/lotus-land-story/main.tf
index 161bf0c..80cb4d5 100644
--- a/lotus-land-story/main.tf
+++ b/lotus-land-story/main.tf
@@ -60,19 +60,10 @@ resource "linode_domain_record" "subdomains" {
for_each = toset([
"auth",
- "loki",
- "prometheus",
])
name = each.key
}
-resource "linode_domain_record" "prometheus" {
- domain_id = data.linode_domain.domain.id
- name = "prometheus"
- record_type = "A"
- target = resource.linode_instance.lotus_land_story.ip_address
-}
-
output "lotus_land_story_ip" {
value = resource.linode_instance.lotus_land_story.ip_address
}
diff --git a/lotus-land-story/main.yml b/lotus-land-story/main.yml
index 0633198..564c7f3 100644
--- a/lotus-land-story/main.yml
+++ b/lotus-land-story/main.yml
@@ -48,12 +48,10 @@
- import_playbook: golink.yml # noqa: name[play]
- import_playbook: grafana.yml # noqa: name[play]
-- import_playbook: loki.yml # noqa: name[play]
- import_playbook: miniflux.yml # noqa: name[play]
- import_playbook: gotosocial.yml # noqa: name[play]
- import_playbook: caddy.yml # noqa: name[play]
- import_playbook: security.yml # noqa: name[play]
- import_playbook: backup.yml # noqa: name[play]
-- import_playbook: prometheus.yml # noqa: name[play]
# vim: ft=yaml.ansible
diff --git a/lotus-land-story/metrics.yml b/lotus-land-story/metrics.yml
index 83e7cc0..ff8e745 100644
--- a/lotus-land-story/metrics.yml
+++ b/lotus-land-story/metrics.yml
@@ -3,16 +3,20 @@
vars_files:
- vars.yml
vars:
- version: v1.119.0
+ version: v1.121.0
tasks:
- name: Make directories
ansible.builtin.file:
- path: /mnt/lotus-land-story/{{ item }}
+ path: /mnt/lotus-land-story/victoria-metrics/data
state: directory
mode: "0755"
- loop:
- - victoria-metrics
+
+ - name: Configure scrape.yml
+ ansible.builtin.template:
+ dest: /mnt/lotus-land-story/victoria-metrics/scrape.yml
+ src: templates/scrape.yml
+ mode: "0600"
- name: Get docker network
community.docker.docker_network:
@@ -24,12 +28,18 @@
restart: true
name: victoria-metrics
image: victoriametrics/victoria-metrics:{{ version }}
- command: --selfScrapeInterval=5s -storageDataPath=victoria-metrics-data
+ command:
+ - --selfScrapeInterval=5s
+ - -storageDataPath=victoria-metrics-data
+ - -promscrape.config=scrape.yml
restart_policy: unless-stopped
volumes:
- - /mnt/lotus-land-story/victoria-metrics:/victoria-metrics-data
+ - /mnt/lotus-land-story/victoria-metrics/data:/victoria-metrics-data
+ - /mnt/lotus-land-story/victoria-metrics/scrape.yml:/scrape.yml
networks:
- name: lotus_land_story
+ etc_hosts:
+ host.docker.internal: host-gateway
handlers:
- name: Import restarts
diff --git a/lotus-land-story/prometheus.yml b/lotus-land-story/prometheus.yml
deleted file mode 100644
index 5b74312..0000000
--- a/lotus-land-story/prometheus.yml
+++ /dev/null
@@ -1,183 +0,0 @@
-- name: Set up Prometheus
- hosts: lotus-land-story
- vars_files:
- - vars.yml
- vars:
- prometheus_version: v3.4.1
- cadvisor_version: v0.52.1
- tasks:
-
- - name: Create Prometheus dir
- ansible.builtin.file:
- path: /mnt/lotus-land-story/prometheus
- state: directory
- owner: prometheus
- group: prometheus
- mode: "0755"
-
- - name: Prometheus config
- ansible.builtin.template:
- dest: /mnt/lotus-land-story/prometheus/prometheus.yml
- src: templates/prometheus.yml
- owner: prometheus
- group: prometheus
- mode: "0600"
-
- - name: Create Prometheus volume
- community.docker.docker_volume:
- name: prometheus
-
- - name: Get prometheus user info
- ansible.builtin.user:
- name: prometheus
- register: prometheus_user
-
- - name: Run Prometheus
- community.docker.docker_container:
- name: prometheus
- image: prom/prometheus:{{ prometheus_version }}
- command:
- - --config.file=/etc/prometheus/prometheus.yml
- - --storage.tsdb.retention.size=5GB
- - --log.format=json
- restart: true
- user: "{{ prometheus_user.uid }}"
- groups: "{{ prometheus_user.group }}"
- volumes:
- - /mnt/lotus-land-story/prometheus:/etc/prometheus
- - prometheus:/prometheus
- restart_policy: unless-stopped
- networks:
- - name: lotus_land_story
- etc_hosts:
- host.docker.internal: host-gateway
-
- - name: Run cAdvisor
- tags: cadvisor
- community.docker.docker_container:
- name: cadvisor
- image: gcr.io/cadvisor/cadvisor:{{ cadvisor_version }}
- restart: true
- command:
- #https://github.com/google/cadvisor/issues/2523#issuecomment-2258042352
- - "--housekeeping_interval=30s"
- - "--docker_only=true"
- - "--store_container_labels=false"
- volumes:
- - /:/rootfs:ro
- - /var/run:/var/run:rw
- - /sys:/sys:ro
- - /var/lib/docker/:/var/lib/docker:ro
- - /etc/machine-id:/etc/machine-id:ro
- restart_policy: unless-stopped
- networks:
- - name: lotus_land_story
-
-- name: Set up postgres-exporter
- hosts: lotus-land-story
- vars_files:
- - vars.yml
- vars:
- version: v0.17.0
- tags:
- - postgres-exporter
- tasks:
- # https://github.com/prometheus-community/postgres_exporter?tab=readme-ov-file#running-as-non-superuser
- - name: Set up postgres_exporter user
- become: true
- become_user: postgres
- community.postgresql.postgresql_query:
- query: |
- -- To use IF statements, hence to be able to check if the user exists before
- -- attempting creation, we need to switch to procedural SQL (PL/pgSQL)
- -- instead of standard SQL.
- -- More: https://www.postgresql.org/docs/9.3/plpgsql-overview.html
- -- To preserve compatibility with <9.0, DO blocks are not used; instead,
- -- a function is created and dropped.
- CREATE OR REPLACE FUNCTION __tmp_create_user() returns void as $$
- BEGIN
- IF NOT EXISTS (
- SELECT -- SELECT list can stay empty for this
- FROM pg_catalog.pg_user
- WHERE usename = 'postgres_exporter') THEN
- CREATE USER postgres_exporter;
- END IF;
- END;
- $$ language plpgsql;
-
- SELECT __tmp_create_user();
- DROP FUNCTION __tmp_create_user();
-
- ALTER USER postgres_exporter WITH PASSWORD '{{ prometheus.postgres_exporter.password }}';
- ALTER USER postgres_exporter SET SEARCH_PATH TO postgres_exporter,pg_catalog;
-
- -- If deploying as non-superuser (for example in AWS RDS), uncomment the GRANT
- -- line below and replace <MASTER_USER> with your root user.
- -- GRANT postgres_exporter TO <MASTER_USER>;
-
- GRANT CONNECT ON DATABASE postgres TO postgres_exporter;
- GRANT pg_monitor to postgres_exporter;
-
- - block:
- - name: Get docker network subnet
- community.docker.docker_network:
- name: lotus_land_story
- register: docker_network
- - name: Allow postgres_exporter to connect to postgres
- community.postgresql.postgresql_pg_hba:
- dest: /etc/postgresql/13/main/pg_hba.conf
- contype: host
- users: postgres_exporter
- source: "{{ docker_network.network.IPAM.Config[0].Subnet }}"
- databases: postgres
- create: true
- notify: Restart postgres
-
- # https://github.com/prometheus-community/postgres_exporter
- - name: Run postgres-exporter
- community.docker.docker_container:
- name: postgres-exporter
- image: quay.io/prometheuscommunity/postgres-exporter:{{ version }}
- restart: true
- env:
- DATA_SOURCE_URI: host.docker.internal:5432/postgres
- DATA_SOURCE_USER: postgres_exporter
- DATA_SOURCE_PASS: "{{ prometheus.postgres_exporter.password }}"
- restart_policy: unless-stopped
- networks:
- - name: lotus_land_story
- etc_hosts:
- host.docker.internal: host-gateway
-
- handlers:
- - name: Import restarts
- ansible.builtin.import_tasks: restarts.yml
-
-- name: Set up node-exporter
- hosts: lotus-land-story
- vars:
- version: v1.9.0
- tags:
- - node-exporter
- tasks:
-
- # https://github.com/prometheus/node_exporter?tab=readme-ov-file#docker
- - name: Run node-exporter
- community.docker.docker_container:
- image: quay.io/prometheus/node-exporter:{{ version }}
- name: node-exporter
- command:
- - "--path.rootfs=/host"
- network_mode: host
- pid_mode: host
- restart_policy: unless-stopped
- volumes:
- - "/:/host:ro,rslave"
- restart: true
-
- handlers:
- - name: Import restarts
- ansible.builtin.import_tasks: restarts.yml
-
-# yaml-language-server: $schema=https://raw.githubusercontent.com/ansible/ansible-lint/refs/heads/main/src/ansiblelint/schemas/ansible.json
-# vim: ft=yaml.ansible
diff --git a/lotus-land-story/templates/Caddyfile b/lotus-land-story/templates/Caddyfile
index 291ec9e..9a919f4 100644
--- a/lotus-land-story/templates/Caddyfile
+++ b/lotus-land-story/templates/Caddyfile
@@ -59,12 +59,6 @@ grafana {
}
}
-loki {
- tls internal
- bind tailscale/loki
- reverse_proxy loki:3100
-}
-
:2019 {
metrics /metrics
}
@@ -82,12 +76,6 @@ https://metrics {
reverse_proxy victoria-metrics:8428
}
-prometheus {
- tls internal
- bind tailscale/prometheus
- reverse_proxy prometheus:9090
-}
-
### Services
irc {
diff --git a/lotus-land-story/templates/prometheus.yml b/lotus-land-story/templates/scrape.yml
similarity index 70%
rename from lotus-land-story/templates/prometheus.yml
rename to lotus-land-story/templates/scrape.yml
index d589818..ae0f49c 100644
--- a/lotus-land-story/templates/prometheus.yml
+++ b/lotus-land-story/templates/scrape.yml
@@ -1,97 +1,70 @@
-global:
- # Attach these labels to any time series or alerts when communicating with
- # external systems (federation, remote storage, Alertmanager).
- # external_labels:
- # monitor: 'codelab-monitor'
-
scrape_configs:
- - job_name: prometheus
+
+ - job_name: anubis
+ static_configs:
+ - targets: ['anubis:9090']
+
+ - job_name: authelia
static_configs:
- - targets: ['localhost:9090']
+ - targets: ['authelia:9959']
- - job_name: node
+ - job_name: blackbox-exporter-dns
+ metrics_path: /probe
+ params:
+ module: [ dns ]
static_configs:
- targets:
- - host.docker.internal:9100
- - on-fire-within:9100
- - ramble-hard:9100
+ - 8.8.8.8
+ - 8.8.4.4
+ - 1.1.1.1
+ - 1.0.0.1
+ - 9.9.9.9
+ - 149.112.112.112
relabel_configs:
- source_labels: [__address__]
- regex: 'host\.docker\.internal:9100'
+ target_label: __param_target
+ - source_labels: [__param_target]
target_label: instance
- replacement: 'lotus-land-story:9100'
+ - source_labels: [__param_target]
+ target_label: target # for the dashboard
+ - target_label: __address__
+ replacement: mountain-of-faith:9115
- - job_name: docker
+ - job_name: cadvisor
+ scrape_interval: 5s
static_configs:
- - targets: ['host.docker.internal:9323']
+ - targets:
+ - cadvisor:8080
- job_name: caddy
static_configs:
- targets: ['caddy:2019']
- - job_name: grafana
- static_configs:
- - targets: ['grafana:3000']
-
- - job_name: postgres
- static_configs:
- - targets: ['postgres-exporter:9187']
-
- - job_name: miniflux
- static_configs:
- - targets: ['miniflux:8080']
-
- - job_name: authelia
+ - job_name: docker
static_configs:
- - targets: ['authelia:9959']
-
- # - job_name: linode
- # linode_sd_configs:
- # - authorization:
- # credentials: {{ prometheus.linode_api_token }}
- # relabel_configs:
- # # Use the public IPv6 address and port 9100 to scrape the target.
- # - source_labels: [__meta_linode_public_ipv6]
- # target_label: __address__
- # replacement: "[$1]:9100"
+ - targets: ['host.docker.internal:9323']
- - job_name: paperless
- metrics_path: /flower/metrics
+ - job_name: gitea
+ bearer_token: {{ gitea.metrics.token }}
static_configs:
- - targets: ['paperless-ngx:5555']
+ - targets: ['gitea:3000']
- job_name: gotosocial
metrics_path: /metrics
scheme: http
basic_auth:
- username: {{ prometheus.gotosocial.username }}
- password: {{ prometheus.gotosocial.password }}
+ username: {{ gotosocial.metrics.username }}
+ password: {{ gotosocial.metrics.password }}
static_configs:
- targets: ['gotosocial:8080']
- - job_name: cadvisor
- scrape_interval: 5s
- static_configs:
- - targets:
- - cadvisor:8080
-
- - job_name: gitea
- bearer_token: {{ gitea.metrics_token }}
- static_configs:
- - targets: ['gitea:3000']
-
- - job_name: victoria-logs
- static_configs:
- - targets: ['victoria-logs:9428']
-
- - job_name: anubis
+ - job_name: grafana
static_configs:
- - targets: ['anubis:9090']
+ - targets: ['grafana:3000']
- - job_name: syncthing
- bearer_token: {{ syncthing.api_key }}
+ - job_name: miniflux
static_configs:
- - targets: ['mountain-of-faith:8384']
+ - targets: ['miniflux:8080']
- job_name: minio-job
bearer_token: {{ minio.cluster_token }}
@@ -107,3 +80,32 @@ scrape_configs:
static_configs:
- targets: ['mountain-of-faith:9000']
+ - job_name: node-exporter
+ static_configs:
+ - targets:
+ - host.docker.internal:9100
+ - mountain-of-faith:9100
+ - ramble-hard:9100
+ relabel_configs:
+ - source_labels: [__address__]
+ regex: 'host\.docker\.internal:9100'
+ target_label: instance
+ replacement: 'lotus-land-story:9100'
+
+ - job_name: paperless
+ metrics_path: /flower/metrics
+ static_configs:
+ - targets: ['paperless-ngx:5555']
+
+ - job_name: postgres
+ static_configs:
+ - targets: ['postgres-exporter:9187']
+
+ - job_name: victoria-logs
+ static_configs:
+ - targets: ['victoria-logs:9428']
+
+ - job_name: syncthing
+ bearer_token: {{ syncthing.api_key }}
+ static_configs:
+ - targets: ['mountain-of-faith:8384']
diff --git a/lotus-land-story/templates/vector.yml b/lotus-land-story/templates/vector.yml
index edfdd67..47af969 100644
--- a/lotus-land-story/templates/vector.yml
+++ b/lotus-land-story/templates/vector.yml
@@ -49,17 +49,9 @@ transforms:
. = merge!(., parse_key_value!(.message))
._time = del(.t)
._msg = del(.msg)
- } else if .container_name == "loki" {
- . = merge!(., parse_key_value!(.message))
- ._time = del(.ts)
- ._msg = del(.msg)
} else if .container_name == "miniflux" {
. = merge!(., parse_json!(.message))
._msg = del(.msg)
- } else if .container_name == "prometheus" {
- . = merge!(., parse_json!(.message))
- ._time = del(.time)
- ._msg = del(.msg)
}
if ._msg == null {