o11y
diff --git a/lotus-land-story/Rakefile b/lotus-land-story/Rakefile
index da4c765..27c9d49 100644
--- a/lotus-land-story/Rakefile
+++ b/lotus-land-story/Rakefile
@@ -10,6 +10,7 @@ namespace :pip do
sh "pip-sync requirements.txt"
end
+ desc ""
task :upgrade do
out, _ = Open3.capture2e("pip-compile --upgrade --resolver=backtracking requirements.in")
File.write("requirements.txt", out)
@@ -21,10 +22,12 @@ namespace :pip do
end
end
+desc ""
task terraform: "terraform.tfvars" do
sh "terraform apply"
end
+desc ""
task ansible: "vars.yml" do
sh "ansible-playbook main.yml"
end
diff --git a/lotus-land-story/main.tf b/lotus-land-story/main.tf
index 1cab02c..cfc9bd7 100644
--- a/lotus-land-story/main.tf
+++ b/lotus-land-story/main.tf
@@ -52,13 +52,21 @@ data "linode_domain" "domain" {
domain = var.domain
}
-resource "linode_domain_record" "rww" {
- domain_id = data.linode_domain.domain.id
- name = "rss"
- record_type = "A"
- target = resource.linode_instance.lotus_land_story.ip_address
+resource "linode_domain_record" "subdomains" {
+ domain_id = data.linode_domain.domain.id
+ record_type = "A"
+ target = resource.linode_instance.lotus_land_story.ip_address
+
+ for_each = toset(["rss", "prometheus", "grafana"])
+ name = each.key
}
+resource "linode_domain_record" "prometheus" {
+ domain_id = data.linode_domain.domain.id
+ name = "prometheus"
+ record_type = "A"
+ target = resource.linode_instance.lotus_land_story.ip_address
+}
output "lotus_land_story_ip" {
value = resource.linode_instance.lotus_land_story.ip_address
diff --git a/lotus-land-story/main.yml b/lotus-land-story/main.yml
index 1b51e11..fd34eed 100644
--- a/lotus-land-story/main.yml
+++ b/lotus-land-story/main.yml
@@ -79,23 +79,40 @@
- docker-buildx-plugin
- docker-compose-plugin
update_cache: true
-
- - name: Set up postgres
- become: true
- become_user: postgres
- block:
-
- name: Get docker0 IP address
ansible.builtin.shell: ip -4 -o addr show docker0 | awk '{print $4}' # noqa: risky-shell-pipe
vars:
executable: /usr/bin/bash
register: docker_ip
changed_when: docker_ip.rc != 0
+ - name: Set docker ip fact
+ ansible.builtin.set_fact:
+ docker_ip:
+ cidr: "{{ docker_ip.stdout }}"
+ address: "{{ docker_ip.stdout | ansible.utils.ipaddr('address') }}"
+ - name: Create Docker volume location
+ ansible.builtin.file:
+ path: /mnt/lotus-land-story/docker
+ state: directory
+ mode: "0755"
+ - name: Docker config
+ ansible.builtin.template:
+ src: templates/daemon.json
+ dest: /etc/docker/daemon.json
+ mode: "0644"
+ notify: Restart docker
+
+
+ - name: Set up postgres
+ become: true
+ become_user: postgres
+ block:
+
- name: Listen on docker0 interface
ansible.builtin.lineinfile:
dest: "/etc/postgresql/13/main/conf.d/listen.conf"
regexp: '^#?listen_addresses='
- line: "listen_addresses='localhost,{{ docker_ip.stdout | ansible.utils.ipaddr('address') }}'"
+ line: "listen_addresses='localhost,{{ docker_ip.address }}'"
state: present
create: true
mode: "0644"
@@ -132,6 +149,48 @@
db: miniflux
notify: Restart postgres
+ - name: Set up postgres-exporter
+ block:
+ - name: Get postgres roles
+ community.postgresql.postgresql_info:
+ filter: roles
+ register: postgres_info
+ - name: Add postgres permissions for postgres-exporter
+ community.postgresql.postgresql_query:
+ query: |
+ CREATE USER prometheus;
+ ALTER USER prometheus SET SEARCH_PATH TO prometheus,pg_catalog;
+
+ CREATE SCHEMA prometheus AUTHORIZATION prometheus;
+
+ CREATE FUNCTION prometheus.f_select_pg_stat_activity()
+ RETURNS setof pg_catalog.pg_stat_activity
+ LANGUAGE sql
+ SECURITY DEFINER
+ AS $$
+ SELECT * from pg_catalog.pg_stat_activity;
+ $$;
+
+ CREATE FUNCTION prometheus.f_select_pg_stat_replication()
+ RETURNS setof pg_catalog.pg_stat_replication
+ LANGUAGE sql
+ SECURITY DEFINER
+ AS $$
+ SELECT * from pg_catalog.pg_stat_replication;
+ $$;
+
+ CREATE VIEW prometheus.pg_stat_replication
+ AS
+ SELECT * FROM prometheus.f_select_pg_stat_replication();
+
+ CREATE VIEW prometheus.pg_stat_activity
+ AS
+ SELECT * FROM prometheus.f_select_pg_stat_activity();
+
+ GRANT SELECT ON prometheus.pg_stat_replication TO prometheus;
+ GRANT SELECT ON prometheus.pg_stat_activity TO prometheus;
+ when: "'prometheus' not in postgres_info.roles"
+
- name: Run miniflux
community.docker.docker_compose:
project_name: miniflux
@@ -149,8 +208,133 @@
- ADMIN_USERNAME=alpha
- ADMIN_PASSWORD={{ miniflux_password }}
- BASE_URL=https://rss.{{ domain }}
+ restart: unless-stopped
+ extra_hosts:
+ - "host.docker.internal:host-gateway"
+
+ - name: Install host exporters
+ ansible.builtin.apt:
+ pkg:
+ - prometheus-node-exporter
+ - prometheus-postgres-exporter
+ state: present
+ # /usr/share/doc/prometheus-postgres-exporter/README.Debian
+ - name: Configure postgres-exporter
+ ansible.builtin.lineinfile:
+ dest: /etc/default/prometheus-postgres-exporter
+ regexp: '^DATA_SOURCE_NAME='
+ line: "DATA_SOURCE_NAME='user=prometheus host=/run/postgresql dbname=postgres'"
+ state: present
+ - name: Create Prometheus dir
+ ansible.builtin.file:
+ path: /mnt/lotus-land-story/prometheus
+ state: directory
+ mode: "0755"
+ - name: Prometheus config
+ ansible.builtin.copy:
+ dest: /mnt/lotus-land-story/prometheus/prometheus.yml
+ content: |
+ global:
+ # Attach these labels to any time series or alerts when communicating with
+ # external systems (federation, remote storage, Alertmanager).
+ external_labels:
+ monitor: 'codelab-monitor'
+
+ scrape_configs:
+ - job_name: 'prometheus'
+ static_configs:
+ - targets: ['localhost:9090']
+
+ - job_name: node
+ static_configs:
+ - targets: ['host.docker.internal:9100']
+
+ - job_name: 'docker'
+ static_configs:
+ - targets: ['host.docker.internal:9323']
+
+ - job_name: caddy
+ static_configs:
+ - targets: ['host.docker.internal:2019']
+
+ - job_name: 'grafana'
+ static_configs:
+ - targets: ['host.docker.internal:3000']
+
+ - job_name: 'postgres'
+ static_configs:
+ - targets: ['host.docker.internal:9187']
+ mode: "0644"
+ - name: Run Prometheus
+ community.docker.docker_compose:
+ project_name: prometheus
+ remove_orphans: true
+ definition:
+ version: "3.3"
+ services:
+ prometheus:
+ image: prom/prometheus:latest
+ ports:
+ - "9090:9090"
+ volumes:
+ - /mnt/lotus-land-story/prometheus:/etc/prometheus
+ restart: unless-stopped
+ extra_hosts:
+ - "host.docker.internal:host-gateway"
+
+ - name: Create Grafana dir
+ ansible.builtin.file:
+ path: /mnt/lotus-land-story/grafana/provisioning/{{ item }}
+ state: directory
+ mode: "0755"
+ loop:
+ - datasources
+ - name: Configure grafana
+ ansible.builtin.copy:
+ dest: /mnt/lotus-land-story/grafana/grafana.ini
+ content: |
+ [metrics]
+ enabled = true
+ disable_total_stats = false
+ mode: "0644"
+ - name: Provision Grafana
+ ansible.builtin.copy:
+ dest: /mnt/lotus-land-story/grafana/provisioning/datasources/prometheus.yml
+ content: |
+ apiVersion: 1
+
+ datasources:
+ - name: Prometheus
+ type: prometheus
+ # Access mode - proxy (server in the UI) or direct (browser in the UI).
+ access: proxy
+ url: http://host.docker.internal:9090
+ jsonData:
+ httpMethod: POST
+ manageAlerts: true
+ prometheusType: Prometheus
+ prometheusVersion: 2.37.0
+ mode: "0644"
+ - name: Run Grafana
+ community.docker.docker_compose:
+ project_name: grafana
+ remove_orphans: true
+ definition:
+ version: "3.3"
+ services:
+ grafana:
+ image: grafana/grafana-oss:latest
+ ports:
+ - "3000:3000"
+ volumes:
+ - /mnt/lotus-land-story/grafana/grafana.ini:/etc/grafana/grafana.ini
+ - /mnt/lotus-land-story/grafana/provisioning:/etc/grafana/provisioning
+ - grafana:/var/lib/grafana
+ restart: unless-stopped
extra_hosts:
- "host.docker.internal:host-gateway"
+ volumes:
+ grafana:
- name: Make /mnt/lotus-land-story/caddy
ansible.builtin.file:
@@ -160,7 +344,6 @@
loop:
- caddy
- caddy/data
- - caddy/config
- name: Set up Caddyfile
ansible.builtin.template:
src: templates/Caddyfile
@@ -178,12 +361,16 @@
ports:
- "80:80"
- "443:443"
+ - "2019:2019"
volumes:
- /mnt/lotus-land-story/caddy/Caddyfile:/etc/caddy/Caddyfile
- /mnt/lotus-land-story/caddy/data:/data
- - /mnt/lotus-land-story/caddy/config:/config
+ - caddy-config:/config
+ restart: unless-stopped
extra_hosts:
- "host.docker.internal:host-gateway"
+ volumes:
+ caddy-config:
handlers:
- name: Restart postgres
@@ -191,4 +378,9 @@
name: postgresql
state: restarted
+ - name: Restart docker
+ ansible.builtin.service:
+ name: docker
+ state: restarted
+
# vim: ft=yaml.ansible
diff --git a/lotus-land-story/templates/Caddyfile b/lotus-land-story/templates/Caddyfile
index f36e473..4bd1534 100644
--- a/lotus-land-story/templates/Caddyfile
+++ b/lotus-land-story/templates/Caddyfile
@@ -1,2 +1,15 @@
-rss.{{ domain }}
-reverse_proxy host.docker.internal:8080
+:2019 {
+ metrics
+}
+
+rss.{{ domain }} {
+ reverse_proxy host.docker.internal:8080
+}
+
+prometheus.{{ domain }} {
+ reverse_proxy host.docker.internal:9090
+}
+
+grafana.{{ domain }} {
+ reverse_proxy host.docker.internal:3000
+}
diff --git a/lotus-land-story/templates/daemon.json b/lotus-land-story/templates/daemon.json
new file mode 100644
index 0000000..ba28dd9
--- /dev/null
+++ b/lotus-land-story/templates/daemon.json
@@ -0,0 +1,4 @@
+{
+ "data-root": "/mnt/lotus-land-story/docker",
+ "metrics-addr": "{{ docker_ip.address }}:9323"
+}