Add list sources with manual sync
Lists now get their items from sources. Each source kind is a class in a hardcoded registry that turns a snapshot into entries, and Sync reconciles a source's items with those entries: new entries become items, and items that disappear are archived and hidden rather than deleted, so their results survive if they come back. The textarea is the first source kind (text), and existing lists are backfilled with one. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01LG4iASZnHb1yEM1JAASiWn
diff --git a/AGENTS.md b/AGENTS.md
index 2f2bae8..ecac0cf 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -21,7 +21,10 @@ db/migrate/ — Sequel migrations
lib/rank_king/ — app code
config.rb — Config data class, reads from ENV
db.rb — Sequel connection + migration runner
- models.rb — Sequel models (List, Item, Axis, Result)
+ models.rb — Sequel models (List, Item, Source, Axis, Result)
+ sources.rb — hardcoded registry of source kinds (Sources::ALL)
+ sources/ — one class per source kind
+ sync.rb — reconciles a source's items with a fresh snapshot
web.rb — Roda routes
views/ — Phlex view components
lib/open_skill/ — OpenSkill rating library (pure Ruby)
@@ -32,8 +35,9 @@ public/ — static assets
## Domain model
-- **List** has many Items and Axes
-- **Item** belongs to a List
+- **List** has many Items, Sources, and Axes
+- **Source** belongs to a List and produces its Items; `kind` picks a class from `Sources::ALL`
+- **Item** belongs to a List and a Source; items missing from the latest sync are archived (`archived_at`) and hidden everywhere
- **Axis** belongs to a List, has many Results
- **Result** belongs to an Axis, a winner (Item), and a loser (Item)
diff --git a/db/migrate/002_create_sources.rb b/db/migrate/002_create_sources.rb
new file mode 100644
index 0000000..f1a905a
--- /dev/null
+++ b/db/migrate/002_create_sources.rb
@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+ up do
+ create_table(:sources) do
+ primary_key :id
+ foreign_key :list_id, :lists, null: false
+ String :kind, null: false
+ String :config, null: false, default: "{}"
+ DateTime :last_synced_at
+ DateTime :created_at
+ DateTime :updated_at
+ end
+
+ alter_table(:items) do
+ add_foreign_key :source_id, :sources
+ add_column :external_id, String
+ add_column :url, String
+ add_column :metadata, String, null: false, default: "{}"
+ add_column :archived_at, DateTime
+ add_index [:source_id, :external_id], unique: true
+ end
+
+ # Existing lists were created from the textarea, so give each one a text
+ # source. Duplicate names keep a nil external_id and are left alone by sync.
+ from(:lists).each do |list|
+ source_id = from(:sources).insert(
+ list_id: list[:id], kind: "text", config: "{}",
+ created_at: Time.now, updated_at: Time.now,
+ )
+ from(:items).where(list_id: list[:id]).update(source_id: source_id)
+
+ first_ids = from(:items).where(list_id: list[:id]).group(:name).select { min(:id) }
+ from(:items).where(id: first_ids).update(external_id: :name)
+ end
+ end
+
+ down do
+ alter_table(:items) do
+ drop_index [:source_id, :external_id]
+ drop_foreign_key :source_id
+ drop_column :external_id
+ drop_column :url
+ drop_column :metadata
+ drop_column :archived_at
+ end
+
+ drop_table(:sources)
+ end
+end
diff --git a/lib/rank_king/models.rb b/lib/rank_king/models.rb
index 6bc49b6..9fa838f 100644
--- a/lib/rank_king/models.rb
+++ b/lib/rank_king/models.rb
@@ -1,17 +1,32 @@
# frozen_string_literal: true
require_relative "db"
+require_relative "sources"
Sequel::Model.plugin :timestamps, update_on_create: true
module RankKing
class List < Sequel::Model(DB[:lists])
- one_to_many :items, class: "RankKing::Item"
+ # Archived items are hidden everywhere for now.
+ one_to_many :items, class: "RankKing::Item", conditions: { archived_at: nil }
one_to_many :axes, class: "RankKing::Axis"
+ one_to_many :sources, class: "RankKing::Source"
end
class Item < Sequel::Model(DB[:items])
+ plugin :serialization, :json, :metadata
+
+ many_to_one :list, class: "RankKing::List"
+ many_to_one :source, class: "RankKing::Source"
+ end
+
+ class Source < Sequel::Model(DB[:sources])
+ plugin :serialization, :json, :config
+
many_to_one :list, class: "RankKing::List"
+ one_to_many :items, class: "RankKing::Item"
+
+ def plugin = Sources.fetch(kind).new(config)
end
class Axis < Sequel::Model(DB[:axes])
diff --git a/lib/rank_king/sources.rb b/lib/rank_king/sources.rb
new file mode 100644
index 0000000..9e65d66
--- /dev/null
+++ b/lib/rank_king/sources.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module RankKing
+ module Sources
+ # What every source produces: one entry per item currently in the source.
+ # external_id is how sync recognizes the same item across snapshots.
+ Entry = Data.define(:external_id, :name, :url, :metadata) do
+ def initialize(external_id:, name:, url: nil, metadata: {})
+ super
+ end
+ end
+ end
+end
+
+require_relative "sources/text"
+
+module RankKing
+ module Sources
+ ALL = {
+ Text.kind => Text,
+ }.freeze
+
+ def self.fetch(kind) = ALL.fetch(kind)
+ end
+end
diff --git a/lib/rank_king/sources/text.rb b/lib/rank_king/sources/text.rb
new file mode 100644
index 0000000..875136a
--- /dev/null
+++ b/lib/rank_king/sources/text.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module RankKing
+ module Sources
+ # A push source: the snapshot is pasted text, one item per line.
+ class Text
+ def self.kind = "text"
+ def self.push? = true
+
+ def initialize(config = {})
+ @config = config
+ end
+
+ def entries(snapshot)
+ snapshot.to_s.lines.map(&:strip).reject(&:empty?).uniq.map {|line|
+ Entry.new(external_id: line, name: line)
+ }
+ end
+ end
+ end
+end
diff --git a/lib/rank_king/sync.rb b/lib/rank_king/sync.rb
new file mode 100644
index 0000000..02f07ce
--- /dev/null
+++ b/lib/rank_king/sync.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require_relative "models"
+
+module RankKing
+ # Reconciles a source's items with a fresh snapshot of entries. Items missing
+ # from the snapshot are archived rather than deleted so their results survive,
+ # and come back with their history if they reappear.
+ module Sync
+ Summary = Data.define(:added, :restored, :archived)
+
+ def self.call(source, entries, now: Time.now)
+ DB.transaction do
+ existing = source.items_dataset.exclude(external_id: nil).to_hash(:external_id)
+ added = restored = 0
+
+ entries.each do |entry|
+ attrs = { name: entry.name, url: entry.url, metadata: entry.metadata, archived_at: nil }
+ if (item = existing.delete(entry.external_id))
+ restored += 1 if item.archived_at
+ item.update(attrs)
+ else
+ Item.create(attrs.merge(list_id: source.list_id, source_id: source.id, external_id: entry.external_id))
+ added += 1
+ end
+ end
+
+ gone = existing.values.select {|item| item.archived_at.nil? }
+ gone.each {|item| item.update(archived_at: now) }
+
+ source.update(last_synced_at: now)
+ Summary.new(added:, restored:, archived: gone.size)
+ end
+ end
+ end
+end
diff --git a/lib/rank_king/views/list/show.rb b/lib/rank_king/views/list/show.rb
index 2629d5c..050855c 100644
--- a/lib/rank_king/views/list/show.rb
+++ b/lib/rank_king/views/list/show.rb
@@ -18,6 +18,7 @@ module RankKing
div(class: "list-show") do
h1 { @list.name }
render_items
+ render_sources
render_axes
render_matchup
render_ratings
@@ -38,6 +39,33 @@ module RankKing
end
end
+ def render_sources
+ section(class: "section") do
+ h2 { "Sources" }
+ @list.sources.each do |source|
+ render_source(source)
+ end
+ end
+ end
+
+ def render_source(source)
+ action = "/lists/#{@list.id}/sources/#{source.id}/sync"
+ form(method: "post", action:, class: "form") do
+ input(type: "hidden", name: "_csrf", value: @csrf.call(action))
+
+ if source.plugin.class.push?
+ div(class: "field") do
+ label(for: "snapshot-#{source.id}") { "#{source.kind.capitalize} (one per line)" }
+ textarea(name: "snapshot", id: "snapshot-#{source.id}", rows: 6) do
+ source.items_dataset.where(archived_at: nil).order(:id).map(:name).join("\n")
+ end
+ end
+ end
+
+ button(type: "submit", class: "btn-primary") { "Sync" }
+ end
+ end
+
def render_axes
section(class: "section") do
h2 { "Axes" }
diff --git a/lib/rank_king/web.rb b/lib/rank_king/web.rb
index 5a539ac..17fd33b 100644
--- a/lib/rank_king/web.rb
+++ b/lib/rank_king/web.rb
@@ -3,6 +3,7 @@
require "roda"
require_relative "models"
+require_relative "sync"
require_relative "views/home"
require_relative "views/list/show"
@@ -27,18 +28,16 @@ module RankKing
name = r.params["name"].to_s.strip
r.halt 422 if name.empty?
- items_text = r.params["items"].to_s.strip
- r.halt 422 if items_text.empty?
-
- item_names = items_text.lines.map(&:strip).reject(&:empty?)
- r.halt 422 if item_names.size < 2
+ entries = Sources::Text.new.entries(r.params["items"])
+ r.halt 422 if entries.size < 2
axes_text = r.params["axes"].to_s.strip
axis_names = axes_text.lines.map(&:strip).reject(&:empty?)
list = DB.transaction do
l = List.create(name: name)
- item_names.each { |n| Item.create(list_id: l.id, name: n) }
+ source = Source.create(list_id: l.id, kind: Sources::Text.kind)
+ Sync.call(source, entries)
axis_names.each { |n| Axis.create(list_id: l.id, name: n) }
l
end
@@ -57,6 +56,17 @@ module RankKing
end
end
+ r.on "sources", Integer, "sync" do |source_id|
+ r.post do
+ source = @list.sources_dataset.where(id: source_id).first
+ r.halt 404 unless source
+
+ Sync.call(source, source.plugin.entries(r.params["snapshot"]))
+
+ r.redirect "/lists/#{@list.id}"
+ end
+ end
+
r.on "results" do
r.post do
axis_id = r.params["axis_id"].to_i
diff --git a/test/test_sync.rb b/test/test_sync.rb
new file mode 100644
index 0000000..59c478d
--- /dev/null
+++ b/test/test_sync.rb
@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require_relative "test_helper"
+
+require "minitest/autorun"
+
+require_relative "../lib/rank_king/sync"
+
+class TestSync < Minitest::Test
+ include RankKing
+
+ def setup
+ DB[:results].delete
+ DB[:axes].delete
+ DB[:items].delete
+ DB[:sources].delete
+ DB[:lists].delete
+
+ @list = List.create(name: "Videos")
+ @source = Source.create(list_id: @list.id, kind: "text")
+ end
+
+ def test_adds_new_entries
+ summary = sync("A", "B")
+
+ assert_equal Sync::Summary.new(added: 2, restored: 0, archived: 0), summary
+ assert_equal %w[A B], names
+ refute_nil @source.reload.last_synced_at
+ end
+
+ def test_archives_missing_entries_and_keeps_their_rows
+ sync("A", "B")
+ summary = sync("A")
+
+ assert_equal 1, summary.archived
+ assert_equal %w[A], names
+ refute_nil Item.first(external_id: "B").archived_at
+ end
+
+ def test_restores_entries_that_come_back
+ sync("A", "B")
+ b = Item.first(external_id: "B")
+ sync("A")
+ summary = sync("A", "B")
+
+ assert_equal 1, summary.restored
+ assert_equal b.id, Item.first(external_id: "B").id
+ assert_equal %w[A B], names
+ end
+
+ def test_refreshes_name_url_and_metadata
+ Sync.call(@source, [Sources::Entry.new(external_id: "v1", name: "Old")])
+ Sync.call(@source, [Sources::Entry.new(external_id: "v1", name: "New", url: "https://example.com/v1", metadata: { "thumb" => "t.jpg" })])
+
+ item = Item.first(external_id: "v1")
+ assert_equal "New", item.name
+ assert_equal "https://example.com/v1", item.url
+ assert_equal({ "thumb" => "t.jpg" }, item.metadata)
+ end
+
+ def test_text_source_strips_blank_and_duplicate_lines
+ entries = Sources::Text.new.entries(" A \n\nB\nA\n")
+ assert_equal %w[A B], entries.map(&:external_id)
+ end
+
+ private
+
+ def sync(*lines) = Sync.call(@source, Sources::Text.new.entries(lines.join("\n")))
+
+ def names = @list.items_dataset.order(:id).map(:name)
+end
diff --git a/test/test_web.rb b/test/test_web.rb
index 85d47f9..c08c568 100644
--- a/test/test_web.rb
+++ b/test/test_web.rb
@@ -16,6 +16,7 @@ class TestWeb < Minitest::Test
RankKing::DB[:results].delete
RankKing::DB[:axes].delete
RankKing::DB[:items].delete
+ RankKing::DB[:sources].delete
RankKing::DB[:lists].delete
end
@@ -121,13 +122,50 @@ class TestWeb < Minitest::Test
assert_equal 403, last_response.status
end
+ def test_create_list_adds_a_text_source
+ create_list(name: "Foods", items: "Pizza\nSushi")
+ list = RankKing::List.first
+
+ assert_equal ["text"], list.sources.map(&:kind)
+ assert_equal %w[Pizza Sushi], list.sources.first.items.map(&:external_id)
+ end
+
+ def test_sync_adds_and_hides_items
+ create_list(name: "Foods", items: "Pizza\nSushi", axes: "Taste")
+ list = RankKing::List.first
+ source = list.sources.first
+
+ csrf_post "/lists/#{list.id}/sources/#{source.id}/sync", { snapshot: "Pizza\nTacos" }
+ assert last_response.redirect?
+
+ assert_equal %w[Pizza Tacos], list.items_dataset.order(:id).map(:name)
+
+ get "/lists/#{list.id}"
+ assert_includes last_response.body, "Tacos"
+ refute_includes last_response.body, "Sushi"
+ end
+
+ def test_record_result_rejects_archived_item
+ create_list(name: "Foods", items: "Pizza\nSushi\nTacos", axes: "Taste")
+ list = RankKing::List.first
+ sushi = list.items.find { |i| i.name == "Sushi" }
+ RankKing::Sync.call(list.sources.first, RankKing::Sources::Text.new.entries("Pizza\nTacos"))
+
+ csrf_post "/lists/#{list.id}/results", {
+ axis_id: list.axes.first.id,
+ winner_id: sushi.id,
+ loser_id: list.items.first.id,
+ }
+ assert_equal 422, last_response.status
+ end
+
private
def csrf_post(path, params = {})
get "/"
token = last_response.body[/name="_csrf" value="([^"]+)"/, 1]
# The home page CSRF token is for /lists, we need one for the target path
- get "/lists/#{path.split("/")[2]}" if path.include?("/results")
+ get "/lists/#{path.split("/")[2]}" if path.match?(%r{\A/lists/\d+/})
token = last_response.body[/action="#{Regexp.escape(path)}".*?name="_csrf" value="([^"]+)"/m, 1] || token
post path, params.merge("_csrf" => token)
end