1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
# frozen_string_literal: true
require "digest"
require "json"
require "net/http"
require "uri"
module RankKing
# InnerTube, the API youtube.com's own pages call, signed in with the
# user's browser cookie. It stands in for OAuth and the YouTube Data
# API, which an Advanced Protection account cannot grant to an
# unverified client and which never shares Watch Later. Every call goes
# through #request, which a subclass overrides to answer without the
# network, as the tests' does.
class InnerTube
# YouTube refused a request, or answered with something unexpected.
class Error < StandardError; end
# A video in a playlist, as YouTube describes it.
# Signed in sig/rank_king/inner_tube.rbs: a Data class.
# @rbs skip
Video = Data.define(:id, :title, :channel_title, :thumbnail_url)
ORIGIN = "https://www.youtube.com"
BROWSE_URL = "#{ORIGIN}/youtubei/v1/browse?prettyPrint=false"
# The web client InnerTube is told it is serving. YouTube may refuse
# a version once it is old enough.
CLIENT = { clientName: "WEB", clientVersion: "2.20261020.00.00", hl: "en", gl: "US" }.freeze
# Each cookie a request is signed with, and the scheme its signature
# goes under in the Authorization header.
API_COOKIES = {
"SAPISID" => "SAPISIDHASH",
"__Secure-1PAPISID" => "SAPISID1PHASH",
"__Secure-3PAPISID" => "SAPISID3PHASH",
}.freeze #: Hash[String, String]
# A Cookie header as pasted, without the "Cookie:" it may have been
# copied with, or nil when it has no cookie to sign requests with.
#: (String text) -> String?
def self.cookie(text)
cookie = text.strip.delete_prefix("Cookie:").strip.gsub(/\s*\n\s*/, " ")
cookie if API_COOKIES.keys.any? { |name| jar(cookie)[name] }
end
#: (String cookie) -> Hash[String, String]
def self.jar(cookie)
jar = {} #: Hash[String, String]
cookie.split(/;\s*/).each do |pair|
name, value = pair.split("=", 2)
jar[name.to_s] = value if value
end
jar
end
# Google's signature for a request made with a cookie: for each API
# cookie, a SHA-1 of the time, its value, and the origin.
#: (String cookie, ?at: Integer) -> String
def self.authorization(cookie, at: Time.now.to_i)
jar = jar(cookie)
API_COOKIES.filter_map { |name, scheme|
value = jar[name]
"#{scheme} #{at}_#{Digest::SHA1.hexdigest("#{at} #{value} #{ORIGIN}")}" if value
}.join(" ")
end
# The user's playlists, as [playlist id, title], in YouTube's order:
# Watch Later and Liked videos among their own and the ones they saved.
#: (String cookie) -> Array[[String, String]]
def playlists(cookie)
lockups = pages(cookie, { browseId: "FEplaylist_aggregation" }).flat_map { |page| find_all(page, "lockupViewModel") }
lockups.filter_map { |lockup|
next unless lockup["contentType"] == "LOCKUP_CONTENT_TYPE_PLAYLIST"
title = lockup.dig("metadata", "lockupMetadataViewModel", "title", "content").to_s
[lockup.fetch("contentId"), title] #: [String, String]
}
end
# The videos in a playlist, in playlist order. Deleted videos, and
# private ones the user cannot see, are not playable and are left out.
# Some videos come with no channel.
#: (String cookie, String playlist_id) -> Array[Video]
def playlist_videos(cookie, playlist_id)
renderers = pages(cookie, { browseId: "VL#{playlist_id}" }).flat_map { |page| find_all(page, "playlistVideoRenderer") }
renderers.filter_map { |renderer|
next unless renderer["isPlayable"]
id = renderer.fetch("videoId")
Video.new(
id:,
title: text(renderer["title"]).to_s,
channel_title: text(renderer["shortBylineText"]),
thumbnail_url: "https://i.ytimg.com/vi/#{id}/mqdefault.jpg",
)
}
end
private
# Every page of a browse, following its continuations.
#: (String cookie, Hash[Symbol, String] body) -> Array[Hash[String, untyped]]
def pages(cookie, body)
pages = [] #: Array[Hash[String, untyped]]
loop do
page = request(cookie, body)
check(page) if pages.empty?
pages << page
token = find_all(page, "continuationItemRenderer")
.flat_map { |item| find_all(item, "continuationCommand") }
.first&.dig("token")
break unless token
body = { continuation: token }
end
pages
end
# A cookie YouTube has signed out still gets answers, as anyone would,
# so a sync must not take them for an empty playlist.
#: (Hash[String, untyped] page) -> void
def check(page)
params = page.dig("responseContext", "serviceTrackingParams").to_a.flat_map { |service| service["params"].to_a }
unless params.any? { |param| param["key"] == "logged_in" && param["value"] == "1" }
raise Error, "YouTube signed the cookie out. Paste a fresh one in settings."
end
alert = find_all(page, "alertRenderer").find { |a| a["type"] == "ERROR" }
raise Error, text(alert["text"]) || "YouTube sent an error" if alert
end
# The words of an InnerTube text, which comes either whole or in runs.
#: (untyped node) -> String?
def text(node)
return unless node.is_a?(Hash)
node["simpleText"] || node["runs"]&.map { |run| run["text"] }&.join
end
# Every value under key, however deep: InnerTube nests the same
# renderers differently from page to page.
#: (untyped node, String key, ?Array[Hash[String, untyped]] found) -> Array[Hash[String, untyped]]
def find_all(node, key, found = [])
case node
when Hash
node.each { |k, v| k == key ? found << v : find_all(v, key, found) }
when Array
node.each { |v| find_all(v, key, found) }
end
found
end
# Makes one browse request and returns its JSON body.
#: (String cookie, Hash[Symbol, String] body) -> Hash[String, untyped]
def request(cookie, body)
uri = URI(BROWSE_URL)
post = Net::HTTP::Post.new(uri)
post["Cookie"] = cookie
post["Authorization"] = self.class.authorization(cookie)
post["Origin"] = ORIGIN
post["X-Origin"] = ORIGIN
post["X-Goog-AuthUser"] = "0"
post["Content-Type"] = "application/json"
post.body = JSON.generate({ context: { client: CLIENT }, **body })
response = Net::HTTP.start(uri.hostname.to_s, uri.port, use_ssl: true) { |http| http.request(post) }
json = JSON.parse(response.body.to_s)
return json if response.is_a?(Net::HTTPSuccess)
raise Error, json.dig("error", "message") || "HTTP #{response.code}"
rescue JSON::ParserError
raise Error, "YouTube answered HTTP #{response&.code} with no JSON"
end
end
end