From 6ccb0520cf5a9c85c89d1a55a5a31773dac9796c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Eriksen?= Date: Sun, 30 Aug 2026 16:41:12 +0200 Subject: [PATCH] Record the content kind of subscription feed entries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upstream has no way to tell a Short from a long-form upload in a feed: `ChannelVideo#to_json` reports `"type": "shortVideo"` for every row, and `channel_videos.length_seconds` is 0 for anything absent from the channel's Videos tab — Shorts and stream VODs alike. 531 of 1586 rows (34%) were in that state. YouTube's per-channel uploads playlists supply the signal, addressed by replacing the "UC" of the channel ID: UULF long-form, UUSH Shorts, UULV live. Measured over 75 channels, 15 newest entries each: 1114 long-form, 679 Shorts, 299 live, one Short leaking into a UULF feed. ClassifyChannelVideosJob labels `kind` from those feeds, falling back to a capped `HEAD /shorts/` probe (200 = Short, 303 = not) for rows older than a 15-entry window. `feed_kinds` restricts the subscription feed via a predicate in each user's materialized view. A separate job rather than an edit to `fetch_channel`, and `kind` is absent from the insert's `ON CONFLICT DO UPDATE` set, so a channel refresh cannot overwrite a label. Unclassified entries are always shown and an empty `feed_kinds` admits everything, so the feed cannot end up blank. Also folds in three pre-existing ameba Performance/ChainedCallWithNoBang fixes in arik_settings.cr and admin_settings.cr, which a newer ameba release started flagging. Co-Authored-By: Claude Opus 5 --- config/config.example.yml | 22 +- config/sql/channel_videos.sql | 1 + locales/en-US.json | 5 + spec/invidious/arik_feed_kinds_spec.cr | 165 ++++++++++++ spec/invidious/arik_settings_spec.cr | 1 + src/invidious.cr | 3 + src/invidious/arik_feed_kinds.cr | 124 +++++++++ src/invidious/arik_settings.cr | 13 +- src/invidious/channels/channels.cr | 8 +- src/invidious/config.cr | 4 + src/invidious/database/channels.cr | 4 +- .../migrations/0012_add_channel_video_kind.cr | 24 ++ .../jobs/classify_channel_videos_job.cr | 245 ++++++++++++++++++ src/invidious/routes/admin_settings.cr | 9 +- src/invidious/routes/feeds.cr | 1 + src/invidious/users.cr | 5 +- src/invidious/views/admin/settings.ecr | 13 + 17 files changed, 639 insertions(+), 8 deletions(-) create mode 100644 spec/invidious/arik_feed_kinds_spec.cr create mode 100644 src/invidious/arik_feed_kinds.cr create mode 100644 src/invidious/database/migrations/0012_add_channel_video_kind.cr create mode 100644 src/invidious/jobs/classify_channel_videos_job.cr diff --git a/config/config.example.yml b/config/config.example.yml index 8ae6bc5ce..474b5dad1 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -424,7 +424,27 @@ https_only: false # - IVPLxxxxxxxxxxxxxxxxxxxx ## -## NOTE: the two settings above and trusted_header_auth can also be +## Subscription feed content kinds (ArikTube extension). +## +## Which kinds of upload the subscription feed shows. YouTube publishes a +## per-channel uploads playlist per kind (UULF long-form, UUSH Shorts, +## UULV live), so ClassifyChannelVideosJob labels each row from YouTube's +## own classification rather than guessing from the length — a 27-second +## clip and a 5-hour stream VOD both reach channel_videos with +## length_seconds = 0. +## +## An entry the classifier has not reached yet is ALWAYS shown, and an +## empty list admits everything. Neither a late job nor a bad value can +## blank the feed. +## +## Accepted values: any of video, short, live +## Default: ["video"] +## +#feed_kinds: +# - video + +## +## NOTE: the settings above and trusted_header_auth can also be ## edited at runtime, on the ArikTube settings page an administrator ## reaches from /preferences. Those edits are stored in the database ## (table arik_settings) and override what is written here, because a diff --git a/config/sql/channel_videos.sql b/config/sql/channel_videos.sql index cd4e0ffdb..699e2faa8 100644 --- a/config/sql/channel_videos.sql +++ b/config/sql/channel_videos.sql @@ -14,6 +14,7 @@ CREATE TABLE IF NOT EXISTS public.channel_videos live_now boolean, premiere_timestamp timestamp with time zone, views bigint, + kind text, CONSTRAINT channel_videos_id_key UNIQUE (id) ); diff --git a/locales/en-US.json b/locales/en-US.json index f0cef76f6..b4923d685 100644 --- a/locales/en-US.json +++ b/locales/en-US.json @@ -490,6 +490,11 @@ "ariktube_playlist_order_label": "Position", "ariktube_extra_playlists_label": "Other playlist IDs, one per line", "ariktube_no_public_playlists": "This instance has no public playlists yet.", + "ariktube_feed_kinds_label": "Subscription feed content", + "ariktube_feed_kinds_help": "Which kinds of upload the subscription feed shows. An entry the classifier has not reached yet is always shown, and ticking nothing shows everything, so the feed can never end up blank. Changing this rebuilds every subscription feed within 15 minutes.", + "ariktube_feed_kind_video": "Long-form videos", + "ariktube_feed_kind_short": "Shorts", + "ariktube_feed_kind_live": "Live streams", "ariktube_trusted_header_auth_label": "Trusted-header authentication", "ariktube_trusted_header_auth_help": "The reverse proxy asserts the user name in a request header. The header is honored only when the direct peer is one of the trusted proxies below, which must be literal IP addresses — an address range stops the instance from starting.", "ariktube_tha_enabled_label": "Trusted-header authentication enabled: ", diff --git a/spec/invidious/arik_feed_kinds_spec.cr b/spec/invidious/arik_feed_kinds_spec.cr new file mode 100644 index 000000000..bfe778d4d --- /dev/null +++ b/spec/invidious/arik_feed_kinds_spec.cr @@ -0,0 +1,165 @@ +require "../spec_helper" +require "../../src/invidious/arik_feed_kinds" + +Spectator.describe Invidious::ArikFeedKinds do + alias Kinds = Invidious::ArikFeedKinds + + UCID = "UC7qUL2EsTHpNcgsz7woW9Iw" + + describe ".uploads_playlist_id" do + it "swaps the UC prefix for the kind's prefix" do + expect(Kinds.uploads_playlist_id(UCID, "video")).to eq("UULF7qUL2EsTHpNcgsz7woW9Iw") + expect(Kinds.uploads_playlist_id(UCID, "short")).to eq("UUSH7qUL2EsTHpNcgsz7woW9Iw") + expect(Kinds.uploads_playlist_id(UCID, "live")).to eq("UULV7qUL2EsTHpNcgsz7woW9Iw") + end + + it "keeps the ID length, so the suffix is never truncated" do + plid = Kinds.uploads_playlist_id(UCID, "short").not_nil! + expect(plid.size).to eq(UCID.size + 2) + expect(plid.ends_with?(UCID[2..])).to be_true + end + + it "refuses an ID that is not a channel ID" do + expect(Kinds.uploads_playlist_id("PL7qUL2EsTHpNcgsz7woW9Iw", "short")).to be_nil + expect(Kinds.uploads_playlist_id("UC", "short")).to be_nil + expect(Kinds.uploads_playlist_id("UCtooshort", "short")).to be_nil + expect(Kinds.uploads_playlist_id("UC7qUL2EsTHpNcgsz7woW9I/", "short")).to be_nil + end + + it "refuses a kind it does not know" do + expect(Kinds.uploads_playlist_id(UCID, "premiere")).to be_nil + expect(Kinds.uploads_playlist_id(UCID, "")).to be_nil + end + end + + describe ".feed_resource" do + it "addresses the playlist feed, not the channel feed" do + expect(Kinds.feed_resource(UCID, "short")) + .to eq("/feeds/videos.xml?playlist_id=UUSH7qUL2EsTHpNcgsz7woW9Iw") + end + + it "is nil for an unusable channel or kind" do + expect(Kinds.feed_resource("nonsense", "short")).to be_nil + expect(Kinds.feed_resource(UCID, "nonsense")).to be_nil + end + end + + describe ".kind_from_probe_status" do + it "reads YouTube's answer: 200 is a Short, a redirect is not" do + expect(Kinds.kind_from_probe_status(200)).to eq("short") + expect(Kinds.kind_from_probe_status(303)).to eq("video") + expect(Kinds.kind_from_probe_status(301)).to eq("video") + expect(Kinds.kind_from_probe_status(302)).to eq("video") + end + + it "gives no answer for a status that carries none" do + expect(Kinds.kind_from_probe_status(429)).to be_nil + expect(Kinds.kind_from_probe_status(500)).to be_nil + expect(Kinds.kind_from_probe_status(404)).to be_nil + end + end + + describe ".visible?" do + it "hides a kind the feed does not admit" do + expect(Kinds.visible?("short", ["video"])).to be_false + expect(Kinds.visible?("live", ["video"])).to be_false + expect(Kinds.visible?("video", ["video"])).to be_true + expect(Kinds.visible?("live", ["video", "live"])).to be_true + end + + it "shows an unclassified entry" do + expect(Kinds.visible?(nil, ["video"])).to be_true + end + + it "shows an entry whose stored kind is not one we know" do + expect(Kinds.visible?("premiere", ["video"])).to be_true + end + + it "shows everything when nothing is configured" do + expect(Kinds.visible?("short", [] of String)).to be_true + end + end + + describe ".json_type" do + it "reports the kind in the vocabulary clients use for search results" do + expect(Kinds.json_type("video")).to eq("video") + expect(Kinds.json_type("short")).to eq("shortVideo") + expect(Kinds.json_type("live")).to eq("stream") + end + + it "calls an unclassified entry a video, not a Short" do + expect(Kinds.json_type(nil)).to eq("video") + expect(Kinds.json_type("premiere")).to eq("video") + end + end + + describe ".clean_kinds" do + it "accepts the known kinds and fixes their order" do + cleaned, errors = Kinds.clean_kinds(["live", "video"]) + expect(cleaned).to eq(["video", "live"]) + expect(errors).to be_empty + end + + it "drops blanks and duplicates, and normalizes case" do + cleaned, errors = Kinds.clean_kinds(["VIDEO", " video ", "", " "]) + expect(cleaned).to eq(["video"]) + expect(errors).to be_empty + end + + it "reports anything that is not a kind" do + cleaned, errors = Kinds.clean_kinds(["video", "premiere"]) + expect(cleaned).to eq(["video"]) + expect(errors.size).to eq(1) + expect(errors[0]).to contain("premiere") + end + end + + describe ".decode_kinds" do + it "returns nothing for a missing row, so the config stays authoritative" do + kinds, error = Kinds.decode_kinds(nil) + expect(kinds).to be_nil + expect(error).to be_nil + end + + it "decodes a stored list" do + kinds, error = Kinds.decode_kinds(%(["video", "live"])) + expect(kinds).to eq(["video", "live"]) + expect(error).to be_nil + end + + it "refuses a malformed row instead of raising" do + kinds, error = Kinds.decode_kinds(%({"video": true})) + expect(kinds).to be_nil + expect(error).not_to be_nil + + kinds, error = Kinds.decode_kinds(%(["premiere"])) + expect(kinds).to be_nil + expect(error.not_nil!).to contain("premiere") + end + end + + describe ".view_predicate" do + it "admits NULL alongside the configured kinds" do + predicate = Kinds.view_predicate(["video"], "cv.kind") + expect(predicate).to contain("cv.kind IS NULL") + expect(predicate).to contain("cv.kind IN ('video')") + expect(predicate).to start_with(" AND (") + end + + it "lists every configured kind" do + predicate = Kinds.view_predicate(["video", "live"], "cv.kind") + expect(predicate).to contain("'video', 'live'") + end + + it "is empty when the feed admits everything" do + expect(Kinds.view_predicate([] of String, "cv.kind")).to eq("") + expect(Kinds.view_predicate(["video", "short", "live"], "cv.kind")).to eq("") + end + + it "only ever emits the kinds it validated" do + cleaned, _ = Kinds.clean_kinds(["video'; DROP TABLE users; --"]) + expect(cleaned).to be_empty + expect(Kinds.view_predicate(cleaned, "cv.kind")).to eq("") + end + end +end diff --git a/spec/invidious/arik_settings_spec.cr b/spec/invidious/arik_settings_spec.cr index 98704119b..55719a708 100644 --- a/spec/invidious/arik_settings_spec.cr +++ b/spec/invidious/arik_settings_spec.cr @@ -17,6 +17,7 @@ private class FakeConfig property popular_playlists : Array(String) = [] of String property trending_playlists : Array(String) = [] of String property trusted_header_auth : FakeTrustedHeaderAuthConfig = FakeTrustedHeaderAuthConfig.new + property feed_kinds : Array(String) = ["video"] end Spectator.describe Invidious::ArikSettings do diff --git a/src/invidious.cr b/src/invidious.cr index 7505696a2..4bc6ae156 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -194,6 +194,9 @@ Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(NOTIFICATION_CHANN Invidious::Jobs.register Invidious::Jobs::ClearExpiredItemsJob.new +# ArikTube: labels subscription feed rows with their content kind. +Invidious::Jobs.register Invidious::Jobs::ClassifyChannelVideosJob.new(PG_DB) + Invidious::Jobs.register Invidious::Jobs::InstanceListRefreshJob.new Invidious::Jobs.start_all diff --git a/src/invidious/arik_feed_kinds.cr b/src/invidious/arik_feed_kinds.cr new file mode 100644 index 000000000..159590312 --- /dev/null +++ b/src/invidious/arik_feed_kinds.cr @@ -0,0 +1,124 @@ +require "json" + +# Content kinds for subscription feed entries (ArikTube extension). +# +# Upstream has no usable signal: `ChannelVideo#to_json` reports +# `"type": "shortVideo"` for every row, and `channel_videos.length_seconds` is +# 0 for anything absent from the channel's Videos tab — Shorts and stream VODs +# alike. YouTube's per-channel uploads playlists supply it instead, addressed +# by replacing the "UC" of the channel ID: UULF long-form, UUSH Shorts, UULV +# live. Measured over 75 channels: 1 Short leaked into a UULF feed out of 679. +# +# Pure module. `Jobs::ClassifyChannelVideosJob` does the IO. +module Invidious::ArikFeedKinds + extend self + + KIND_VIDEO = "video" + KIND_SHORT = "short" + KIND_LIVE = "live" + + KINDS = [KIND_VIDEO, KIND_SHORT, KIND_LIVE] + + PLAYLIST_PREFIX = { + KIND_VIDEO => "UULF", + KIND_SHORT => "UUSH", + KIND_LIVE => "UULV", + } + + # Validated, not assumed: a malformed ID would become a playlist ID that + # returns somebody else's feed. + UCID_REGEX = /\AUC[A-Za-z0-9_-]{22}\z/ + + def uploads_playlist_id(ucid : String, kind : String) : String? + return nil if !UCID_REGEX.matches?(ucid) + + prefix = PLAYLIST_PREFIX[kind]? + return nil if prefix.nil? + + "#{prefix}#{ucid[2..]}" + end + + def feed_resource(ucid : String, kind : String) : String? + plid = uploads_playlist_id(ucid, kind) + return nil if plid.nil? + + "/feeds/videos.xml?playlist_id=#{plid}" + end + + def shorts_probe_resource(id : String) : String + "/shorts/#{id}" + end + + # YouTube answers 200 for a Short, 303 for anything else. A status that says + # nothing leaves the row unclassified rather than guessing. + def kind_from_probe_status(status : Int32) : String? + case status + when 200 then KIND_SHORT + when 301, 302, 303, 307, 308 then KIND_VIDEO + else nil + end + end + + def known?(kind : String?) : Bool + !kind.nil? && KINDS.includes?(kind) + end + + def json_type(kind : String?) : String + case kind + when KIND_SHORT then "shortVideo" + when KIND_LIVE then "stream" + else "video" + end + end + + # Fail-open: an unclassified row is shown, and an empty `allowed` shows + # everything. A late or broken classifier must never blank a feed. + def visible?(kind : String?, allowed : Array(String)) : Bool + return true if allowed.empty? + return true if !known?(kind) + + allowed.includes?(kind) + end + + def clean_kinds(entries : Array(String)) : {Array(String), Array(String)} + cleaned = [] of String + errors = [] of String + + entries.each do |entry| + kind = entry.strip.downcase + next if kind.empty? + + if !KINDS.includes?(kind) + errors << "'#{kind}' is not a content kind (#{KINDS.join(", ")})" + next + end + + cleaned << kind if !cleaned.includes?(kind) + end + + {KINDS.select { |kind| cleaned.includes?(kind) }, errors} + end + + # Never raises: one bad row must not stop the instance from booting. + def decode_kinds(raw : String?) : {Array(String)?, String?} + return {nil, nil} if raw.nil? + + kinds = Array(String).from_json(raw) + cleaned, errors = clean_kinds(kinds) + return {nil, errors.join("; ")} if !errors.empty? + + {cleaned, nil} + rescue ex + {nil, "not a JSON list of content kinds (#{ex.message})"} + end + + # `IS NULL` is inside the predicate so the fail-open rule holds within the + # materialized view too. Only ever receives `clean_kinds` output. + def view_predicate(allowed : Array(String), column : String = "kind") : String + return "" if allowed.empty? + return "" if KINDS.all? { |kind| allowed.includes?(kind) } + + list = allowed.map { |kind| "'#{kind}'" }.join(", ") + " AND (#{column} IS NULL OR #{column} IN (#{list}))" + end +end diff --git a/src/invidious/arik_settings.cr b/src/invidious/arik_settings.cr index 69e6cd69a..3e4c626b2 100644 --- a/src/invidious/arik_settings.cr +++ b/src/invidious/arik_settings.cr @@ -25,6 +25,7 @@ module Invidious::ArikSettings KEY_POPULAR_PLAYLISTS = "popular_playlists" KEY_TRENDING_PLAYLISTS = "trending_playlists" KEY_TRUSTED_HEADER_AUTH = "trusted_header_auth" + KEY_FEED_KINDS = "feed_kinds" # Playlist IDs are opaque strings; this accepts both the local "IVPL…" and # the YouTube "PL…" shapes while rejecting anything with separators in it. @@ -271,10 +272,10 @@ module Invidious::ArikSettings self.class.new( enabled: @enabled, header: @header.strip.presence || "Remote-User", - trusted_proxies: @trusted_proxies.map(&.strip).reject(&.empty?).uniq, + trusted_proxies: @trusted_proxies.map(&.strip).reject(&.empty?).uniq!, logout_url: @logout_url.strip, password_self_service: @password_self_service, - auto_approve_token_callbacks: @auto_approve_token_callbacks.map(&.strip).reject(&.empty?).uniq, + auto_approve_token_callbacks: @auto_approve_token_callbacks.map(&.strip).reject(&.empty?).uniq!, ) end @@ -331,10 +332,12 @@ module Invidious::ArikSettings popular_playlists : Array(String)?, trending_playlists : Array(String)?, trusted_header_auth : TrustedHeaderAuthSettings?, + feed_kinds : Array(String)? = nil, ) : Nil config.popular_playlists = popular_playlists if popular_playlists config.trending_playlists = trending_playlists if trending_playlists trusted_header_auth.try &.apply_to(config.trusted_header_auth) + config.feed_kinds = feed_kinds if feed_kinds end # ------------------------------------------------------------------ @@ -369,6 +372,7 @@ module Invidious::ArikSettings popular, popular_error = decode_playlists(fetch(KEY_POPULAR_PLAYLISTS)) trending, trending_error = decode_playlists(fetch(KEY_TRENDING_PLAYLISTS)) trusted_header_auth, tha_error = decode_trusted_header_auth(fetch(KEY_TRUSTED_HEADER_AUTH)) + feed_kinds, feed_kinds_error = ArikFeedKinds.decode_kinds(fetch(KEY_FEED_KINDS)) if error = popular_error LOGGER.error("ArikSettings: ignoring stored '#{KEY_POPULAR_PLAYLISTS}' — #{error}") @@ -379,7 +383,10 @@ module Invidious::ArikSettings if error = tha_error LOGGER.error("ArikSettings: ignoring stored '#{KEY_TRUSTED_HEADER_AUTH}' — #{error}") end + if error = feed_kinds_error + LOGGER.error("ArikSettings: ignoring stored '#{KEY_FEED_KINDS}' — #{error}") + end - merge_overrides!(config, popular, trending, trusted_header_auth) + merge_overrides!(config, popular, trending, trusted_header_auth, feed_kinds) end end diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr index 64f0484df..15eec6322 100644 --- a/src/invidious/channels/channels.cr +++ b/src/invidious/channels/channels.cr @@ -21,10 +21,14 @@ struct ChannelVideo property live_now : Bool = false property premiere_timestamp : Time? = nil property views : Int64? = nil + # ArikTube: content kind, nil until the classifier job has answered. + property kind : String? = nil def to_json(locale, json : JSON::Builder) json.object do - json.field "type", "shortVideo" + # ArikTube: upstream hardcodes "shortVideo" for every row, which tells a + # client nothing. Report the real kind. + json.field "type", Invidious::ArikFeedKinds.json_type(self.kind) json.field "title", self.title json.field "videoId", self.id @@ -239,6 +243,7 @@ def fetch_channel(ucid, pull_all_videos : Bool) live_now: live_now, premiere_timestamp: premiere_timestamp, views: views, + kind: nil, }) LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updating or inserting video") @@ -274,6 +279,7 @@ def fetch_channel(ucid, pull_all_videos : Bool) live_now: video.badges.live_now?, premiere_timestamp: video.premiere_timestamp, views: video.views, + kind: nil, }) # We are notified of Red videos elsewhere (PubSub), which includes a correct published date, diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 2966c2345..583ad90c7 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -160,6 +160,10 @@ class Config # duplicates dropped) instead of the stock feed content. property trending_playlists : Array(String) = [] of String property popular_playlists : Array(String) = [] of String + # Content kinds the subscription feed admits (ArikTube extension): "video", + # "short", "live". Unclassified entries are always shown and an empty list + # admits everything, so the feed cannot end up blank. + property feed_kinds : Array(String) = ["video"] property captcha_enabled : Bool = true property login_enabled : Bool = true property registration_enabled : Bool = true diff --git a/src/invidious/database/channels.cr b/src/invidious/database/channels.cr index df44e485d..d04133954 100644 --- a/src/invidious/database/channels.cr +++ b/src/invidious/database/channels.cr @@ -105,9 +105,11 @@ module Invidious::Database::ChannelVideos last_items = "views = $10" end + # ArikTube: `kind` ($11) is set on insert and deliberately absent from the + # UPDATE set, so a channel refresh cannot wipe the classifier's answer. request = <<-SQL INSERT INTO channel_videos - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, updated = $4, ucid = $5, author = $6, length_seconds = $7, live_now = $8, #{last_items} diff --git a/src/invidious/database/migrations/0012_add_channel_video_kind.cr b/src/invidious/database/migrations/0012_add_channel_video_kind.cr new file mode 100644 index 000000000..98da29bff --- /dev/null +++ b/src/invidious/database/migrations/0012_add_channel_video_kind.cr @@ -0,0 +1,24 @@ +module Invidious::Database::Migrations + # Content kind of a subscription feed entry. See `Invidious::ArikFeedKinds`. + # + # Nullable with no default: NULL means "not classified yet" and the feed + # shows those. A default of 'video' would hide which rows still owe an + # answer. + class AddChannelVideoKind < Migration + version 12 + + def up(conn : DB::Connection) + conn.exec <<-SQL + ALTER TABLE public.channel_videos + ADD COLUMN IF NOT EXISTS kind text; + SQL + + conn.exec <<-SQL + CREATE INDEX IF NOT EXISTS channel_videos_unclassified_idx + ON public.channel_videos + USING btree (ucid COLLATE pg_catalog."default") + WHERE kind IS NULL; + SQL + end + end +end diff --git a/src/invidious/jobs/classify_channel_videos_job.cr b/src/invidious/jobs/classify_channel_videos_job.cr new file mode 100644 index 000000000..840c3f86e --- /dev/null +++ b/src/invidious/jobs/classify_channel_videos_job.cr @@ -0,0 +1,245 @@ +# Labels subscription feed rows with their content kind (ArikTube extension). +# +# A separate job rather than an edit to `fetch_channel`: this branch rebases +# onto release tags, and that function is one upstream changes most. `kind` is +# also absent from the insert's `ON CONFLICT DO UPDATE` set, so a channel +# refresh cannot overwrite a label. +# +# Two passes, because a 15-entry feed window cannot answer for the whole table: +# the Shorts and live windows per channel, then a capped `/shorts/` probe +# for rows older than those windows. The probe only answers short/not-short, so +# a stream VOD in the tail is labelled `video`. +class Invidious::Jobs::ClassifyChannelVideosJob < Invidious::Jobs::BaseJob + private getter db : DB::Database + + INTERVAL = 15.minutes + CHANNELS_PER_TICK = 25 + PROBES_PER_TICK = 60 + REQUEST_DELAY = 500.milliseconds + + # `oldest` nil means the window carried nothing, which is a valid answer — + # no uploads of that kind, so YouTube 404s the playlist. A window that could + # not be *read* is a nil window. + alias Window = {ids: Array(String), oldest: Time?} + + # Kinds the existing views were built for. In the database, not memory, so a + # restart does no DDL. + APPLIED_KEY = "feed_kinds_applied" + + def initialize(@db) + end + + def begin + loop do + begin + reconcile_subscription_views + rescue ex + LOGGER.error("ClassifyChannelVideosJob: cannot reconcile the subscription views (#{ex.message})") + end + + begin + classified = run_window_pass + run_tail_pass + LOGGER.debug("ClassifyChannelVideosJob: classified #{classified} video(s)") + rescue ex + LOGGER.error("ClassifyChannelVideosJob: #{ex.message}") + end + + sleep INTERVAL + end + end + + private def run_window_pass : Int32 + labelled = 0 + + pending_channels.each do |ucid| + begin + shorts = feed_window(ucid, Invidious::ArikFeedKinds::KIND_SHORT) + lives = feed_window(ucid, Invidious::ArikFeedKinds::KIND_LIVE) + + # Eliminating against a half-read pair would call a Short long-form. + next if shorts.nil? || lives.nil? + + labelled += apply_window(ucid, shorts, lives) + rescue ex + LOGGER.error("ClassifyChannelVideosJob: #{ucid} : #{ex.message}") + end + end + + labelled + end + + private def apply_window(ucid : String, shorts : Window, lives : Window) : Int32 + labelled = write_kind(shorts[:ids], Invidious::ArikFeedKinds::KIND_SHORT) + labelled += write_kind(lives[:ids], Invidious::ArikFeedKinds::KIND_LIVE) + + if shorts[:ids].empty? && lives[:ids].empty? + # Neither kind exists for this channel, so elimination holds for its + # whole history and the tail pass never has to see it. + request = <<-SQL + UPDATE channel_videos SET kind = $1 + WHERE ucid = $2 AND kind IS NULL + SQL + + return labelled + PG_DB.exec( + request, Invidious::ArikFeedKinds::KIND_VIDEO, ucid + ).rows_affected.to_i32 + end + + # Elimination is sound only as far back as both windows saw. Past that a + # row's absence proves nothing, so it is left to the tail pass. + horizon = [shorts[:oldest], lives[:oldest]].compact.max? + return labelled if horizon.nil? + + request = <<-SQL + UPDATE channel_videos SET kind = $1 + WHERE ucid = $2 AND kind IS NULL AND published >= $3 + SQL + + labelled + PG_DB.exec( + request, Invidious::ArikFeedKinds::KIND_VIDEO, ucid, horizon + ).rows_affected.to_i32 + end + + private def run_tail_pass : Int32 + labelled = 0 + + pending_tail_videos.each do |id| + begin + kind = probe_kind(id) + next if kind.nil? + + labelled += write_kind([id], kind) + rescue ex + LOGGER.error("ClassifyChannelVideosJob: probe #{id} : #{ex.message}") + end + end + + labelled + end + + private def feed_window(ucid : String, kind : String) : Window? + resource = Invidious::ArikFeedKinds.feed_resource(ucid, kind) + return nil if resource.nil? + + sleep REQUEST_DELAY + response = YT_POOL.client &.get(resource) + + return {ids: [] of String, oldest: nil} if response.status_code == 404 + return nil if response.status_code != 200 + + namespaces = { + "yt" => "http://www.youtube.com/xml/schemas/2015", + "default" => "http://www.w3.org/2005/Atom", + } + + rss = XML.parse(response.body) + ids = [] of String + oldest = nil.as(Time?) + + rss.xpath_nodes("//default:feed/default:entry", namespaces).each do |entry| + id = entry.xpath_node("yt:videoId", namespaces).try &.content + next if id.nil? + ids << id + + raw = entry.xpath_node("default:published", namespaces).try &.content + next if raw.nil? + + published = Time.parse_rfc3339(raw) rescue nil + next if published.nil? + + current = oldest + oldest = published if current.nil? || published < current + end + + {ids: ids, oldest: oldest} + rescue ex + LOGGER.trace("ClassifyChannelVideosJob: #{ucid} #{kind} feed : #{ex.message}") + nil + end + + private def probe_kind(id : String) : String? + sleep REQUEST_DELAY + response = YT_POOL.client &.head(Invidious::ArikFeedKinds.shorts_probe_resource(id)) + Invidious::ArikFeedKinds.kind_from_probe_status(response.status_code) + rescue ex + LOGGER.trace("ClassifyChannelVideosJob: probe #{id} : #{ex.message}") + nil + end + + private def pending_channels : Array(String) + request = <<-SQL + SELECT ucid FROM channel_videos + WHERE kind IS NULL AND ucid IS NOT NULL + GROUP BY ucid + ORDER BY count(*) DESC + LIMIT $1 + SQL + + PG_DB.query_all(request, CHANNELS_PER_TICK, as: String) + end + + private def pending_tail_videos : Array(String) + request = <<-SQL + SELECT id FROM channel_videos + WHERE kind IS NULL + ORDER BY published DESC + LIMIT $1 + SQL + + PG_DB.query_all(request, PROBES_PER_TICK, as: String) + end + + # Only ever writes over NULL, so a later pass cannot undo the probe's answer. + private def write_kind(ids : Array(String), kind : String) : Int32 + return 0 if ids.empty? + + request = <<-SQL + UPDATE channel_videos SET kind = $1 + WHERE id = ANY($2) AND kind IS NULL + SQL + + PG_DB.exec(request, kind, ids).rows_affected.to_i32 + end + + # A materialized view bakes the predicate in at CREATE time and `REFRESH` + # re-runs that stored definition, so a view made before this feature keeps + # serving every kind until it is recreated. + private def reconcile_subscription_views : Nil + wanted = CONFIG.feed_kinds.to_json + return if Invidious::ArikSettings.fetch(APPLIED_KEY) == wanted && !stale_views? + + rebuild_subscription_views + Invidious::ArikSettings.store(APPLIED_KEY, wanted) + end + + # A view predating the `kind` column cannot carry the predicate whatever the + # marker claims. This is what makes a restored backup heal itself. + private def stale_views? : Bool + request = <<-SQL + SELECT count(*) FROM pg_matviews m + WHERE m.matviewname LIKE 'subscriptions\\_%' + AND NOT EXISTS ( + SELECT 1 FROM information_schema.columns c + WHERE c.table_name = m.matviewname AND c.column_name = 'kind' + ) + SQL + + PG_DB.query_one(request, as: Int64) > 0 + end + + private def rebuild_subscription_views : Nil + emails = PG_DB.query_all("SELECT email FROM users", as: String) + + emails.each do |email| + view_name = "subscriptions_#{sha256(email)}" + + begin + PG_DB.exec("DROP MATERIALIZED VIEW IF EXISTS #{view_name}") + PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(email)}") + LOGGER.info("ClassifyChannelVideosJob: rebuilt #{view_name} for kinds #{CONFIG.feed_kinds.join(",")}") + rescue ex + LOGGER.error("ClassifyChannelVideosJob: cannot rebuild #{view_name} (#{ex.message})") + end + end + end +end diff --git a/src/invidious/routes/admin_settings.cr b/src/invidious/routes/admin_settings.cr index 83167de77..68c7fcbc3 100644 --- a/src/invidious/routes/admin_settings.cr +++ b/src/invidious/routes/admin_settings.cr @@ -32,6 +32,7 @@ module Invidious::Routes::AdminSettings popular = CONFIG.popular_playlists trending = CONFIG.trending_playlists trusted_header_auth = ArikSettings::TrustedHeaderAuthSettings.from_config(CONFIG.trusted_header_auth) + feed_kinds = CONFIG.feed_kinds saved = false errors = [] of String @@ -77,6 +78,9 @@ module Invidious::Routes::AdminSettings trusted_header_auth = self.submitted_trusted_header_auth(env).cleaned errors.concat(trusted_header_auth.errors) + feed_kinds, feed_kind_errors = ArikFeedKinds.clean_kinds(env.params.body.fetch_all("feed_kind[]")) + errors.concat(feed_kind_errors.map { |error| "Subscription feed content: #{error}" }) + # Nothing is stored while anything is wrong: a half-applied trusted-header # block is exactly the state this page exists to prevent. if errors.empty? @@ -84,10 +88,13 @@ module Invidious::Routes::AdminSettings ArikSettings.store(ArikSettings::KEY_POPULAR_PLAYLISTS, popular.to_json) ArikSettings.store(ArikSettings::KEY_TRENDING_PLAYLISTS, trending.to_json) ArikSettings.store(ArikSettings::KEY_TRUSTED_HEADER_AUTH, trusted_header_auth.to_json) + ArikSettings.store(ArikSettings::KEY_FEED_KINDS, feed_kinds.to_json) CONFIG.popular_playlists = popular CONFIG.trending_playlists = trending trusted_header_auth.apply_to(CONFIG.trusted_header_auth) + # ClassifyChannelVideosJob rebuilds the views on its next tick. + CONFIG.feed_kinds = feed_kinds saved = true LOGGER.info("AdminSettings: #{user.email} updated the ArikTube settings") @@ -118,7 +125,7 @@ module Invidious::Routes::AdminSettings selected = env.params.body.fetch_all("#{prefix}_playlist[]") ordered = selected.map_with_index { |plid, index| {plid, index} } - .sort_by do |(plid, index)| + .sort_by! do |(plid, index)| position = env.params.body["#{prefix}_order[#{plid}]"]?.try &.to_i? {position || Int32::MAX, index} end diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr index ebb5f04b1..39b78e64e 100644 --- a/src/invidious/routes/feeds.cr +++ b/src/invidious/routes/feeds.cr @@ -443,6 +443,7 @@ module Invidious::Routes::Feeds live_now: video.live_now, premiere_timestamp: video.premiere_timestamp, views: video.views, + kind: nil, # ArikTube: the classifier job answers this }) was_insert = Invidious::Database::ChannelVideos.insert(video, with_premiere_timestamp: true) diff --git a/src/invidious/users.cr b/src/invidious/users.cr index 65566d207..fc11f16ec 100644 --- a/src/invidious/users.cr +++ b/src/invidious/users.cr @@ -1,7 +1,10 @@ require "crypto/bcrypt/password" # Materialized views may not be defined using bound parameters (`$1` as used elsewhere) -MATERIALIZED_VIEW_SQL = ->(email : String) { "SELECT cv.* FROM channel_videos cv WHERE EXISTS (SELECT subscriptions FROM users u WHERE cv.ucid = ANY (u.subscriptions) AND u.email = E'#{email.gsub({'\'' => "\\'", '\\' => "\\\\"})}') ORDER BY published DESC" } +# ArikTube: the trailing `view_predicate` restricts the feed to the configured +# content kinds. Baked in at CREATE time, so ClassifyChannelVideosJob rebuilds +# every view when the setting changes. +MATERIALIZED_VIEW_SQL = ->(email : String) { "SELECT cv.* FROM channel_videos cv WHERE EXISTS (SELECT subscriptions FROM users u WHERE cv.ucid = ANY (u.subscriptions) AND u.email = E'#{email.gsub({'\'' => "\\'", '\\' => "\\\\"})}')#{Invidious::ArikFeedKinds.view_predicate(CONFIG.feed_kinds, "cv.kind")} ORDER BY published DESC" } def create_user(sid, email, password) password = Crypto::Bcrypt::Password.create(password, cost: 10) diff --git a/src/invidious/views/admin/settings.ecr b/src/invidious/views/admin/settings.ecr index 5b05bf24e..b0b1f246e 100644 --- a/src/invidious/views/admin/settings.ecr +++ b/src/invidious/views/admin/settings.ecr @@ -73,6 +73,19 @@ <% end %> +
+ <%= I18n.translate(locale, "ariktube_feed_kinds_label") %> +

<%= I18n.translate(locale, "ariktube_feed_kinds_help") %>

+ + <% Invidious::ArikFeedKinds::KINDS.each do |kind| %> +
+ checked<% end %>> + +
+ <% end %> +
+
<%= I18n.translate(locale, "ariktube_trusted_header_auth_label") %>

<%= I18n.translate(locale, "ariktube_trusted_header_auth_help") %>