VeritableValor/invidious
1
0
Fork 0
mirror of https://github.com/iv-org/invidious.git synced 2026-10-26 06:49:15 -05:00
Code Issues Packages Projects Releases Wiki Activity

Use InnerTube instead of RSS for channel video refresh

Browse Source 
Replace /feeds/videos.xml parsing in fetch_channel with the existing
get_about_info and Channel::Tabs.get_videos helpers. Preserve stored
published/updated timestamps on conflict so relative InnerTube dates do
not drift and Atom feeds do not churn on unchanged videos. Mark channels
deleted via NotFoundException from InnerTube about errors.

Fixes #2410
This commit is contained in:
LaptopsPlural 2026-09-11 15:41:01 +00:00
parent 049d591d29
commit d78d3091ed
4 changed files with 67 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/invidious/channels/about.cr
View File

@ -19,6 +19,11 @@ record AboutChannel,
verified : Bool, verified : Bool,
is_age_gated : Bool is_age_gated : Bool
private DELETED_CHANNEL_ERROR_MESSAGES = {
"This channel does not exist.",
"This account has been terminated for a violation of YouTube's Terms of Service.",
}
def get_about_info(ucid) : AboutChannel def get_about_info(ucid) : AboutChannel
begin begin
# Fetch channel information from channel home page # Fetch channel information from channel home page
@ -29,7 +34,7 @@ def get_about_info(ucid) : AboutChannel
if initdata.dig?("alerts", 0, "alertRenderer", "type") == "ERROR" if initdata.dig?("alerts", 0, "alertRenderer", "type") == "ERROR"
error_message = initdata["alerts"][0]["alertRenderer"]["text"]["simpleText"].as_s error_message = initdata["alerts"][0]["alertRenderer"]["text"]["simpleText"].as_s
if error_message == "This channel does not exist." if DELETED_CHANNEL_ERROR_MESSAGES.includes?(error_message)
raise NotFoundException.new(error_message) raise NotFoundException.new(error_message)
else else
raise InfoException.new(error_message) raise InfoException.new(error_message)

137
src/invidious/channels/channels.cr
View File

@ -159,127 +159,65 @@ def fetch_channel(ucid, pull_all_videos : Bool)
LOGGER.debug("fetch_channel: #{ucid}") LOGGER.debug("fetch_channel: #{ucid}")
LOGGER.trace("fetch_channel: #{ucid} : pull_all_videos = #{pull_all_videos}") LOGGER.trace("fetch_channel: #{ucid} : pull_all_videos = #{pull_all_videos}")
namespaces = { # Channel RSS (/feeds/videos.xml) is unreliable and rate-limited. Use the
"yt" => "http://www.youtube.com/xml/schemas/2015", # existing InnerTube-backed helpers instead (see #2410).
"media" => "http://search.yahoo.com/mrss/", LOGGER.trace("fetch_channel: #{ucid} : Downloading channel information")
"default" => "http://www.w3.org/2005/Atom", about = get_about_info(ucid)
}
LOGGER.trace("fetch_channel: #{ucid} : Downloading RSS feed") LOGGER.trace("fetch_channel: #{ucid} : author = #{about.author}, auto_generated = #{about.auto_generated}")
rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}").body
LOGGER.trace("fetch_channel: #{ucid} : Parsing RSS feed")
rss = XML.parse(rss)
author = rss.xpath_node("//default:feed/default:title", namespaces)
if !author
raise InfoException.new("Deleted or invalid channel")
end
author = author.content
# Auto-generated channels
# https://support.google.com/youtube/answer/2579942
if author.ends_with?(" - Topic") ||
{"Popular on YouTube", "Music", "Sports", "Gaming"}.includes? author
auto_generated = true
end
LOGGER.trace("fetch_channel: #{ucid} : author = #{author}, auto_generated = #{auto_generated}")
channel = InvidiousChannel.new({ channel = InvidiousChannel.new({
id: ucid, id: about.ucid,
author: author, author: about.author,
updated: Time.utc, updated: Time.utc,
deleted: false, deleted: false,
subscribed: nil, subscribed: nil,
}) })
LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos page") LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos page")
videos, continuation = IV::Channel::Tabs.get_videos(channel) videos, continuation = IV::Channel::Tabs.get_videos(about)
LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel RSS feed") LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel tabs")
rss.xpath_nodes("//default:feed/default:entry", namespaces).each do |entry| videos.select(SearchVideo).each do |search_video|
video_id = entry.xpath_node("yt:videoId", namespaces).not_nil!.content video = channel_video_from_search(search_video)
title = entry.xpath_node("default:title", namespaces).not_nil!.content
published = Time.parse_rfc3339( LOGGER.trace("fetch_channel: #{ucid} : video #{video.id} : Updating or inserting video")
entry.xpath_node("default:published", namespaces).not_nil!.content
# InnerTube only exposes relative publication labels. Keep previously stored
# published/updated values on conflict so refreshes do not drift timestamps
# or churn Atom <updated> for unchanged entries.
was_insert = Invidious::Database::ChannelVideos.insert(
video,
update_published: false,
update_updated: false,
) )
updated = Time.parse_rfc3339(
entry.xpath_node("default:updated", namespaces).not_nil!.content
)
author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content
ucid = entry.xpath_node("yt:channelId", namespaces).not_nil!.content
views = entry
.xpath_node("media:group/media:community/media:statistics", namespaces)
.try &.["views"]?.try &.to_i64? || 0_i64
channel_video = videos
.select(SearchVideo)
.select(&.id.== video_id)[0]?
length_seconds = channel_video.try &.length_seconds
length_seconds ||= 0
live_now = channel_video.try &.badges.live_now?
live_now ||= false
premiere_timestamp = channel_video.try &.premiere_timestamp
video = ChannelVideo.new({
id: video_id,
title: title,
published: published,
updated: updated,
ucid: ucid,
author: author,
length_seconds: length_seconds,
live_now: live_now,
premiere_timestamp: premiere_timestamp,
views: views,
})
LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updating or inserting video")
# We don't include the 'premiere_timestamp' here because channel pages don't include them,
# meaning the above timestamp is always null
was_insert = Invidious::Database::ChannelVideos.insert(video)
if was_insert if was_insert
LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Inserted, updating subscriptions") LOGGER.trace("fetch_channel: #{ucid} : video #{video.id} : Inserted, updating subscriptions")
NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video)) NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video))
else else
LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updated") LOGGER.trace("fetch_channel: #{ucid} : video #{video.id} : Updated")
end end
end end
if pull_all_videos if pull_all_videos
loop do loop do
# Keep fetching videos using the continuation token retrieved earlier # Keep fetching videos using the continuation token retrieved earlier
videos, continuation = IV::Channel::Tabs.get_videos(channel, continuation: continuation) videos, continuation = IV::Channel::Tabs.get_videos(about, continuation: continuation)
count = 0 count = 0
videos.select(SearchVideo).each do |video| videos.select(SearchVideo).each do |search_video|
count += 1 count += 1
video = ChannelVideo.new({ video = channel_video_from_search(search_video)
id: video.id,
title: video.title,
published: video.published,
updated: Time.utc,
ucid: video.ucid,
author: video.author,
length_seconds: video.length_seconds,
live_now: video.badges.live_now?,
premiere_timestamp: video.premiere_timestamp,
views: video.views,
})
# We are notified of Red videos elsewhere (PubSub), which includes a correct published date, # We are notified of Red videos elsewhere (PubSub), which includes a correct published date,
# so since they don't provide a published date here we can safely ignore them. # so since they don't provide a published date here we can safely ignore them.
if Time.utc - video.published > 1.minute if Time.utc - video.published > 1.minute
was_insert = Invidious::Database::ChannelVideos.insert(video) was_insert = Invidious::Database::ChannelVideos.insert(
video,
update_published: false,
update_updated: false,
)
if was_insert if was_insert
NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video)) NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video))
end end
@ -294,3 +232,18 @@ def fetch_channel(ucid, pull_all_videos : Bool)
channel.updated = Time.utc channel.updated = Time.utc
return channel return channel
end end
private def channel_video_from_search(video : SearchVideo) : ChannelVideo
ChannelVideo.new({
id: video.id,
title: video.title,
published: video.published,
updated: Time.utc,
ucid: video.ucid,
author: video.author,
length_seconds: video.length_seconds,
live_now: video.badges.live_now?,
premiere_timestamp: video.premiere_timestamp,
views: video.views,
})
end

17
src/invidious/database/channels.cr
View File

@ -98,18 +98,31 @@ module Invidious::Database::ChannelVideos
# ------------------- # -------------------
# This function returns the status of the query (i.e: success?) # This function returns the status of the query (i.e: success?)
def insert(video : ChannelVideo, with_premiere_timestamp : Bool = false) : Bool # update_published / update_updated control whether those columns are
# overwritten on conflict. InnerTube refreshes pass false so relative
# publication labels and refresh-time "updated" values do not drift or
# churn Atom feeds for unchanged videos.
def insert(
video : ChannelVideo,
with_premiere_timestamp : Bool = false,
*,
update_published : Bool = true,
update_updated : Bool = true,
) : Bool
if with_premiere_timestamp if with_premiere_timestamp
last_items = "premiere_timestamp = $9, views = $10" last_items = "premiere_timestamp = $9, views = $10"
else else
last_items = "views = $10" last_items = "views = $10"
end end
published_item = update_published ? "published = $3," : ""
updated_item = update_updated ? "updated = $4," : ""
request = <<-SQL request = <<-SQL
INSERT INTO channel_videos INSERT INTO channel_videos
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
ON CONFLICT (id) DO UPDATE ON CONFLICT (id) DO UPDATE
SET title = $2, published = $3, updated = $4, ucid = $5, SET title = $2, #{published_item} #{updated_item} ucid = $5,
author = $6, length_seconds = $7, live_now = $8, #{last_items} author = $6, length_seconds = $7, live_now = $8, #{last_items}
RETURNING (xmax=0) AS was_insert RETURNING (xmax=0) AS was_insert
SQL SQL

2
src/invidious/jobs/refresh_channels_job.cr
View File

@ -43,7 +43,7 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob
end end
rescue ex rescue ex
LOGGER.error("RefreshChannelsJob: #{id} : #{ex.message}") LOGGER.error("RefreshChannelsJob: #{id} : #{ex.message}")
if ex.message == "Deleted or invalid channel" if ex.is_a?(NotFoundException)
Invidious::Database::Channels.update_mark_deleted(id) Invidious::Database::Channels.update_mark_deleted(id)
else else
lim_fibers = 1 lim_fibers = 1