From 0880760caa29a0179d529597906f98e4ed322d0a Mon Sep 17 00:00:00 2001 From: AssassinNAT10NS <83136258+AssassinNAT10NS@users.noreply.github.com> Date: Tue, 28 Jul 2026 15:12:26 -0500 Subject: [PATCH] Use InnerTube for channel feed refresh AI-assisted change prepared with OpenAI Codex (GPT-5). Manual human review and runtime testing are pending; this branch has not been submitted upstream. --- src/invidious/channels/channels.cr | 143 +++++++---------------------- 1 file changed, 31 insertions(+), 112 deletions(-) diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr index 64f0484df..a45eb9ab1 100644 --- a/src/invidious/channels/channels.cr +++ b/src/invidious/channels/channels.cr @@ -159,30 +159,10 @@ def fetch_channel(ucid, pull_all_videos : Bool) LOGGER.debug("fetch_channel: #{ucid}") LOGGER.trace("fetch_channel: #{ucid} : pull_all_videos = #{pull_all_videos}") - namespaces = { - "yt" => "http://www.youtube.com/xml/schemas/2015", - "media" => "http://search.yahoo.com/mrss/", - "default" => "http://www.w3.org/2005/Atom", - } - - LOGGER.trace("fetch_channel: #{ucid} : Downloading RSS feed") - rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}").body - LOGGER.trace("fetch_channel: #{ucid} : Parsing RSS feed") - rss = XML.parse(rss) - - author = rss.xpath_node("//default:feed/default:title", namespaces) - if !author - raise InfoException.new("Deleted or invalid channel") - end - - author = author.content - - # Auto-generated channels - # https://support.google.com/youtube/answer/2579942 - if author.ends_with?(" - Topic") || - {"Popular on YouTube", "Music", "Sports", "Gaming"}.includes? author - auto_generated = true - end + LOGGER.trace("fetch_channel: #{ucid} : Downloading channel information") + about_channel = get_about_info(ucid, CONFIG.default_user_preferences.locale) + author = about_channel.author + auto_generated = about_channel.auto_generated LOGGER.trace("fetch_channel: #{ucid} : author = #{author}, auto_generated = #{auto_generated}") @@ -197,98 +177,37 @@ def fetch_channel(ucid, pull_all_videos : Bool) LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos page") videos, continuation = IV::Channel::Tabs.get_videos(channel) - LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel RSS feed") - rss.xpath_nodes("//default:feed/default:entry", namespaces).each do |entry| - video_id = entry.xpath_node("yt:videoId", namespaces).not_nil!.content - title = entry.xpath_node("default:title", namespaces).not_nil!.content + loop do + count = 0 + videos.select(SearchVideo).each do |video| + count += 1 + video = ChannelVideo.new({ + id: video.id, + title: video.title, + published: video.published, + updated: Time.utc, + ucid: video.ucid, + author: video.author, + length_seconds: video.length_seconds, + live_now: video.badges.live_now?, + premiere_timestamp: video.premiere_timestamp, + views: video.views, + }) - published = Time.parse_rfc3339( - entry.xpath_node("default:published", namespaces).not_nil!.content - ) - updated = Time.parse_rfc3339( - entry.xpath_node("default:updated", namespaces).not_nil!.content - ) - - author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content - ucid = entry.xpath_node("yt:channelId", namespaces).not_nil!.content - - views = entry - .xpath_node("media:group/media:community/media:statistics", namespaces) - .try &.["views"]?.try &.to_i64? || 0_i64 - - channel_video = videos - .select(SearchVideo) - .select(&.id.== video_id)[0]? - - length_seconds = channel_video.try &.length_seconds - length_seconds ||= 0 - - live_now = channel_video.try &.badges.live_now? - live_now ||= false - - premiere_timestamp = channel_video.try &.premiere_timestamp - - video = ChannelVideo.new({ - id: video_id, - title: title, - published: published, - updated: updated, - ucid: ucid, - author: author, - length_seconds: length_seconds, - live_now: live_now, - premiere_timestamp: premiere_timestamp, - views: views, - }) - - LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updating or inserting video") - - # We don't include the 'premiere_timestamp' here because channel pages don't include them, - # meaning the above timestamp is always null - was_insert = Invidious::Database::ChannelVideos.insert(video) - - if was_insert - LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Inserted, updating subscriptions") - NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video)) - else - LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updated") - end - end - - if pull_all_videos - loop do - # Keep fetching videos using the continuation token retrieved earlier - videos, continuation = IV::Channel::Tabs.get_videos(channel, continuation: continuation) - - count = 0 - videos.select(SearchVideo).each do |video| - count += 1 - video = ChannelVideo.new({ - id: video.id, - title: video.title, - published: video.published, - updated: Time.utc, - ucid: video.ucid, - author: video.author, - length_seconds: video.length_seconds, - live_now: video.badges.live_now?, - premiere_timestamp: video.premiere_timestamp, - views: video.views, - }) - - # We are notified of Red videos elsewhere (PubSub), which includes a correct published date, - # so since they don't provide a published date here we can safely ignore them. - if Time.utc - video.published > 1.minute - was_insert = Invidious::Database::ChannelVideos.insert(video) - if was_insert - NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video)) - end + # We are notified of Red videos elsewhere (PubSub), which includes a correct published date, + # so since they don't provide a published date here we can safely ignore them. + if Time.utc - video.published > 1.minute + was_insert = Invidious::Database::ChannelVideos.insert(video) + if was_insert + NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video)) end end - - break if count < 25 - sleep 500.milliseconds end + + break if !pull_all_videos || continuation.nil? || count < 25 + + sleep 500.milliseconds + videos, continuation = IV::Channel::Tabs.get_videos(channel, continuation: continuation) end channel.updated = Time.utc