From fc52b5fd8b7d0dd4d0eb4262c0ed0e46047e5837 Mon Sep 17 00:00:00 2001 From: leeweisa3-maker Date: Wed, 12 Aug 2026 21:40:12 +0800 Subject: [PATCH 1/3] Use InnerTube to fetch channel videos --- src/invidious/channels/channels.cr | 91 +++++----------------- src/invidious/jobs/refresh_channels_job.cr | 2 +- 2 files changed, 20 insertions(+), 73 deletions(-) diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr index 64f0484d..bac0a8f8 100644 --- a/src/invidious/channels/channels.cr +++ b/src/invidious/channels/channels.cr @@ -159,36 +159,12 @@ def fetch_channel(ucid, pull_all_videos : Bool) LOGGER.debug("fetch_channel: #{ucid}") LOGGER.trace("fetch_channel: #{ucid} : pull_all_videos = #{pull_all_videos}") - namespaces = { - "yt" => "http://www.youtube.com/xml/schemas/2015", - "media" => "http://search.yahoo.com/mrss/", - "default" => "http://www.w3.org/2005/Atom", - } - - LOGGER.trace("fetch_channel: #{ucid} : Downloading RSS feed") - rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}").body - LOGGER.trace("fetch_channel: #{ucid} : Parsing RSS feed") - rss = XML.parse(rss) - - author = rss.xpath_node("//default:feed/default:title", namespaces) - if !author - raise InfoException.new("Deleted or invalid channel") - end - - author = author.content - - # Auto-generated channels - # https://support.google.com/youtube/answer/2579942 - if author.ends_with?(" - Topic") || - {"Popular on YouTube", "Music", "Sports", "Gaming"}.includes? author - auto_generated = true - end - - LOGGER.trace("fetch_channel: #{ucid} : author = #{author}, auto_generated = #{auto_generated}") + LOGGER.trace("fetch_channel: #{ucid} : Downloading channel information") + channel_info = get_about_info(ucid, nil) channel = InvidiousChannel.new({ - id: ucid, - author: author, + id: channel_info.ucid, + author: channel_info.author, updated: Time.utc, deleted: false, subscribed: nil, @@ -197,61 +173,32 @@ def fetch_channel(ucid, pull_all_videos : Bool) LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos page") videos, continuation = IV::Channel::Tabs.get_videos(channel) - LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel RSS feed") - rss.xpath_nodes("//default:feed/default:entry", namespaces).each do |entry| - video_id = entry.xpath_node("yt:videoId", namespaces).not_nil!.content - title = entry.xpath_node("default:title", namespaces).not_nil!.content - - published = Time.parse_rfc3339( - entry.xpath_node("default:published", namespaces).not_nil!.content - ) - updated = Time.parse_rfc3339( - entry.xpath_node("default:updated", namespaces).not_nil!.content - ) - - author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content - ucid = entry.xpath_node("yt:channelId", namespaces).not_nil!.content - - views = entry - .xpath_node("media:group/media:community/media:statistics", namespaces) - .try &.["views"]?.try &.to_i64? || 0_i64 - - channel_video = videos - .select(SearchVideo) - .select(&.id.== video_id)[0]? - - length_seconds = channel_video.try &.length_seconds - length_seconds ||= 0 - - live_now = channel_video.try &.badges.live_now? - live_now ||= false - - premiere_timestamp = channel_video.try &.premiere_timestamp - + LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel response") + videos.select(SearchVideo).each do |channel_video| video = ChannelVideo.new({ - id: video_id, - title: title, - published: published, - updated: updated, - ucid: ucid, - author: author, - length_seconds: length_seconds, - live_now: live_now, - premiere_timestamp: premiere_timestamp, - views: views, + id: channel_video.id, + title: channel_video.title, + published: channel_video.published, + updated: Time.utc, + ucid: channel.id, + author: channel.author, + length_seconds: channel_video.length_seconds, + live_now: channel_video.badges.live_now?, + premiere_timestamp: channel_video.premiere_timestamp, + views: channel_video.views, }) - LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updating or inserting video") + LOGGER.trace("fetch_channel: #{ucid} : video #{channel_video.id} : Updating or inserting video") # We don't include the 'premiere_timestamp' here because channel pages don't include them, # meaning the above timestamp is always null was_insert = Invidious::Database::ChannelVideos.insert(video) if was_insert - LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Inserted, updating subscriptions") + LOGGER.trace("fetch_channel: #{ucid} : video #{channel_video.id} : Inserted, updating subscriptions") NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video)) else - LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updated") + LOGGER.trace("fetch_channel: #{ucid} : video #{channel_video.id} : Updated") end end diff --git a/src/invidious/jobs/refresh_channels_job.cr b/src/invidious/jobs/refresh_channels_job.cr index 5c5ecc83..7e459149 100644 --- a/src/invidious/jobs/refresh_channels_job.cr +++ b/src/invidious/jobs/refresh_channels_job.cr @@ -43,7 +43,7 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob end rescue ex LOGGER.error("RefreshChannelsJob: #{id} : #{ex.message}") - if ex.message == "Deleted or invalid channel" + if {"Deleted or invalid channel", "This channel does not exist."}.includes?(ex.message) Invidious::Database::Channels.update_mark_deleted(id) else lim_fibers = 1 From 867f4fce82fe02bd035b9e2dde77e7126fdf6706 Mon Sep 17 00:00:00 2001 From: leeweisa3-maker Date: Wed, 12 Aug 2026 23:19:22 +0800 Subject: [PATCH 2/3] Preserve channel video publication times --- src/invidious/channels/channels.cr | 6 ++++-- src/invidious/database/channels.cr | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr index bac0a8f8..c7bbf7ea 100644 --- a/src/invidious/channels/channels.cr +++ b/src/invidious/channels/channels.cr @@ -192,7 +192,9 @@ def fetch_channel(ucid, pull_all_videos : Bool) # We don't include the 'premiere_timestamp' here because channel pages don't include them, # meaning the above timestamp is always null - was_insert = Invidious::Database::ChannelVideos.insert(video) + # InnerTube provides relative publication labels. Once persisted, keep the + # existing timestamp so a later refresh cannot shift it forward. + was_insert = Invidious::Database::ChannelVideos.insert(video, update_published: false) if was_insert LOGGER.trace("fetch_channel: #{ucid} : video #{channel_video.id} : Inserted, updating subscriptions") @@ -226,7 +228,7 @@ def fetch_channel(ucid, pull_all_videos : Bool) # We are notified of Red videos elsewhere (PubSub), which includes a correct published date, # so since they don't provide a published date here we can safely ignore them. if Time.utc - video.published > 1.minute - was_insert = Invidious::Database::ChannelVideos.insert(video) + was_insert = Invidious::Database::ChannelVideos.insert(video, update_published: false) if was_insert NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video)) end diff --git a/src/invidious/database/channels.cr b/src/invidious/database/channels.cr index df44e485..444d046d 100644 --- a/src/invidious/database/channels.cr +++ b/src/invidious/database/channels.cr @@ -98,18 +98,20 @@ module Invidious::Database::ChannelVideos # ------------------- # This function returns the status of the query (i.e: success?) - def insert(video : ChannelVideo, with_premiere_timestamp : Bool = false) : Bool + def insert(video : ChannelVideo, with_premiere_timestamp : Bool = false, update_published : Bool = true) : Bool if with_premiere_timestamp last_items = "premiere_timestamp = $9, views = $10" else last_items = "views = $10" end + published_item = update_published ? "published = $3," : "" + request = <<-SQL INSERT INTO channel_videos VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) ON CONFLICT (id) DO UPDATE - SET title = $2, published = $3, updated = $4, ucid = $5, + SET title = $2, #{published_item} updated = $4, ucid = $5, author = $6, length_seconds = $7, live_now = $8, #{last_items} RETURNING (xmax=0) AS was_insert SQL From f2a550efe694437ec5144c60672f394460771f03 Mon Sep 17 00:00:00 2001 From: leeweisa3-maker Date: Thu, 13 Aug 2026 10:38:17 +0800 Subject: [PATCH 3/3] Preserve auto-generated channel video identity --- src/invidious/channels/channels.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr index c7bbf7ea..fe8d5fe9 100644 --- a/src/invidious/channels/channels.cr +++ b/src/invidious/channels/channels.cr @@ -180,8 +180,8 @@ def fetch_channel(ucid, pull_all_videos : Bool) title: channel_video.title, published: channel_video.published, updated: Time.utc, - ucid: channel.id, - author: channel.author, + ucid: channel_info.auto_generated ? channel_video.ucid : channel.id, + author: channel_info.auto_generated ? channel_video.author : channel.author, length_seconds: channel_video.length_seconds, live_now: channel_video.badges.live_now?, premiere_timestamp: channel_video.premiere_timestamp,