From 8118fa612ef64d9d36a6a71beb11150727994247 Mon Sep 17 00:00:00 2001 From: Agastya Date: Sat, 26 Sep 2026 09:54:55 +0530 Subject: [PATCH] fix: refresh all channel content tabs --- src/invidious/channels/channels.cr | 139 ++++++++++++++++++++--------- src/invidious/database/channels.cr | 13 ++- 2 files changed, 104 insertions(+), 48 deletions(-) diff --git a/src/invidious/channels/channels.cr b/src/invidious/channels/channels.cr index 3d771b8cb..4fb92a502 100644 --- a/src/invidious/channels/channels.cr +++ b/src/invidious/channels/channels.cr @@ -171,15 +171,86 @@ def fetch_channel(ucid, pull_all_videos : Bool) subscribed: nil, }) - LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos page") + LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos, shorts, and streams pages") videos, continuation = IV::Channel::Tabs.get_videos(channel) + shorts, shorts_continuation = IV::Channel::Tabs.get_shorts(channel_info) + livestreams, livestreams_continuation = IV::Channel::Tabs.get_livestreams(channel_info) + + LOGGER.trace("fetch_channel: #{ucid} : Extracting channel tab videos") + update_channel_videos(ucid, videos.select(SearchVideo)) + update_channel_videos(ucid, shorts.select(SearchVideo)) + update_channel_videos(ucid, livestreams.select(SearchVideo)) + + if pull_all_videos + update_all_channel_videos(ucid, channel, continuation) + update_all_channel_shorts(ucid, channel_info, shorts_continuation) + update_all_channel_livestreams(ucid, channel_info, livestreams_continuation) + end + + channel.updated = Time.utc + return channel +end + +private def update_all_channel_videos(ucid : String, channel : InvidiousChannel, continuation : String?) + loop do + break if continuation.nil? + + items, continuation = IV::Channel::Tabs.get_videos(channel, continuation: continuation) + videos = items.select(SearchVideo) + update_channel_videos(ucid, videos, skip_recent: true) + + break if videos.size < 25 + sleep 500.milliseconds + end +end + +private def update_all_channel_shorts(ucid : String, channel : AboutChannel, continuation : String?) + loop do + break if continuation.nil? + + items, continuation = IV::Channel::Tabs.get_shorts(channel, continuation: continuation) + videos = items.select(SearchVideo) + update_channel_videos(ucid, videos, skip_recent: true) + + break if videos.size < 25 + sleep 500.milliseconds + end +end + +private def update_all_channel_livestreams(ucid : String, channel : AboutChannel, continuation : String?) + loop do + break if continuation.nil? + + items, continuation = IV::Channel::Tabs.get_livestreams(channel, continuation: continuation) + videos = items.select(SearchVideo) + update_channel_videos(ucid, videos, skip_recent: true) + + break if videos.size < 25 + sleep 500.milliseconds + end +end + +private def update_channel_videos(ucid : String, videos : Array(SearchVideo), skip_recent : Bool = false) + stored_videos = {} of String => ChannelVideo + Invidious::Database::ChannelVideos.select(videos.map(&.id)).each do |video| + stored_videos[video.id] = video + end + + videos.each do |video| + published = stored_videos[video.id]?.try(&.published) || fetch_video_published_at(video.id) + unless published + LOGGER.warn("fetch_channel: #{ucid} : video #{video.id} : Could not fetch publication date") + next + end + + # We are notified of Red videos elsewhere (PubSub), which includes a correct published date, + # so since they don't provide a published date here we can safely ignore them. + next if skip_recent && Time.utc - published <= 1.minute - LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel") - videos.select(SearchVideo).each do |video| channel_video = ChannelVideo.new({ id: video.id, title: video.title, - published: video.published, + published: published, updated: Time.utc, ucid: video.ucid, author: video.author, @@ -192,7 +263,7 @@ def fetch_channel(ucid, pull_all_videos : Bool) LOGGER.trace("fetch_channel: #{ucid} : video #{video.id} : Updating or inserting video") # We don't include the 'premiere_timestamp' here because channel pages don't include them, - # meaning the above timestamp is always null + # meaning the above timestamp is always null. was_insert = Invidious::Database::ChannelVideos.insert(channel_video, preserve_timestamps_on_conflict: true) if was_insert @@ -202,43 +273,29 @@ def fetch_channel(ucid, pull_all_videos : Bool) LOGGER.trace("fetch_channel: #{ucid} : video #{video.id} : Updated") end end +end - if pull_all_videos - loop do - # Keep fetching videos using the continuation token retrieved earlier - videos, continuation = IV::Channel::Tabs.get_videos(channel, continuation: continuation) - - count = 0 - videos.select(SearchVideo).each do |video| - count += 1 - video = ChannelVideo.new({ - id: video.id, - title: video.title, - published: video.published, - updated: Time.utc, - ucid: video.ucid, - author: video.author, - length_seconds: video.length_seconds, - live_now: video.badges.live_now?, - premiere_timestamp: video.premiere_timestamp, - views: video.views, - }) - - # We are notified of Red videos elsewhere (PubSub), which includes a correct published date, - # so since they don't provide a published date here we can safely ignore them. - if Time.utc - video.published > 1.minute - was_insert = Invidious::Database::ChannelVideos.insert(video, preserve_timestamps_on_conflict: true) - if was_insert - NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video)) - end - end - end - - break if count < 25 - sleep 500.milliseconds - end +private def fetch_video_published_at(video_id : String) : Time? + response = YoutubeAPI.next({"videoId" => video_id, "params" => ""}) + if published = response.dig?("microformat", "playerMicroformatRenderer", "publishDate").try(&.as_s) + return parse_video_published_at(published) end - channel.updated = Time.utc - return channel + date_text = response.dig?( + "contents", "twoColumnWatchNextResults", "results", "results", "contents", 0, + "videoPrimaryInfoRenderer", "dateText", "simpleText" + ).try(&.as_s) + + return date_text.try do |text| + Time.parse(text.lchop("Scheduled for "), "%b %-d, %Y", Time::Location::UTC) + end +rescue ex + LOGGER.debug("fetch_video_published_at: #{video_id}: #{ex.message}") + return nil +end + +private def parse_video_published_at(published : String) : Time + return Time.parse_rfc3339(published) +rescue + return Time.parse(published, "%Y-%m-%d", Time::Location::UTC) end diff --git a/src/invidious/database/channels.cr b/src/invidious/database/channels.cr index 68f58ec1f..0ff97fa07 100644 --- a/src/invidious/database/channels.cr +++ b/src/invidious/database/channels.cr @@ -103,18 +103,16 @@ module Invidious::Database::ChannelVideos with_premiere_timestamp : Bool = false, preserve_timestamps_on_conflict : Bool = false, ) : Bool - if with_premiere_timestamp - last_items = "premiere_timestamp = $9, views = $10" - else - last_items = "views = $10" - end + premiere_timestamp_on_conflict = with_premiere_timestamp ? "premiere_timestamp = $9," : "" if preserve_timestamps_on_conflict published_on_conflict = "published = COALESCE(channel_videos.published, EXCLUDED.published)" - updated_on_conflict = "updated = CASE WHEN channel_videos.title IS DISTINCT FROM EXCLUDED.title OR channel_videos.ucid IS DISTINCT FROM EXCLUDED.ucid OR channel_videos.author IS DISTINCT FROM EXCLUDED.author OR channel_videos.length_seconds IS DISTINCT FROM EXCLUDED.length_seconds OR channel_videos.live_now IS DISTINCT FROM EXCLUDED.live_now THEN EXCLUDED.updated ELSE channel_videos.updated END" + views_on_conflict = "views = CASE WHEN EXCLUDED.views = 0 AND channel_videos.views > 0 THEN channel_videos.views ELSE EXCLUDED.views END" + updated_on_conflict = "updated = CASE WHEN channel_videos.title IS DISTINCT FROM EXCLUDED.title OR channel_videos.ucid IS DISTINCT FROM EXCLUDED.ucid OR channel_videos.author IS DISTINCT FROM EXCLUDED.author OR channel_videos.length_seconds IS DISTINCT FROM EXCLUDED.length_seconds OR channel_videos.live_now IS DISTINCT FROM EXCLUDED.live_now OR channel_videos.views IS DISTINCT FROM CASE WHEN EXCLUDED.views = 0 AND channel_videos.views > 0 THEN channel_videos.views ELSE EXCLUDED.views END THEN EXCLUDED.updated ELSE channel_videos.updated END" else published_on_conflict = "published = $3" updated_on_conflict = "updated = $4" + views_on_conflict = "views = $10" end request = <<-SQL @@ -122,7 +120,8 @@ module Invidious::Database::ChannelVideos VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) ON CONFLICT (id) DO UPDATE SET title = $2, #{published_on_conflict}, #{updated_on_conflict}, ucid = $5, - author = $6, length_seconds = $7, live_now = $8, #{last_items} + author = $6, length_seconds = $7, live_now = $8, + #{premiere_timestamp_on_conflict} #{views_on_conflict} RETURNING (xmax=0) AS was_insert SQL