VeritableValor/invidious
1
0
Fork 0
mirror of https://github.com/iv-org/invidious.git synced 2026-09-26 11:35:27 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix: refresh all channel content tabs

Browse Source
This commit is contained in:
Agastya 2026-09-26 09:54:55 +05:30
parent b5fcc9bdbc
commit 8118fa612e
2 changed files with 104 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

139
src/invidious/channels/channels.cr
View File

@ -171,15 +171,86 @@ def fetch_channel(ucid, pull_all_videos : Bool)
subscribed: nil,
})
LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos page")
LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos, shorts, and streams pages")
videos, continuation = IV::Channel::Tabs.get_videos(channel)
shorts, shorts_continuation = IV::Channel::Tabs.get_shorts(channel_info)
livestreams, livestreams_continuation = IV::Channel::Tabs.get_livestreams(channel_info)
LOGGER.trace("fetch_channel: #{ucid} : Extracting channel tab videos")
update_channel_videos(ucid, videos.select(SearchVideo))
update_channel_videos(ucid, shorts.select(SearchVideo))
update_channel_videos(ucid, livestreams.select(SearchVideo))
if pull_all_videos
update_all_channel_videos(ucid, channel, continuation)
update_all_channel_shorts(ucid, channel_info, shorts_continuation)
update_all_channel_livestreams(ucid, channel_info, livestreams_continuation)
end
channel.updated = Time.utc
return channel
end
private def update_all_channel_videos(ucid : String, channel : InvidiousChannel, continuation : String?)
loop do
break if continuation.nil?
items, continuation = IV::Channel::Tabs.get_videos(channel, continuation: continuation)
videos = items.select(SearchVideo)
update_channel_videos(ucid, videos, skip_recent: true)
break if videos.size < 25
sleep 500.milliseconds
end
end
private def update_all_channel_shorts(ucid : String, channel : AboutChannel, continuation : String?)
loop do
break if continuation.nil?
items, continuation = IV::Channel::Tabs.get_shorts(channel, continuation: continuation)
videos = items.select(SearchVideo)
update_channel_videos(ucid, videos, skip_recent: true)
break if videos.size < 25
sleep 500.milliseconds
end
end
private def update_all_channel_livestreams(ucid : String, channel : AboutChannel, continuation : String?)
loop do
break if continuation.nil?
items, continuation = IV::Channel::Tabs.get_livestreams(channel, continuation: continuation)
videos = items.select(SearchVideo)
update_channel_videos(ucid, videos, skip_recent: true)
break if videos.size < 25
sleep 500.milliseconds
end
end
private def update_channel_videos(ucid : String, videos : Array(SearchVideo), skip_recent : Bool = false)
stored_videos = {} of String => ChannelVideo
Invidious::Database::ChannelVideos.select(videos.map(&.id)).each do |video|
stored_videos[video.id] = video
end
videos.each do |video|
published = stored_videos[video.id]?.try(&.published) || fetch_video_published_at(video.id)
unless published
LOGGER.warn("fetch_channel: #{ucid} : video #{video.id} : Could not fetch publication date")
next
end
# We are notified of Red videos elsewhere (PubSub), which includes a correct published date,
# so since they don't provide a published date here we can safely ignore them.
next if skip_recent && Time.utc - published <= 1.minute
LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel")
videos.select(SearchVideo).each do |video|
channel_video = ChannelVideo.new({
id: video.id,
title: video.title,
published: video.published,
published: published,
updated: Time.utc,
ucid: video.ucid,
author: video.author,
@ -192,7 +263,7 @@ def fetch_channel(ucid, pull_all_videos : Bool)
LOGGER.trace("fetch_channel: #{ucid} : video #{video.id} : Updating or inserting video")
# We don't include the 'premiere_timestamp' here because channel pages don't include them,
# meaning the above timestamp is always null
# meaning the above timestamp is always null.
was_insert = Invidious::Database::ChannelVideos.insert(channel_video, preserve_timestamps_on_conflict: true)
if was_insert
@ -202,43 +273,29 @@ def fetch_channel(ucid, pull_all_videos : Bool)
LOGGER.trace("fetch_channel: #{ucid} : video #{video.id} : Updated")
end
end
end
if pull_all_videos
loop do
# Keep fetching videos using the continuation token retrieved earlier
videos, continuation = IV::Channel::Tabs.get_videos(channel, continuation: continuation)
count = 0
videos.select(SearchVideo).each do |video|
count += 1
video = ChannelVideo.new({
id: video.id,
title: video.title,
published: video.published,
updated: Time.utc,
ucid: video.ucid,
author: video.author,
length_seconds: video.length_seconds,
live_now: video.badges.live_now?,
premiere_timestamp: video.premiere_timestamp,
views: video.views,
})
# We are notified of Red videos elsewhere (PubSub), which includes a correct published date,
# so since they don't provide a published date here we can safely ignore them.
if Time.utc - video.published > 1.minute
was_insert = Invidious::Database::ChannelVideos.insert(video, preserve_timestamps_on_conflict: true)
if was_insert
NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video))
end
end
end
break if count < 25
sleep 500.milliseconds
end
private def fetch_video_published_at(video_id : String) : Time?
response = YoutubeAPI.next({"videoId" => video_id, "params" => ""})
if published = response.dig?("microformat", "playerMicroformatRenderer", "publishDate").try(&.as_s)
return parse_video_published_at(published)
end
channel.updated = Time.utc
return channel
date_text = response.dig?(
"contents", "twoColumnWatchNextResults", "results", "results", "contents", 0,
"videoPrimaryInfoRenderer", "dateText", "simpleText"
).try(&.as_s)
return date_text.try do |text|
Time.parse(text.lchop("Scheduled for "), "%b %-d, %Y", Time::Location::UTC)
end
rescue ex
LOGGER.debug("fetch_video_published_at: #{video_id}: #{ex.message}")
return nil
end
private def parse_video_published_at(published : String) : Time
return Time.parse_rfc3339(published)
rescue
return Time.parse(published, "%Y-%m-%d", Time::Location::UTC)
end

13
src/invidious/database/channels.cr
View File

@ -103,18 +103,16 @@ module Invidious::Database::ChannelVideos
with_premiere_timestamp : Bool = false,
preserve_timestamps_on_conflict : Bool = false,
) : Bool
if with_premiere_timestamp
last_items = "premiere_timestamp = $9, views = $10"
else
last_items = "views = $10"
end
premiere_timestamp_on_conflict = with_premiere_timestamp ? "premiere_timestamp = $9," : ""
if preserve_timestamps_on_conflict
published_on_conflict = "published = COALESCE(channel_videos.published, EXCLUDED.published)"
updated_on_conflict = "updated = CASE WHEN channel_videos.title IS DISTINCT FROM EXCLUDED.title OR channel_videos.ucid IS DISTINCT FROM EXCLUDED.ucid OR channel_videos.author IS DISTINCT FROM EXCLUDED.author OR channel_videos.length_seconds IS DISTINCT FROM EXCLUDED.length_seconds OR channel_videos.live_now IS DISTINCT FROM EXCLUDED.live_now THEN EXCLUDED.updated ELSE channel_videos.updated END"
views_on_conflict = "views = CASE WHEN EXCLUDED.views = 0 AND channel_videos.views > 0 THEN channel_videos.views ELSE EXCLUDED.views END"
updated_on_conflict = "updated = CASE WHEN channel_videos.title IS DISTINCT FROM EXCLUDED.title OR channel_videos.ucid IS DISTINCT FROM EXCLUDED.ucid OR channel_videos.author IS DISTINCT FROM EXCLUDED.author OR channel_videos.length_seconds IS DISTINCT FROM EXCLUDED.length_seconds OR channel_videos.live_now IS DISTINCT FROM EXCLUDED.live_now OR channel_videos.views IS DISTINCT FROM CASE WHEN EXCLUDED.views = 0 AND channel_videos.views > 0 THEN channel_videos.views ELSE EXCLUDED.views END THEN EXCLUDED.updated ELSE channel_videos.updated END"
else
published_on_conflict = "published = $3"
updated_on_conflict = "updated = $4"
views_on_conflict = "views = $10"
end
request = <<-SQL
@ -122,7 +120,8 @@ module Invidious::Database::ChannelVideos
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
ON CONFLICT (id) DO UPDATE
SET title = $2, #{published_on_conflict}, #{updated_on_conflict}, ucid = $5,
author = $6, length_seconds = $7, live_now = $8, #{last_items}
author = $6, length_seconds = $7, live_now = $8,
#{premiere_timestamp_on_conflict} #{views_on_conflict}
RETURNING (xmax=0) AS was_insert
SQL