mirror of
https://github.com/iv-org/invidious.git
synced 2026-08-16 21:10:53 -05:00
Merge f2a550efe694437ec5144c60672f394460771f03 into d10f2a48021f1768f2d6ff3bd1b9f3de4e0b2d22
This commit is contained in:
leeweisa3-maker
GitHub
commit
b7eebc77c7
@ -159,36 +159,12 @@ def fetch_channel(ucid, pull_all_videos : Bool)
|
||||
LOGGER.debug("fetch_channel: #{ucid}")
|
||||
LOGGER.trace("fetch_channel: #{ucid} : pull_all_videos = #{pull_all_videos}")
|
||||
|
||||
namespaces = {
|
||||
"yt" => "http://www.youtube.com/xml/schemas/2015",
|
||||
"media" => "http://search.yahoo.com/mrss/",
|
||||
"default" => "http://www.w3.org/2005/Atom",
|
||||
}
|
||||
|
||||
LOGGER.trace("fetch_channel: #{ucid} : Downloading RSS feed")
|
||||
rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}").body
|
||||
LOGGER.trace("fetch_channel: #{ucid} : Parsing RSS feed")
|
||||
rss = XML.parse(rss)
|
||||
|
||||
author = rss.xpath_node("//default:feed/default:title", namespaces)
|
||||
if !author
|
||||
raise InfoException.new("Deleted or invalid channel")
|
||||
end
|
||||
|
||||
author = author.content
|
||||
|
||||
# Auto-generated channels
|
||||
# https://support.google.com/youtube/answer/2579942
|
||||
if author.ends_with?(" - Topic") ||
|
||||
{"Popular on YouTube", "Music", "Sports", "Gaming"}.includes? author
|
||||
auto_generated = true
|
||||
end
|
||||
|
||||
LOGGER.trace("fetch_channel: #{ucid} : author = #{author}, auto_generated = #{auto_generated}")
|
||||
LOGGER.trace("fetch_channel: #{ucid} : Downloading channel information")
|
||||
channel_info = get_about_info(ucid, nil)
|
||||
|
||||
channel = InvidiousChannel.new({
|
||||
id: ucid,
|
||||
author: author,
|
||||
id: channel_info.ucid,
|
||||
author: channel_info.author,
|
||||
updated: Time.utc,
|
||||
deleted: false,
|
||||
subscribed: nil,
|
||||
@ -197,61 +173,34 @@ def fetch_channel(ucid, pull_all_videos : Bool)
|
||||
LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos page")
|
||||
videos, continuation = IV::Channel::Tabs.get_videos(channel)
|
||||
|
||||
LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel RSS feed")
|
||||
rss.xpath_nodes("//default:feed/default:entry", namespaces).each do |entry|
|
||||
video_id = entry.xpath_node("yt:videoId", namespaces).not_nil!.content
|
||||
title = entry.xpath_node("default:title", namespaces).not_nil!.content
|
||||
|
||||
published = Time.parse_rfc3339(
|
||||
entry.xpath_node("default:published", namespaces).not_nil!.content
|
||||
)
|
||||
updated = Time.parse_rfc3339(
|
||||
entry.xpath_node("default:updated", namespaces).not_nil!.content
|
||||
)
|
||||
|
||||
author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content
|
||||
ucid = entry.xpath_node("yt:channelId", namespaces).not_nil!.content
|
||||
|
||||
views = entry
|
||||
.xpath_node("media:group/media:community/media:statistics", namespaces)
|
||||
.try &.["views"]?.try &.to_i64? || 0_i64
|
||||
|
||||
channel_video = videos
|
||||
.select(SearchVideo)
|
||||
.select(&.id.== video_id)[0]?
|
||||
|
||||
length_seconds = channel_video.try &.length_seconds
|
||||
length_seconds ||= 0
|
||||
|
||||
live_now = channel_video.try &.badges.live_now?
|
||||
live_now ||= false
|
||||
|
||||
premiere_timestamp = channel_video.try &.premiere_timestamp
|
||||
|
||||
LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel response")
|
||||
videos.select(SearchVideo).each do |channel_video|
|
||||
video = ChannelVideo.new({
|
||||
id: video_id,
|
||||
title: title,
|
||||
published: published,
|
||||
updated: updated,
|
||||
ucid: ucid,
|
||||
author: author,
|
||||
length_seconds: length_seconds,
|
||||
live_now: live_now,
|
||||
premiere_timestamp: premiere_timestamp,
|
||||
views: views,
|
||||
id: channel_video.id,
|
||||
title: channel_video.title,
|
||||
published: channel_video.published,
|
||||
updated: Time.utc,
|
||||
ucid: channel_info.auto_generated ? channel_video.ucid : channel.id,
|
||||
author: channel_info.auto_generated ? channel_video.author : channel.author,
|
||||
length_seconds: channel_video.length_seconds,
|
||||
live_now: channel_video.badges.live_now?,
|
||||
premiere_timestamp: channel_video.premiere_timestamp,
|
||||
views: channel_video.views,
|
||||
})
|
||||
|
||||
LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updating or inserting video")
|
||||
LOGGER.trace("fetch_channel: #{ucid} : video #{channel_video.id} : Updating or inserting video")
|
||||
|
||||
# We don't include the 'premiere_timestamp' here because channel pages don't include them,
|
||||
# meaning the above timestamp is always null
|
||||
was_insert = Invidious::Database::ChannelVideos.insert(video)
|
||||
# InnerTube provides relative publication labels. Once persisted, keep the
|
||||
# existing timestamp so a later refresh cannot shift it forward.
|
||||
was_insert = Invidious::Database::ChannelVideos.insert(video, update_published: false)
|
||||
|
||||
if was_insert
|
||||
LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Inserted, updating subscriptions")
|
||||
LOGGER.trace("fetch_channel: #{ucid} : video #{channel_video.id} : Inserted, updating subscriptions")
|
||||
NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video))
|
||||
else
|
||||
LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updated")
|
||||
LOGGER.trace("fetch_channel: #{ucid} : video #{channel_video.id} : Updated")
|
||||
end
|
||||
end
|
||||
|
||||
@ -279,7 +228,7 @@ def fetch_channel(ucid, pull_all_videos : Bool)
|
||||
# We are notified of Red videos elsewhere (PubSub), which includes a correct published date,
|
||||
# so since they don't provide a published date here we can safely ignore them.
|
||||
if Time.utc - video.published > 1.minute
|
||||
was_insert = Invidious::Database::ChannelVideos.insert(video)
|
||||
was_insert = Invidious::Database::ChannelVideos.insert(video, update_published: false)
|
||||
if was_insert
|
||||
NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video))
|
||||
end
|
||||
|
||||
@ -98,18 +98,20 @@ module Invidious::Database::ChannelVideos
|
||||
# -------------------
|
||||
|
||||
# This function returns the status of the query (i.e: success?)
|
||||
def insert(video : ChannelVideo, with_premiere_timestamp : Bool = false) : Bool
|
||||
def insert(video : ChannelVideo, with_premiere_timestamp : Bool = false, update_published : Bool = true) : Bool
|
||||
if with_premiere_timestamp
|
||||
last_items = "premiere_timestamp = $9, views = $10"
|
||||
else
|
||||
last_items = "views = $10"
|
||||
end
|
||||
|
||||
published_item = update_published ? "published = $3," : ""
|
||||
|
||||
request = <<-SQL
|
||||
INSERT INTO channel_videos
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
|
||||
ON CONFLICT (id) DO UPDATE
|
||||
SET title = $2, published = $3, updated = $4, ucid = $5,
|
||||
SET title = $2, #{published_item} updated = $4, ucid = $5,
|
||||
author = $6, length_seconds = $7, live_now = $8, #{last_items}
|
||||
RETURNING (xmax=0) AS was_insert
|
||||
SQL
|
||||
|
||||
@ -43,7 +43,7 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob
|
||||
end
|
||||
rescue ex
|
||||
LOGGER.error("RefreshChannelsJob: #{id} : #{ex.message}")
|
||||
if ex.message == "Deleted or invalid channel"
|
||||
if {"Deleted or invalid channel", "This channel does not exist."}.includes?(ex.message)
|
||||
Invidious::Database::Channels.update_mark_deleted(id)
|
||||
else
|
||||
lim_fibers = 1
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user