Merge f2a550efe694437ec5144c60672f394460771f03 into d10f2a48021f1768f2d6ff3bd1b9f3de4e0b2d22

src/invidious/channels/channels.cr

@@ -159,36 +159,12 @@ def fetch_channel(ucid, pull_all_videos : Bool)
   LOGGER.debug("fetch_channel: #{ucid}")
   LOGGER.trace("fetch_channel: #{ucid} : pull_all_videos = #{pull_all_videos}")
 
-  namespaces = {
+  LOGGER.trace("fetch_channel: #{ucid} : Downloading channel information")
-    "yt"      => "http://www.youtube.com/xml/schemas/2015",
+  channel_info = get_about_info(ucid, nil)
-    "media"   => "http://search.yahoo.com/mrss/",
-    "default" => "http://www.w3.org/2005/Atom",
-  }
-
-  LOGGER.trace("fetch_channel: #{ucid} : Downloading RSS feed")
-  rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}").body
-  LOGGER.trace("fetch_channel: #{ucid} : Parsing RSS feed")
-  rss = XML.parse(rss)
-
-  author = rss.xpath_node("//default:feed/default:title", namespaces)
-  if !author
-    raise InfoException.new("Deleted or invalid channel")
-  end
-
-  author = author.content
-
-  # Auto-generated channels
-  # https://support.google.com/youtube/answer/2579942
-  if author.ends_with?(" - Topic") ||
-     {"Popular on YouTube", "Music", "Sports", "Gaming"}.includes? author
-    auto_generated = true
-  end
-
-  LOGGER.trace("fetch_channel: #{ucid} : author = #{author}, auto_generated = #{auto_generated}")
 
   channel = InvidiousChannel.new({
-    id:         ucid,
+    id:         channel_info.ucid,
-    author:     author,
+    author:     channel_info.author,
     updated:    Time.utc,
     deleted:    false,
     subscribed: nil,
@@ -197,61 +173,34 @@ def fetch_channel(ucid, pull_all_videos : Bool)
   LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos page")
   videos, continuation = IV::Channel::Tabs.get_videos(channel)
 
-  LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel RSS feed")
+  LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel response")
-  rss.xpath_nodes("//default:feed/default:entry", namespaces).each do |entry|
+  videos.select(SearchVideo).each do |channel_video|
-    video_id = entry.xpath_node("yt:videoId", namespaces).not_nil!.content
-    title = entry.xpath_node("default:title", namespaces).not_nil!.content
-
-    published = Time.parse_rfc3339(
-      entry.xpath_node("default:published", namespaces).not_nil!.content
-    )
-    updated = Time.parse_rfc3339(
-      entry.xpath_node("default:updated", namespaces).not_nil!.content
-    )
-
-    author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content
-    ucid = entry.xpath_node("yt:channelId", namespaces).not_nil!.content
-
-    views = entry
-      .xpath_node("media:group/media:community/media:statistics", namespaces)
-      .try &.["views"]?.try &.to_i64? || 0_i64
-
-    channel_video = videos
-      .select(SearchVideo)
-      .select(&.id.== video_id)[0]?
-
-    length_seconds = channel_video.try &.length_seconds
-    length_seconds ||= 0
-
-    live_now = channel_video.try &.badges.live_now?
-    live_now ||= false
-
-    premiere_timestamp = channel_video.try &.premiere_timestamp
-
     video = ChannelVideo.new({
-      id:                 video_id,
+      id:                 channel_video.id,
-      title:              title,
+      title:              channel_video.title,
-      published:          published,
+      published:          channel_video.published,
-      updated:            updated,
+      updated:            Time.utc,
-      ucid:               ucid,
+      ucid:               channel_info.auto_generated ? channel_video.ucid : channel.id,
-      author:             author,
+      author:             channel_info.auto_generated ? channel_video.author : channel.author,
-      length_seconds:     length_seconds,
+      length_seconds:     channel_video.length_seconds,
-      live_now:           live_now,
+      live_now:           channel_video.badges.live_now?,
-      premiere_timestamp: premiere_timestamp,
+      premiere_timestamp: channel_video.premiere_timestamp,
-      views:              views,
+      views:              channel_video.views,
     })
 
-    LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updating or inserting video")
+    LOGGER.trace("fetch_channel: #{ucid} : video #{channel_video.id} : Updating or inserting video")
 
     # We don't include the 'premiere_timestamp' here because channel pages don't include them,
     # meaning the above timestamp is always null
-    was_insert = Invidious::Database::ChannelVideos.insert(video)
+    # InnerTube provides relative publication labels. Once persisted, keep the
+    # existing timestamp so a later refresh cannot shift it forward.
+    was_insert = Invidious::Database::ChannelVideos.insert(video, update_published: false)
 
     if was_insert
-      LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Inserted, updating subscriptions")
+      LOGGER.trace("fetch_channel: #{ucid} : video #{channel_video.id} : Inserted, updating subscriptions")
       NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video))
     else
-      LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updated")
+      LOGGER.trace("fetch_channel: #{ucid} : video #{channel_video.id} : Updated")
     end
   end
 
@@ -279,7 +228,7 @@ def fetch_channel(ucid, pull_all_videos : Bool)
         # We are notified of Red videos elsewhere (PubSub), which includes a correct published date,
         # so since they don't provide a published date here we can safely ignore them.
         if Time.utc - video.published > 1.minute
-          was_insert = Invidious::Database::ChannelVideos.insert(video)
+          was_insert = Invidious::Database::ChannelVideos.insert(video, update_published: false)
           if was_insert
             NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video))
           end

src/invidious/database/channels.cr

@@ -98,18 +98,20 @@ module Invidious::Database::ChannelVideos
   # -------------------
 
   # This function returns the status of the query (i.e: success?)
-  def insert(video : ChannelVideo, with_premiere_timestamp : Bool = false) : Bool
+  def insert(video : ChannelVideo, with_premiere_timestamp : Bool = false, update_published : Bool = true) : Bool
     if with_premiere_timestamp
       last_items = "premiere_timestamp = $9, views = $10"
     else
       last_items = "views = $10"
     end
 
+    published_item = update_published ? "published = $3," : ""
+
     request = <<-SQL
       INSERT INTO channel_videos
       VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
       ON CONFLICT (id) DO UPDATE
-      SET title = $2, published = $3, updated = $4, ucid = $5,
+      SET title = $2, #{published_item} updated = $4, ucid = $5,
           author = $6, length_seconds = $7, live_now = $8, #{last_items}
       RETURNING (xmax=0) AS was_insert
     SQL

src/invidious/jobs/refresh_channels_job.cr

@@ -43,7 +43,7 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob
               end
             rescue ex
               LOGGER.error("RefreshChannelsJob: #{id} : #{ex.message}")
-              if ex.message == "Deleted or invalid channel"
+              if {"Deleted or invalid channel", "This channel does not exist."}.includes?(ex.message)
                 Invidious::Database::Channels.update_mark_deleted(id)
               else
                 lim_fibers = 1