Merge 073396ce3387441155ec3270a1386e8a3b01229c into d51a7a44ad91d2fa7d1330970a15a0d8f365f250

config/config.example.yml

@@ -939,6 +939,21 @@ default_user_preferences:
   ##
   #sort: published
 
+  ##
+  ## In the "Subscription" feed, hide shorts.
+  ##
+  ## Accepted values: true, false
+  ## Default: false
+  ##
+  #hide_shorts: false
+
+  ##
+  ## In the "Subscription" feed, hide livestreams.
+  ##
+  ## Accepted values: true, false
+  ## Default: false
+  ##
+  #hide_livestreams: false
 
   # -----------------------------
   #  Miscellaneous

config/sql/channel_videos.sql

@@ -2,6 +2,14 @@
 
 -- DROP TABLE public.channel_videos;
 
+CREATE TYPE public.video_type AS ENUM
+(
+    'Video',
+    'Short',
+    'Livestream',
+    'Scheduled'
+);
+
 CREATE TABLE IF NOT EXISTS public.channel_videos
 (
   id text NOT NULL,
@@ -14,6 +22,7 @@ CREATE TABLE IF NOT EXISTS public.channel_videos
   live_now boolean,
   premiere_timestamp timestamp with time zone,
   views bigint,
+  video_type video_type,
   CONSTRAINT channel_videos_id_key UNIQUE (id)
 );
 

locales/en-US.json

@@ -133,6 +133,8 @@
     "Only show latest video from channel: ": "Only show latest video from channel: ",
     "Only show latest unwatched video from channel: ": "Only show latest unwatched video from channel: ",
     "preferences_unseen_only_label": "Only show unwatched: ",
+    "preferences_hide_shorts_label": "Hide shorts: ",
+    "preferences_hide_livestreams_label": "Hide livestreams: ",
     "preferences_notifications_only_label": "Only show notifications (if there are any): ",
     "Enable web notifications": "Enable web notifications",
     "`x` uploaded a video": "`x` uploaded a video",

src/invidious/channels/channels.cr

@@ -21,6 +21,14 @@ struct ChannelVideo
   property live_now : Bool = false
   property premiere_timestamp : Time? = nil
   property views : Int64? = nil
+  @[DB::Field(converter: ChannelVideo::VideoTypeConverter)]
+  property video_type : VideoType = VideoType::Video
+
+  module VideoTypeConverter
+    def self.from_rs(rs)
+      return VideoType.parse(String.new(rs.read(Slice(UInt8))))
+    end
+  end
 
   def to_json(locale, json : JSON::Builder)
     json.object do
@@ -200,6 +208,8 @@ def fetch_channel(ucid, pull_all_videos : Bool)
   LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel RSS feed")
   rss.xpath_nodes("//default:feed/default:entry", namespaces).each do |entry|
     video_id = entry.xpath_node("yt:videoId", namespaces).not_nil!.content
+    database_video = Invidious::Database::ChannelVideos.select([video_id])
+
     title = entry.xpath_node("default:title", namespaces).not_nil!.content
 
     published = Time.parse_rfc3339(
@@ -216,30 +226,46 @@ def fetch_channel(ucid, pull_all_videos : Bool)
       .xpath_node("media:group/media:community/media:statistics", namespaces)
       .try &.["views"]?.try &.to_i64? || 0_i64
 
-    channel_video = videos
+    # If there is no update for the video, only update the views
-      .select(SearchVideo)
+    if database_video.size > 0 && updated == database_video[0].updated
-      .select(&.id.== video_id)[0]?
+      video = database_video[0]
+      video.views = views
+    else
+      channel_video = videos
+        .select(SearchVideo)
+        .select(&.id.== video_id)[0]?
 
-    length_seconds = channel_video.try &.length_seconds
+      # Not a video, either a short or a livestream
-    length_seconds ||= 0
+      # Fetch individual for info
+      if channel_video.nil?
+        short_or_live = fetch_video(video_id, "")
+        video_type = short_or_live.video_type
+        length_seconds = short_or_live.length_seconds
+        live_now = short_or_live.live_now
+        premiere_timestamp = short_or_live.premiere_timestamp
+      else
+        video_type = VideoType::Video
+        length_seconds = channel_video.try &.length_seconds
+        live_now = channel_video.try &.badges.live_now?
+      end
 
-    live_now = channel_video.try &.badges.live_now?
+      length_seconds ||= 0
-    live_now ||= false
+      live_now ||= false
 
-    premiere_timestamp = channel_video.try &.premiere_timestamp
+      video = ChannelVideo.new({
-
+        id:                 video_id,
-    video = ChannelVideo.new({
+        title:              title,
-      id:                 video_id,
+        published:          published,
-      title:              title,
+        updated:            updated,
-      published:          published,
+        ucid:               ucid,
-      updated:            updated,
+        author:             author,
-      ucid:               ucid,
+        length_seconds:     length_seconds,
-      author:             author,
+        live_now:           live_now,
-      length_seconds:     length_seconds,
+        premiere_timestamp: premiere_timestamp,
-      live_now:           live_now,
+        views:              views,
-      premiere_timestamp: premiere_timestamp,
+        video_type:         video_type,
-      views:              views,
+      })
-    })
+    end
 
     LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updating or inserting video")
 
@@ -274,6 +300,7 @@ def fetch_channel(ucid, pull_all_videos : Bool)
           live_now:           video.badges.live_now?,
           premiere_timestamp: video.premiere_timestamp,
           views:              video.views,
+          video_type:         VideoType::Video
         })
 
         # We are notified of Red videos elsewhere (PubSub), which includes a correct published date,

src/invidious/config.cr

@@ -47,6 +47,8 @@ struct ConfigPreferences
   property thin_mode : Bool = false
   property unseen_only : Bool = false
   property video_loop : Bool = false
+  property hide_shorts : Bool = false
+  property hide_livestreams : Bool = false
   property extend_desc : Bool = false
   property volume : Int32 = 100
   property vr_mode : Bool = true

src/invidious/database/channels.cr

@@ -100,14 +100,14 @@ module Invidious::Database::ChannelVideos
   # This function returns the status of the query (i.e: success?)
   def insert(video : ChannelVideo, with_premiere_timestamp : Bool = false) : Bool
     if with_premiere_timestamp
-      last_items = "premiere_timestamp = $9, views = $10"
+      last_items = "premiere_timestamp = $9, views = $10, video_type = $11"
     else
-      last_items = "views = $10"
+      last_items = "views = $10, video_type = $11"
     end
 
     request = <<-SQL
       INSERT INTO channel_videos
-      VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
+      VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
       ON CONFLICT (id) DO UPDATE
       SET title = $2, published = $3, updated = $4, ucid = $5,
           author = $6, length_seconds = $7, live_now = $8, #{last_items}

src/invidious/routes/feeds.cr

@@ -438,6 +438,7 @@ module Invidious::Routes::Feeds
           live_now:           video.live_now,
           premiere_timestamp: video.premiere_timestamp,
           views:              video.views,
+          video_type:         video.video_type
         })
 
         was_insert = Invidious::Database::ChannelVideos.insert(video, with_premiere_timestamp: true)

src/invidious/routes/preferences.cr

@@ -143,6 +143,14 @@ module Invidious::Routes::PreferencesRoute
     notifications_only ||= "off"
     notifications_only = notifications_only == "on"
 
+    hide_shorts = env.params.body["hide_shorts"]?.try &.as(String)
+    hide_shorts ||= "off"
+    hide_shorts = hide_shorts == "on"
+
+    hide_livestreams = env.params.body["hide_livestreams"]?.try &.as(String)
+    hide_livestreams ||= "off"
+    hide_livestreams = hide_livestreams == "on"
+
     default_playlist = env.params.body["default_playlist"]?.try &.as(String)
 
     # Convert to JSON and back again to take advantage of converters used for compatibility
@@ -182,6 +190,8 @@ module Invidious::Routes::PreferencesRoute
       show_nick:                   show_nick,
       save_player_pos:             save_player_pos,
       default_playlist:            default_playlist,
+      hide_shorts:                 hide_shorts,
+      hide_livestreams:            hide_livestreams,
     }.to_json)
 
     if user = env.get? "user"

src/invidious/user/preferences.cr

@@ -57,6 +57,8 @@ struct Preferences
   property volume : Int32 = CONFIG.default_user_preferences.volume
   property save_player_pos : Bool = CONFIG.default_user_preferences.save_player_pos
   property default_playlist : String? = nil
+  property hide_shorts : Bool = CONFIG.default_user_preferences.hide_shorts
+  property hide_livestreams : Bool = CONFIG.default_user_preferences.hide_livestreams
 
   module BoolToString
     def self.to_json(value : String, json : JSON::Builder)

src/invidious/users.cr

@@ -29,6 +29,19 @@ def get_subscription_feed(user, max_results = 40, page = 1)
   notifications = Invidious::Database::Users.select_notifications(user)
   view_name = "subscriptions_#{sha256(user.email)}"
 
+  types_to_fetch = [VideoType::Video, VideoType::Short, VideoType::Livestream, VideoType::Scheduled]
+  if user.preferences.hide_shorts
+    types_to_fetch.delete(VideoType::Short)
+  end
+  # Note: hide_livestreams also hides scheduled/premiere videos (VideoType::Scheduled),
+  # since they are typically upcoming livestreams.
+  if user.preferences.hide_livestreams
+    [VideoType::Livestream, VideoType::Scheduled].each { |v| types_to_fetch.delete(v) }
+  end
+
+  types_to_fetch = types_to_fetch.map { |type| "'#{type}'" }.join(", ")
+  LOGGER.trace("Types to fetch: #{types_to_fetch}")
+
   if user.preferences.notifications_only && !notifications.empty?
     # Only show notifications
     notifications = Invidious::Database::ChannelVideos.select(notifications)
@@ -58,11 +71,11 @@ def get_subscription_feed(user, max_results = 40, page = 1)
         else
           values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}"
         end
-        videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY ucid, published DESC", as: ChannelVideo)
+
+        videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} WHERE NOT id = ANY (#{values}) AND video_type IN (#{types_to_fetch}) ORDER BY ucid, published DESC", as: ChannelVideo)
       else
         # Show latest video from each channel
-
+        videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} WHERE video_type IN (#{types_to_fetch}) ORDER BY ucid, published DESC", as: ChannelVideo)
-        videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} ORDER BY ucid, published DESC", as: ChannelVideo)
       end
 
       videos.sort_by!(&.published).reverse!
@@ -75,11 +88,11 @@ def get_subscription_feed(user, max_results = 40, page = 1)
         else
           values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}"
         end
-        videos = PG_DB.query_all("SELECT * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo)
+
+        videos = PG_DB.query_all("SELECT * FROM #{view_name} WHERE NOT id = ANY (#{values}) AND video_type IN (#{types_to_fetch}) ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo)
       else
         # Sort subscriptions as normal
-
+        videos = PG_DB.query_all("SELECT * FROM #{view_name} WHERE video_type IN (#{types_to_fetch}) ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo)
-        videos = PG_DB.query_all("SELECT * FROM #{view_name} ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo)
       end
     end
 

src/invidious/videos.cr

@@ -1,5 +1,6 @@
 enum VideoType
   Video
+  Short
   Livestream
   Scheduled
 end

src/invidious/videos/parser.cr

@@ -218,6 +218,8 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
   post_live_dvr = video_details.dig?("isPostLiveDvr")
     .try &.as_bool || false
 
+  is_short = microformat.dig?("isShortsEligible").try &.as_bool || false
+
   # Extra video infos
 
   allowed_regions = microformat["availableCountries"]?
@@ -394,8 +396,9 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
   end
 
   # Return data
-
+  if is_short
-  if live_now
+    video_type = VideoType::Short
+  elsif live_now
     video_type = VideoType::Livestream
   elsif !premiere_timestamp.nil?
     video_type = VideoType::Scheduled

src/invidious/views/user/preferences.ecr

@@ -229,6 +229,16 @@
                     <input name="watch_history" id="watch_history" type="checkbox" <% if preferences.watch_history %>checked<% end %>>
                 </div>
 
+                <div class="pure-control-group">
+                    <label for="hide_shorts"><%= translate(locale, "preferences_hide_shorts_label") %></label>
+                    <input name="hide_shorts" id="hide_shorts" type="checkbox" <% if preferences.hide_shorts %>checked<% end %>>
+                </div>
+
+                <div class="pure-control-group">
+                    <label for="hide_livestreams"><%= translate(locale, "preferences_hide_livestreams_label") %></label>
+                    <input name="hide_livestreams" id="hide_livestreams" type="checkbox" <% if preferences.hide_livestreams %>checked<% end %>>
+                </div>
+
                 <div class="pure-control-group">
                     <label for="annotations_subscribed"><%= translate(locale, "preferences_annotations_subscribed_label") %></label>
                     <input name="annotations_subscribed" id="annotations_subscribed" type="checkbox" <% if preferences.annotations_subscribed %>checked<% end %>>