Update logic to process shorts/livestreams

[config/config.example.yml]
- Separate hide_shorts_and_live to hide_shorts and hide_livestreams

[config/sql/channel_videos.sql]
- Introduce enum video_type
- Include video_type as new column for channel_videos

[locales/en-US.json]
- Add labels for new settings

[src/invidious/channels/channels.cr]
- Add property video_type of type VideoType to ChannelVideo struct
- Add deserializer module for conversion from database entry to enum
- Add check if we already have a video in the database.
  If the video `updated` field has no been updated, only update views
- Add check whether a video is in the `videos` array. If this is not
  the case, fetch the individual video for `video_type` as well as
  `length_videos`

[src/invidious/config.cr]
- Separate hide_shorts_and_live property
  to hide_shorts and hide_livestreams properties

[src/invidious/database/channels.cr]
- Include video_type in database insert for ChannelVideo

[src/invidious/routes/preferences.cr]
- Separate hide_shorts_and_live setting to hide_shorts and hide_livestreams

[src/invidious/users.cr]
- Accumulate VideoTypes in an array and query on these types
- Remove paths for hide_shorts_and_live

[src/invidious/videos.cr]
- Add `Short` entry to VideoType enum

[src/invidious/videos/parser.cr]
- Add check whether a video is a short

config/config.example.yml

@@ -940,13 +940,20 @@ default_user_preferences:
   #sort: published
 
   ##
-  ## In the "Subscription" feed, Only show the videos, no shorts
+  ## In the "Subscription" feed, hide shorts.
-  ## or livestreams.
   ##
   ## Accepted values: true, false
   ## Default: false
   ##
-  #hide_shorts_and_live: false
+  #hide_shorts: false
+
+  ##
+  ## In the "Subscription" feed, hide livestreams.
+  ##
+  ## Accepted values: true, false
+  ## Default: false
+  ##
+  #hide_livestreams: false
 
   # -----------------------------
   #  Miscellaneous

config/sql/channel_videos.sql

@@ -2,6 +2,14 @@
 
 -- DROP TABLE public.channel_videos;
 
+CREATE TYPE public.video_type AS ENUM
+(
+    'Video',
+    'Short',
+    'Livestream',
+    'Scheduled'
+);
+
 CREATE TABLE IF NOT EXISTS public.channel_videos
 (
   id text NOT NULL,
@@ -14,6 +22,7 @@ CREATE TABLE IF NOT EXISTS public.channel_videos
   live_now boolean,
   premiere_timestamp timestamp with time zone,
   views bigint,
+  video_type video_type,
   CONSTRAINT channel_videos_id_key UNIQUE (id)
 );
 

locales/en-US.json

@@ -133,7 +133,8 @@
     "Only show latest video from channel: ": "Only show latest video from channel: ",
     "Only show latest unwatched video from channel: ": "Only show latest unwatched video from channel: ",
     "preferences_unseen_only_label": "Only show unwatched: ",
-    "preferences_hide_shorts_and_live_label": "Hide shorts and live streams: ",
+    "preferences_hide_shorts_label": "Hide shorts: ",
+    "preferences_hide_livestreams_label": "Hide livestreams: ",
     "preferences_notifications_only_label": "Only show notifications (if there are any): ",
     "Enable web notifications": "Enable web notifications",
     "`x` uploaded a video": "`x` uploaded a video",

src/invidious/channels/channels.cr

@@ -21,6 +21,14 @@ struct ChannelVideo
   property live_now : Bool = false
   property premiere_timestamp : Time? = nil
   property views : Int64? = nil
+  @[DB::Field(converter: ChannelVideo::VideoTypeConverter)]
+  property video_type : VideoType = VideoType::Video
+
+  module VideoTypeConverter
+    def self.from_rs(rs)
+      return VideoType.parse(String.new(rs.read(Slice(UInt8))))
+    end
+  end
 
   def to_json(locale, json : JSON::Builder)
     json.object do
@@ -200,6 +208,8 @@ def fetch_channel(ucid, pull_all_videos : Bool)
   LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel RSS feed")
   rss.xpath_nodes("//default:feed/default:entry", namespaces).each do |entry|
     video_id = entry.xpath_node("yt:videoId", namespaces).not_nil!.content
+    database_video = Invidious::Database::ChannelVideos.select([video_id])
+
     title = entry.xpath_node("default:title", namespaces).not_nil!.content
 
     published = Time.parse_rfc3339(
@@ -216,17 +226,31 @@ def fetch_channel(ucid, pull_all_videos : Bool)
       .xpath_node("media:group/media:community/media:statistics", namespaces)
       .try &.["views"]?.try &.to_i64? || 0_i64
 
+    # If there is no update for the video, only update the views
+    if database_video.size > 0 && updated == database_video[0].updated
+        video = database_video[0]
+        video.views = views
+    else
       channel_video = videos
         .select(SearchVideo)
         .select(&.id.== video_id)[0]?
 
+      # Not a video, either a short of a livestream
+      # Fetch invididual for info
+      if channel_video.nil?
+        short_or_live = fetch_video(video_id, "")
+        video_type = short_or_live.video_type
+        length_seconds = short_or_live.length_seconds
+        live_now = short_or_live.live_now
+        premiere_timestamp = short_or_live.premiere_timestamp
+      else
+        video_type = VideoType::Video
         length_seconds = channel_video.try &.length_seconds
-    length_seconds ||= 0
-
         live_now = channel_video.try &.badges.live_now?
-    live_now ||= false
+      end
 
-    premiere_timestamp = channel_video.try &.premiere_timestamp
+      length_seconds ||= 0
+      live_now ||= false
 
       video = ChannelVideo.new({
         id:                 video_id,
@@ -239,7 +263,9 @@ def fetch_channel(ucid, pull_all_videos : Bool)
         live_now:           live_now,
         premiere_timestamp: premiere_timestamp,
         views:              views,
+        video_type:         video_type,
       })
+    end
 
     LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updating or inserting video")
 
@@ -274,6 +300,7 @@ def fetch_channel(ucid, pull_all_videos : Bool)
           live_now:           video.badges.live_now?,
           premiere_timestamp: video.premiere_timestamp,
           views:              video.views,
+          video_type:         VideoType::Video
         })
 
         # We are notified of Red videos elsewhere (PubSub), which includes a correct published date,

src/invidious/config.cr

@@ -47,7 +47,8 @@ struct ConfigPreferences
   property thin_mode : Bool = false
   property unseen_only : Bool = false
   property video_loop : Bool = false
-  property hide_shorts_and_live : Bool = false
+  property hide_shorts : Bool = false
+  property hide_livestreams : Bool = false
   property extend_desc : Bool = false
   property volume : Int32 = 100
   property vr_mode : Bool = true

src/invidious/database/channels.cr

@@ -100,14 +100,14 @@ module Invidious::Database::ChannelVideos
   # This function returns the status of the query (i.e: success?)
   def insert(video : ChannelVideo, with_premiere_timestamp : Bool = false) : Bool
     if with_premiere_timestamp
-      last_items = "premiere_timestamp = $9, views = $10"
+      last_items = "premiere_timestamp = $9, views = $10, video_type = $11"
     else
-      last_items = "views = $10"
+      last_items = "views = $10, video_type = $11"
     end
 
     request = <<-SQL
       INSERT INTO channel_videos
-      VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
+      VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
       ON CONFLICT (id) DO UPDATE
       SET title = $2, published = $3, updated = $4, ucid = $5,
           author = $6, length_seconds = $7, live_now = $8, #{last_items}

src/invidious/routes/feeds.cr

@@ -438,6 +438,7 @@ module Invidious::Routes::Feeds
           live_now:           video.live_now,
           premiere_timestamp: video.premiere_timestamp,
           views:              video.views,
+          video_type:         VideoType::Video
         })
 
         was_insert = Invidious::Database::ChannelVideos.insert(video, with_premiere_timestamp: true)

src/invidious/routes/preferences.cr

@@ -143,9 +143,13 @@ module Invidious::Routes::PreferencesRoute
     notifications_only ||= "off"
     notifications_only = notifications_only == "on"
 
-    hide_shorts_and_live = env.params.body["hide_shorts_and_live"]?.try &.as(String)
+    hide_shorts = env.params.body["hide_shorts"]?.try &.as(String)
-    hide_shorts_and_live ||= "off"
+    hide_shorts ||= "off"
-    hide_shorts_and_live = hide_shorts_and_live == "on"
+    hide_shorts = hide_shorts == "on"
+
+    hide_livestreams = env.params.body["hide_livestreams"]?.try &.as(String)
+    hide_livestreams ||= "off"
+    hide_livestreams = hide_livestreams == "on"
 
     default_playlist = env.params.body["default_playlist"]?.try &.as(String)
 
@@ -186,7 +190,8 @@ module Invidious::Routes::PreferencesRoute
       show_nick:                   show_nick,
       save_player_pos:             save_player_pos,
       default_playlist:            default_playlist,
-      hide_shorts_and_live:        hide_shorts_and_live,
+      hide_shorts:                 hide_shorts,
+      hide_livestreams:            hide_livestreams,
     }.to_json)
 
     if user = env.get? "user"

src/invidious/user/preferences.cr

@@ -57,7 +57,8 @@ struct Preferences
   property volume : Int32 = CONFIG.default_user_preferences.volume
   property save_player_pos : Bool = CONFIG.default_user_preferences.save_player_pos
   property default_playlist : String? = nil
-  property hide_shorts_and_live : Bool = CONFIG.default_user_preferences.hide_shorts_and_live
+  property hide_shorts : Bool = CONFIG.default_user_preferences.hide_shorts
+  property hide_livestreams : Bool = CONFIG.default_user_preferences.hide_livestreams
 
   module BoolToString
     def self.to_json(value : String, json : JSON::Builder)

src/invidious/users.cr

@@ -29,6 +29,17 @@ def get_subscription_feed(user, max_results = 40, page = 1)
   notifications = Invidious::Database::Users.select_notifications(user)
   view_name = "subscriptions_#{sha256(user.email)}"
 
+  types_to_fetch = [VideoType::Video, VideoType::Short, VideoType::Livestream, VideoType::Scheduled]
+  if user.preferences.hide_shorts
+    types_to_fetch.delete(VideoType::Short)
+  end
+  if user.preferences.hide_livestreams
+    [VideoType::Livestream, VideoType::Scheduled].each { |v| types_to_fetch.delete(v) }
+  end
+
+  types_to_fetch = types_to_fetch.map { |type| "'#{type}'" }.join(", ")
+  LOGGER.trace("Types to fetch: #{types_to_fetch}")
+
   if user.preferences.notifications_only && !notifications.empty?
     # Only show notifications
     notifications = Invidious::Database::ChannelVideos.select(notifications)
@@ -58,18 +69,10 @@ def get_subscription_feed(user, max_results = 40, page = 1)
         else
           values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}"
         end
-        if user.preferences.hide_shorts_and_live
+        videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} WHERE NOT id = ANY (#{values}) AND video_type IN (#{types_to_fetch}) ORDER BY ucid, published DESC", as: ChannelVideo)
-          videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} WHERE NOT id = ANY (#{values}) AND length_seconds > 0 ORDER BY ucid, published DESC", as: ChannelVideo)
-        else
-          videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY ucid, published DESC", as: ChannelVideo)
-        end
       else
         # Show latest video from each channel
-        if user.preferences.hide_shorts_and_live
+          videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} WHERE video_type IN (#{types_to_fetch}) ORDER BY ucid, published DESC", as: ChannelVideo)
-          videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} WHERE length_seconds > 0 ORDER BY ucid, published DESC", as: ChannelVideo)
-        else
-          videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} ORDER BY ucid, published DESC", as: ChannelVideo)
-        end
       end
 
       videos.sort_by!(&.published).reverse!
@@ -82,18 +85,10 @@ def get_subscription_feed(user, max_results = 40, page = 1)
         else
           values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}"
         end
-        if user.preferences.hide_shorts_and_live
+          videos = PG_DB.query_all("SELECT * FROM #{view_name} WHERE NOT id = ANY (#{values}) AND video_type IN (#{types_to_fetch}) ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo)
-          videos = PG_DB.query_all("SELECT * FROM #{view_name} WHERE NOT id = ANY (#{values}) AND length_seconds > 0 ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo)
-        else
-          videos = PG_DB.query_all("SELECT * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo)
-        end
       else
         # Sort subscriptions as normal
-        if user.preferences.hide_shorts_and_live
+          videos = PG_DB.query_all("SELECT * FROM #{view_name} WHERE video_type IN (#{types_to_fetch}) ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo)
-          videos = PG_DB.query_all("SELECT * FROM #{view_name} WHERE length_seconds > 0 ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo)
-        else
-          videos = PG_DB.query_all("SELECT * FROM #{view_name} ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo)
-        end
       end
     end
 

src/invidious/videos.cr

@@ -1,5 +1,6 @@
 enum VideoType
   Video
+  Short
   Livestream
   Scheduled
 end

src/invidious/videos/parser.cr

@@ -218,6 +218,8 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
   post_live_dvr = video_details.dig?("isPostLiveDvr")
     .try &.as_bool || false
 
+  is_short = microformat["isShortsEligible"].try &.as_bool || false
+
   # Extra video infos
 
   allowed_regions = microformat["availableCountries"]?
@@ -394,8 +396,9 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
   end
 
   # Return data
-
+  if is_short
-  if live_now
+    video_type = VideoType::Short
+  elsif live_now
     video_type = VideoType::Livestream
   elsif !premiere_timestamp.nil?
     video_type = VideoType::Scheduled

src/invidious/views/user/preferences.ecr

@@ -230,8 +230,13 @@
                 </div>
 
                 <div class="pure-control-group">
-                    <label for="hide_shorts_and_live"><%= translate(locale, "preferences_hide_shorts_and_live_label") %></label>
+                    <label for="hide_shorts"><%= translate(locale, "preferences_hide_shorts_label") %></label>
-                    <input name="hide_shorts_and_live" id="hide_shorts_and_live" type="checkbox" <% if preferences.hide_shorts_and_live %>checked<% end %>>
+                    <input name="hide_shorts" id="hide_shorts" type="checkbox" <% if preferences.hide_shorts %>checked<% end %>>
+                </div>
+
+                <div class="pure-control-group">
+                    <label for="hide_livestreams"><%= translate(locale, "preferences_hide_livestreams_label") %></label>
+                    <input name="hide_livestreams" id="hide_livestreams" type="checkbox" <% if preferences.hide_livestreams %>checked<% end %>>
                 </div>
 
                 <div class="pure-control-group">