Channel: parse subscriber count and channel banner (#4785)

This PR adds support for parsing the newer channel header format
(banner + subscription parsing)

Before this change:
* 0 subscribers
* No banner image

After this change:
* Example with Mr Breast channel: 299M
* Image banner is visible

Closes issue 4783

assets/js/player.js

@@ -351,7 +351,12 @@ if (video_data.params.save_player_pos) {
     const rememberedTime = get_video_time();
     let lastUpdated = 0;
 
-    if(!hasTimeParam) set_seconds_after_start(rememberedTime);
+    if(!hasTimeParam) {
+      if (rememberedTime >= video_data.length_seconds - 20)
+        set_seconds_after_start(0);
+      else
+        set_seconds_after_start(rememberedTime);
+    }
 
     player.on('timeupdate', function () {
         const raw = player.currentTime();

spec/invidious/videos/regular_videos_extract_spec.cr

@@ -67,7 +67,7 @@ Spectator.describe "parse_video_info" do
     # Video metadata
 
     expect(info["genre"].as_s).to eq("Entertainment")
-    expect(info["genreUcid"].as_s).to be_empty
+    expect(info["genreUcid"].as_s?).to be_nil
     expect(info["license"].as_s).to be_empty
 
     # Author infos
@@ -151,7 +151,7 @@ Spectator.describe "parse_video_info" do
     # Video metadata
 
     expect(info["genre"].as_s).to eq("Music")
-    expect(info["genreUcid"].as_s).to be_empty
+    expect(info["genreUcid"].as_s?).to be_nil
     expect(info["license"].as_s).to be_empty
 
     # Author infos

spec/invidious/videos/scheduled_live_extract_spec.cr

@@ -94,7 +94,7 @@ Spectator.describe "parse_video_info" do
     # Video metadata
 
     expect(info["genre"].as_s).to eq("Entertainment")
-    expect(info["genreUcid"].as_s).to be_empty
+    expect(info["genreUcid"].as_s?).to be_nil
     expect(info["license"].as_s).to be_empty
 
     # Author infos

src/invidious/channels/about.cr

@@ -72,6 +72,7 @@ def get_about_info(ucid, locale) : AboutChannel
 
     # Raises a KeyError on failure.
     banners = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]?
+    banners ||= initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "banner", "imageBannerViewModel", "image", "sources")
     banner = banners.try &.[-1]?.try &.["url"].as_s?
 
     # if banner.includes? "channels/c4/default_banner"
@@ -147,9 +148,17 @@ def get_about_info(ucid, locale) : AboutChannel
     end
   end
 
-  sub_count = initdata
+  sub_count = 0
-    .dig?("header", "c4TabbedHeaderRenderer", "subscriberCountText", "simpleText").try &.as_s?
+
-    .try { |text| short_text_to_number(text.split(" ")[0]).to_i32 } || 0
+  if (metadata_rows = initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "metadata", "contentMetadataViewModel", "metadataRows").try &.as_a)
+    metadata_rows.each do |row|
+      metadata_part = row.dig?("metadataParts").try &.as_a.find { |i| i.dig?("text", "content").try &.as_s.includes?("subscribers") }
+      if !metadata_part.nil?
+        sub_count = short_text_to_number(metadata_part.dig("text", "content").as_s.split(" ")[0]).to_i32
+      end
+      break if sub_count != 0
+    end
+  end
 
   AboutChannel.new(
     ucid: ucid,

src/invidious/http_server/utils.cr

@@ -11,11 +11,12 @@ module Invidious::HttpServer
       params = url.query_params
       params["host"] = url.host.not_nil! # Should never be nil, in theory
       params["region"] = region if !region.nil?
+      url.query_params = params
 
       if absolute
-        return "#{HOST_URL}#{url.request_target}?#{params}"
+        return "#{HOST_URL}#{url.request_target}"
       else
-        return "#{url.request_target}?#{params}"
+        return url.request_target
       end
     end
 

src/invidious/jsonify/api_v1/video_json.cr

@@ -114,25 +114,31 @@ module Invidious::JSONify::APIv1
 
               json.field "projectionType", fmt["projectionType"]
 
-              if fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"])
+              height = fmt["height"]?.try &.as_i
-                fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30
+              width = fmt["width"]?.try &.as_i
+
+              fps = fmt["fps"]?.try &.as_i
+
+              if fps
                 json.field "fps", fps
+              end
+
+              if height && width
+                json.field "size", "#{width}x#{height}"
+                json.field "resolution", "#{height}p"
+
+                quality_label = "#{width > height ? height : width}p"
+
+                if fps && fps > 30
+                  quality_label += fps.to_s
+                end
+
+                json.field "qualityLabel", quality_label
+              end
+
+              if fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"])
                 json.field "container", fmt_info["ext"]
                 json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"]
-
-                if fmt_info["height"]?
-                  json.field "resolution", "#{fmt_info["height"]}p"
-
-                  quality_label = "#{fmt_info["height"]}p"
-                  if fps > 30
-                    quality_label += "60"
-                  end
-                  json.field "qualityLabel", quality_label
-
-                  if fmt_info["width"]?
-                    json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}"
-                  end
-                end
               end
 
               # Livestream chunk infos
@@ -163,26 +169,31 @@ module Invidious::JSONify::APIv1
 
               json.field "bitrate", fmt["bitrate"].as_i.to_s if fmt["bitrate"]?
 
-              fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"])
+              height = fmt["height"]?.try &.as_i
-              if fmt_info
+              width = fmt["width"]?.try &.as_i
-                fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30
+
+              fps = fmt["fps"]?.try &.as_i
+
+              if fps
                 json.field "fps", fps
+              end
+
+              if height && width
+                json.field "size", "#{width}x#{height}"
+                json.field "resolution", "#{height}p"
+
+                quality_label = "#{width > height ? height : width}p"
+
+                if fps && fps > 30
+                  quality_label += fps.to_s
+                end
+
+                json.field "qualityLabel", quality_label
+              end
+
+              if fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"])
                 json.field "container", fmt_info["ext"]
                 json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"]
-
-                if fmt_info["height"]?
-                  json.field "resolution", "#{fmt_info["height"]}p"
-
-                  quality_label = "#{fmt_info["height"]}p"
-                  if fps > 30
-                    quality_label += "60"
-                  end
-                  json.field "qualityLabel", quality_label
-
-                  if fmt_info["width"]?
-                    json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}"
-                  end
-                end
               end
             end
           end

src/invidious/playlists.cr

@@ -366,6 +366,8 @@ def fetch_playlist(plid : String)
 
     if text.includes? "video"
       video_count = text.gsub(/\D/, "").to_i? || 0
+    elsif text.includes? "episode"
+      video_count = text.gsub(/\D/, "").to_i? || 0
     elsif text.includes? "view"
       views = text.gsub(/\D/, "").to_i64? || 0_i64
     else

src/invidious/routes/api/v1/misc.cr

@@ -74,7 +74,9 @@ module Invidious::Routes::API::V1::Misc
       response = playlist.to_json(offset, video_id: video_id)
       json_response = JSON.parse(response)
 
-      if json_response["videos"].as_a[0]["index"] != offset
+      if json_response["videos"].as_a.empty?
+        json_response = JSON.parse(response)
+      elsif json_response["videos"].as_a[0]["index"] != offset
         offset = json_response["videos"].as_a[0]["index"].as_i
         lookback = offset < 50 ? offset : 50
         response = playlist.to_json(offset - lookback)

src/invidious/routes/api/v1/videos.cr

@@ -141,7 +141,11 @@ module Invidious::Routes::API::V1::Videos
           end
         end
       else
-        webvtt = YT_POOL.client &.get("#{url}&fmt=vtt").body
+        uri = URI.parse(url)
+        query_params = uri.query_params
+        query_params["fmt"] = "vtt"
+        uri.query_params = query_params
+        webvtt = YT_POOL.client &.get(uri.request_target).body
 
         if webvtt.starts_with?("<?xml")
           webvtt = caption.timedtext_to_vtt(webvtt)

src/invidious/routes/preferences.cr

@@ -214,7 +214,7 @@ module Invidious::Routes::PreferencesRoute
         statistics_enabled ||= "off"
         CONFIG.statistics_enabled = statistics_enabled == "on"
 
-        CONFIG.modified_source_code_url = env.params.body["modified_source_code_url"]?.try &.as(String)
+        CONFIG.modified_source_code_url = env.params.body["modified_source_code_url"]?.presence
 
         File.write("config/config.yml", CONFIG.to_yaml)
       end

src/invidious/videos.cr

@@ -250,7 +250,7 @@ struct Video
   end
 
   def genre_url : String?
-    info["genreUcid"]? ? "/channel/#{info["genreUcid"]}" : nil
+    info["genreUcid"].try &.as_s? ? "/channel/#{info["genreUcid"]}" : nil
   end
 
   def is_vr : Bool?

src/invidious/videos/parser.cr

@@ -424,7 +424,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
     "shortDescription" => JSON::Any.new(short_description.try &.as_s || nil),
     # Video metadata
     "genre"     => JSON::Any.new(genre.try &.as_s || ""),
-    "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s || ""),
+    "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s?),
     "license"   => JSON::Any.new(license.try &.as_s || ""),
     # Music section
     "music" => JSON.parse(music_list.to_json),

src/invidious/views/user/preferences.ecr

@@ -310,7 +310,7 @@
 
                 <div class="pure-control-group">
                     <label for="modified_source_code_url"><%= translate(locale, "adminprefs_modified_source_code_url_label") %></label>
-                    <input name="modified_source_code_url" id="modified_source_code_url" type="input" <% if CONFIG.modified_source_code_url %>checked<% end %>>
+                    <input name="modified_source_code_url" id="modified_source_code_url" type="url" value="<%= CONFIG.modified_source_code_url %>">
                 </div>
             <% end %>