Merge 2c6f804d5a20c00dc006900c02f9cc3e5611b008 into d10f2a48021f1768f2d6ff3bd1b9f3de4e0b2d22

spec/invidious/channels/about_spec.cr

@@ -0,0 +1,295 @@
+require "../../../src/invidious/exceptions"
+require "../../spec_helper"
+
+Spectator.describe "extract_auto_generated_channel_header" do
+  it "parses the carouselHeaderRenderer shape" do
+    # ex: https://www.youtube.com/channel/UCEgdi0XIXXZ-qJOFPf4JSKw (Sports)
+    initdata = JSON.parse(<<-JSON).as_h
+      {
+        "header": {
+          "carouselHeaderRenderer": {
+            "contents": [
+              {
+                "carouselItemRenderer": {
+                  "carouselItems": []
+                }
+              },
+              {
+                "topicChannelDetailsRenderer": {
+                  "title": {"simpleText": "Sports"},
+                  "avatar": {
+                    "thumbnails": [
+                      {"url": "//yt3.example/topic-avatar", "width": 88, "height": 88}
+                    ]
+                  },
+                  "subtitle": {"simpleText": "74.3M subscribers"}
+                }
+              }
+            ]
+          }
+        }
+      }
+      JSON
+
+    header = extract_auto_generated_channel_header(initdata, "UCEgdi0XIXXZ-qJOFPf4JSKw")
+
+    expect(header[:author]).to eq("Sports")
+    expect(header[:author_url]).to eq("https://www.youtube.com/channel/UCEgdi0XIXXZ-qJOFPf4JSKw")
+    expect(header[:author_thumbnail]).to eq("//yt3.example/topic-avatar")
+    expect(header[:banner]).to be_nil
+    expect(header[:description_node]).to be_nil
+    expect(header[:tags]).to be_empty
+    expect(header[:is_family_friendly]).to be_true
+  end
+
+  it "finds the topic details regardless of their position in the carousel" do
+    initdata = JSON.parse(<<-JSON).as_h
+      {
+        "header": {
+          "carouselHeaderRenderer": {
+            "contents": [
+              {
+                "topicChannelDetailsRenderer": {
+                  "title": {"simpleText": "Sports"},
+                  "avatar": {"thumbnails": [{"url": "//yt3.example/first"}]}
+                }
+              },
+              {
+                "carouselItemRenderer": {"carouselItems": []}
+              }
+            ]
+          }
+        }
+      }
+      JSON
+
+    header = extract_auto_generated_channel_header(initdata, "UCEgdi0XIXXZ-qJOFPf4JSKw")
+
+    expect(header[:author]).to eq("Sports")
+    expect(header[:author_thumbnail]).to eq("//yt3.example/first")
+  end
+
+  it "raises when the carousel carries no topic details" do
+    initdata = JSON.parse(<<-JSON).as_h
+      {
+        "header": {
+          "carouselHeaderRenderer": {
+            "contents": [
+              {"carouselItemRenderer": {"carouselItems": []}}
+            ]
+          }
+        }
+      }
+      JSON
+
+    expect do
+      extract_auto_generated_channel_header(initdata, "UCEgdi0XIXXZ-qJOFPf4JSKw")
+    end.to raise_error(InfoException)
+  end
+
+  it "raises when the carousel topic details have no title" do
+    initdata = JSON.parse(<<-JSON).as_h
+      {
+        "header": {
+          "carouselHeaderRenderer": {
+            "contents": [
+              {
+                "topicChannelDetailsRenderer": {
+                  "avatar": {"thumbnails": [{"url": "//yt3.example/avatar"}]}
+                }
+              }
+            ]
+          }
+        }
+      }
+      JSON
+
+    expect do
+      extract_auto_generated_channel_header(initdata, "UCEgdi0XIXXZ-qJOFPf4JSKw")
+    end.to raise_error(InfoException)
+  end
+
+  it "still renders a carousel channel that carries no avatar" do
+    initdata = JSON.parse(<<-JSON).as_h
+      {
+        "header": {
+          "carouselHeaderRenderer": {
+            "contents": [
+              {
+                "topicChannelDetailsRenderer": {
+                  "title": {"simpleText": "Sports"}
+                }
+              }
+            ]
+          }
+        }
+      }
+      JSON
+
+    header = extract_auto_generated_channel_header(initdata, "UCEgdi0XIXXZ-qJOFPf4JSKw")
+
+    expect(header[:author]).to eq("Sports")
+    expect(header[:author_thumbnail]).to eq("")
+  end
+
+  it "parses the current pageHeaderRenderer shape" do
+    # ex: https://www.youtube.com/channel/UCOpNcN46UbXVtpKMrmU4Abg (Gaming)
+    initdata = JSON.parse(<<-JSON).as_h
+      {
+        "header": {
+          "pageHeaderRenderer": {
+            "pageTitle": "Gaming",
+            "content": {
+              "pageHeaderViewModel": {
+                "title": {
+                  "dynamicTextViewModel": {
+                    "text": {"content": "Gaming"}
+                  }
+                },
+                "animatedImage": {
+                  "contentPreviewImageViewModel": {
+                    "image": {
+                      "sources": [
+                        {"url": "//yt3.example/avatar", "width": 48, "height": 48}
+                      ]
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+      JSON
+
+    header = extract_auto_generated_channel_header(initdata, "UCOpNcN46UbXVtpKMrmU4Abg")
+
+    expect(header[:author]).to eq("Gaming")
+    expect(header[:author_url]).to eq("https://www.youtube.com/channel/UCOpNcN46UbXVtpKMrmU4Abg")
+    expect(header[:author_thumbnail]).to eq("//yt3.example/avatar")
+  end
+
+  it "preserves the legacy interactiveTabbedHeaderRenderer shape" do
+    initdata = JSON.parse(<<-JSON).as_h
+      {
+        "header": {
+          "interactiveTabbedHeaderRenderer": {
+            "title": {"simpleText": "Legacy gaming"},
+            "boxArt": {"thumbnails": [{"url": "//yt3.example/legacy-avatar"}]},
+            "banner": {"thumbnails": [{"url": "//yt3.example/legacy-banner"}]},
+            "description": {"simpleText": "A legacy description"},
+            "badges": [
+              {"metadataBadgeRenderer": {"label": "Gaming"}}
+            ]
+          }
+        },
+        "microformat": {
+          "microformatDataRenderer": {
+            "urlCanonical": "https://www.youtube.com/channel/UCLegacy"
+          }
+        }
+      }
+      JSON
+
+    header = extract_auto_generated_channel_header(initdata, "UCLegacy")
+
+    expect(header[:author]).to eq("Legacy gaming")
+    expect(header[:author_url]).to eq("https://www.youtube.com/channel/UCLegacy")
+    expect(header[:author_thumbnail]).to eq("//yt3.example/legacy-avatar")
+    expect(header[:banner]).to eq("//yt3.example/legacy-banner")
+    expect(header[:tags]).to eq(["Gaming"])
+  end
+
+  it "keeps an explicit familySafe: false" do
+    initdata = JSON.parse(<<-JSON).as_h
+      {
+        "header": {
+          "pageHeaderRenderer": {"pageTitle": "Gaming"}
+        },
+        "microformat": {
+          "microformatDataRenderer": {"familySafe": false}
+        }
+      }
+      JSON
+
+    header = extract_auto_generated_channel_header(initdata, "UCOpNcN46UbXVtpKMrmU4Abg")
+
+    expect(header[:is_family_friendly]).to be_false
+  end
+
+  it "raises when the header shape is unknown" do
+    initdata = JSON.parse(<<-JSON).as_h
+      {
+        "header": {
+          "someFutureHeaderRenderer": {}
+        }
+      }
+      JSON
+
+    expect do
+      extract_auto_generated_channel_header(initdata, "UCUnknown")
+    end.to raise_error(InfoException)
+  end
+end
+
+Spectator.describe "extract_topic_channel_details" do
+  it "returns nil when the payload carries no carousel header" do
+    initdata = JSON.parse(<<-JSON).as_h
+      {
+        "header": {
+          "pageHeaderRenderer": {"pageTitle": "Gaming"}
+        }
+      }
+      JSON
+
+    expect(extract_topic_channel_details(initdata)).to be_nil
+  end
+
+  it "returns nil when the carousel carries no topic details" do
+    initdata = JSON.parse(<<-JSON).as_h
+      {
+        "header": {
+          "carouselHeaderRenderer": {
+            "contents": [
+              {"carouselItemRenderer": {"carouselItems": []}}
+            ]
+          }
+        }
+      }
+      JSON
+
+    expect(extract_topic_channel_details(initdata)).to be_nil
+  end
+
+  it "exposes the subscriber count carried by the subtitle" do
+    # `subscriberCountText` is part of the renderer but comes back null, so the
+    # count is only available as free text in the subtitle.
+    # ex: https://www.youtube.com/channel/UCEgdi0XIXXZ-qJOFPf4JSKw (Sports)
+    initdata = JSON.parse(<<-JSON).as_h
+      {
+        "header": {
+          "carouselHeaderRenderer": {
+            "contents": [
+              {"carouselItemRenderer": {"carouselItems": []}},
+              {
+                "topicChannelDetailsRenderer": {
+                  "title": {"simpleText": "Sports"},
+                  "avatar": {"thumbnails": [{"url": "//yt3.example/topic-avatar"}]},
+                  "subscriberCountText": null,
+                  "subtitle": {"simpleText": "74.3M subscribers"}
+                }
+              }
+            ]
+          }
+        }
+      }
+      JSON
+
+    details = extract_topic_channel_details(initdata)
+
+    expect(details).not_to be_nil
+    sub_text = details.not_nil!.dig("subtitle", "simpleText").as_s
+    expect(sub_text).to eq("74.3M subscribers")
+    expect(short_text_to_number(sub_text.split(" ")[0])).to eq(74_300_000_i64)
+  end
+end

spec/invidious/yt_backend/extractors_spec.cr

@@ -0,0 +1,22 @@
+require "../../parsers_helper"
+
+Spectator.describe "YouTubeTabs" do
+  it "treats a selected tab without content as empty" do
+    initdata = JSON.parse(<<-JSON).as_h
+      {
+        "contents": {
+          "twoColumnBrowseResultsRenderer": {
+            "tabs": [
+              {"tabRenderer": {"selected": true}}
+            ]
+          }
+        }
+      }
+      JSON
+
+    items, continuation = extract_items(initdata)
+
+    expect(items).to be_empty
+    expect(continuation).to be_nil
+  end
+end

src/invidious/channels/about.cr

@@ -19,6 +19,92 @@ record AboutChannel,
   verified : Bool,
   is_age_gated : Bool
 
+# Topic channels keep the channel details inside one of the carousel entries.
+# The position of that entry varies between channels, so it is looked up by key
+# rather than by index.
+# ex: https://www.youtube.com/channel/UCEgdi0XIXXZ-qJOFPf4JSKw
+def extract_topic_channel_details(initdata : Hash(String, JSON::Any)) : JSON::Any?
+  contents = initdata.dig?("header", "carouselHeaderRenderer", "contents")
+  return nil if contents.nil?
+
+  contents.as_a
+    .find { |content| !content.dig?("topicChannelDetailsRenderer").nil? }
+    .try &.dig?("topicChannelDetailsRenderer")
+end
+
+# Auto-generated channels come with one of three header shapes. This is only
+# reached when the payload has no `metadata` object, i.e. when the regular
+# `channelMetadataRenderer` path is not available.
+def extract_auto_generated_channel_header(initdata : Hash(String, JSON::Any), ucid : String)
+  banner = nil
+  description_node = nil
+  tags = [] of String
+
+  if header = initdata.dig?("header", "interactiveTabbedHeaderRenderer")
+    author = header.dig("title", "simpleText").as_s
+    author_url = initdata.dig("microformat", "microformatDataRenderer", "urlCanonical").as_s
+    author_thumbnail = header.dig("boxArt", "thumbnails", 0, "url").as_s
+
+    banner = header.dig?("banner", "thumbnails").try &.[-1]?.try &.["url"].as_s?
+
+    description_base_node = header["description"]
+    # some channels have the description in a simpleText
+    # ex: https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/
+    description_node = description_base_node.dig?("simpleText") || description_base_node
+
+    tags = header.dig?("badges")
+      .try &.as_a.map(&.["metadataBadgeRenderer"]["label"].as_s) || [] of String
+  elsif header = initdata.dig?("header", "pageHeaderRenderer")
+    # ex: https://www.youtube.com/channel/UCOpNcN46UbXVtpKMrmU4Abg
+    view_model = header.dig?("content", "pageHeaderViewModel")
+
+    author = view_model.try &.dig?("title", "dynamicTextViewModel", "text", "content").try &.as_s
+    author ||= header.dig?("pageTitle").try &.as_s
+    author ||= ucid
+
+    author_url = "https://www.youtube.com/channel/#{ucid}"
+
+    author_thumbnail = view_model.try &.dig?("image", "decoratedAvatarViewModel", "avatar", "avatarViewModel", "image", "sources", 0, "url").try &.as_s
+    author_thumbnail ||= view_model.try &.dig?("animatedImage", "contentPreviewImageViewModel", "image", "sources", 0, "url").try &.as_s
+    author_thumbnail ||= ""
+
+    banner = view_model.try &.dig?("banner", "imageBannerViewModel", "image", "sources")
+      .try &.[-1]?.try &.["url"].as_s?
+  elsif initdata.dig?("header", "carouselHeaderRenderer")
+    # ex: https://www.youtube.com/channel/UCEgdi0XIXXZ-qJOFPf4JSKw
+    # This shape carries neither a banner nor a description.
+    details = extract_topic_channel_details(initdata)
+
+    unless details
+      raise InfoException.new("Could not extract the carousel header of channel #{ucid}")
+    end
+
+    author = details.dig?("title", "simpleText").try &.as_s
+    author ||= raise InfoException.new("Could not extract the carousel title of channel #{ucid}")
+    author_url = "https://www.youtube.com/channel/#{ucid}"
+
+    # A missing avatar is not worth failing the whole page over, and the
+    # `pageHeaderRenderer` branch above falls back to an empty string too.
+    author_thumbnail = details.dig?("avatar", "thumbnails", 0, "url").try &.as_s || ""
+  else
+    raise InfoException.new("Could not extract the header of channel #{ucid}")
+  end
+
+  # `microformat` is absent from these payloads, so a missing flag defaults to
+  # safe. An explicit `false` is still preserved.
+  family_safe = initdata.dig?("microformat", "microformatDataRenderer", "familySafe").try(&.as_bool)
+
+  {
+    author:             author,
+    author_url:         author_url,
+    author_thumbnail:   author_thumbnail,
+    banner:             banner,
+    description_node:   description_node,
+    tags:               tags,
+    is_family_friendly: family_safe.nil? ? true : family_safe,
+  }
+end
+
 def get_about_info(ucid, locale) : AboutChannel
   begin
     # Fetch channel information from channel home page
@@ -64,21 +150,14 @@ def get_about_info(ucid, locale) : AboutChannel
     auto_generated = false
   else
     if auto_generated
-      author = initdata["header"]["interactiveTabbedHeaderRenderer"]["title"]["simpleText"].as_s
+      channel_header = extract_auto_generated_channel_header(initdata, ucid)
-      author_url = initdata["microformat"]["microformatDataRenderer"]["urlCanonical"].as_s
+      author = channel_header[:author]
-      author_thumbnail = initdata["header"]["interactiveTabbedHeaderRenderer"]["boxArt"]["thumbnails"][0]["url"].as_s
+      author_url = channel_header[:author_url]
-
+      author_thumbnail = channel_header[:author_thumbnail]
-      # Raises a KeyError on failure.
+      banner = channel_header[:banner]
-      banners = initdata["header"]["interactiveTabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]?
+      description_node = channel_header[:description_node]
-      banner = banners.try &.[-1]?.try &.["url"].as_s?
+      tags = channel_header[:tags]
-
+      is_family_friendly = channel_header[:is_family_friendly]
-      description_base_node = initdata["header"]["interactiveTabbedHeaderRenderer"]["description"]
-      # some channels have the description in a simpleText
-      # ex: https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/
-      description_node = description_base_node.dig?("simpleText") || description_base_node
-
-      tags = initdata.dig?("header", "interactiveTabbedHeaderRenderer", "badges")
-        .try &.as_a.map(&.["metadataBadgeRenderer"]["label"].as_s) || [] of String
     else
       author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s
       author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s
@@ -103,9 +182,9 @@ def get_about_info(ucid, locale) : AboutChannel
 
       description_node = initdata["metadata"]["channelMetadataRenderer"]?.try &.["description"]?
       tags = initdata.dig?("microformat", "microformatDataRenderer", "tags").try &.as_a.map(&.as_s) || [] of String
+      is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool
     end
 
-    is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool
     if tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]?
       # Get the name of the tabs available on this channel
       tab_names = tabs_json.as_a.compact_map do |entry|
@@ -182,6 +261,16 @@ def get_about_info(ucid, locale) : AboutChannel
 
       break if sub_count != 0 && !pronouns.nil?
     end
+  elsif (topic_details = extract_topic_channel_details(initdata))
+    # Topic channels carry the subscriber count as free text in the subtitle,
+    # ex: "74.3M subscribers". `subscriberCountText` is part of the same
+    # renderer but comes back null, so it is only used as a first choice.
+    sub_text = topic_details.dig?("subscriberCountText", "simpleText").try &.as_s
+    sub_text ||= topic_details.dig?("subtitle", "simpleText").try &.as_s
+
+    if sub_text && sub_text.includes?("subscriber")
+      sub_count = short_text_to_number(sub_text.split(" ")[0]).to_i32
+    end
   end
 
   AboutChannel.new(

src/invidious/yt_backend/extractors.cr

@@ -964,7 +964,8 @@ private module Extractors
 
     private def self.extract(target)
       raw_items = [] of JSON::Any
-      content = extract_selected_tab(target["tabs"])["content"]
+      content = extract_selected_tab(target["tabs"])["content"]?
+      return raw_items if content.nil?
 
       if section_list_contents = content.dig?("sectionListRenderer", "contents")
         raw_items = unpack_section_list(section_list_contents)