diff --git a/spec/invidious/channels/about_spec.cr b/spec/invidious/channels/about_spec.cr new file mode 100644 index 000000000..29bcc70f3 --- /dev/null +++ b/spec/invidious/channels/about_spec.cr @@ -0,0 +1,378 @@ +require "../../../src/invidious/exceptions" +require "../../spec_helper" + +Spectator.describe "extract_auto_generated_channel_header" do + it "parses the carouselHeaderRenderer shape" do + # ex: https://www.youtube.com/channel/UCEgdi0XIXXZ-qJOFPf4JSKw (Sports) + initdata = JSON.parse(<<-JSON).as_h + { + "header": { + "carouselHeaderRenderer": { + "contents": [ + { + "carouselItemRenderer": { + "carouselItems": [] + } + }, + { + "topicChannelDetailsRenderer": { + "title": {"simpleText": "Sports"}, + "avatar": { + "thumbnails": [ + {"url": "//yt3.example/topic-avatar", "width": 88, "height": 88} + ] + }, + "subtitle": {"simpleText": "74.3M subscribers"} + } + } + ] + } + } + } + JSON + + header = extract_auto_generated_channel_header(initdata, "UCEgdi0XIXXZ-qJOFPf4JSKw") + + expect(header[:author]).to eq("Sports") + expect(header[:author_url]).to eq("https://www.youtube.com/channel/UCEgdi0XIXXZ-qJOFPf4JSKw") + expect(header[:author_thumbnail]).to eq("//yt3.example/topic-avatar") + expect(header[:banner]).to be_nil + expect(header[:description_node]).to be_nil + expect(header[:tags]).to be_empty + expect(header[:is_family_friendly]).to be_true + end + + it "finds the topic details regardless of their position in the carousel" do + initdata = JSON.parse(<<-JSON).as_h + { + "header": { + "carouselHeaderRenderer": { + "contents": [ + { + "topicChannelDetailsRenderer": { + "title": {"simpleText": "Sports"}, + "avatar": {"thumbnails": [{"url": "//yt3.example/first"}]} + } + }, + { + "carouselItemRenderer": {"carouselItems": []} + } + ] + } + } + } + JSON + + header = extract_auto_generated_channel_header(initdata, "UCEgdi0XIXXZ-qJOFPf4JSKw") + + expect(header[:author]).to eq("Sports") + expect(header[:author_thumbnail]).to eq("//yt3.example/first") + end + + it "raises when the carousel carries no topic details" do + initdata = JSON.parse(<<-JSON).as_h + { + "header": { + "carouselHeaderRenderer": { + "contents": [ + {"carouselItemRenderer": {"carouselItems": []}} + ] + } + } + } + JSON + + expect do + extract_auto_generated_channel_header(initdata, "UCEgdi0XIXXZ-qJOFPf4JSKw") + end.to raise_error(InfoException) + end + + it "raises when the carousel topic details have no title" do + initdata = JSON.parse(<<-JSON).as_h + { + "header": { + "carouselHeaderRenderer": { + "contents": [ + { + "topicChannelDetailsRenderer": { + "avatar": {"thumbnails": [{"url": "//yt3.example/avatar"}]} + } + } + ] + } + } + } + JSON + + expect do + extract_auto_generated_channel_header(initdata, "UCEgdi0XIXXZ-qJOFPf4JSKw") + end.to raise_error(InfoException) + end + + it "still renders a carousel channel that carries no avatar" do + initdata = JSON.parse(<<-JSON).as_h + { + "header": { + "carouselHeaderRenderer": { + "contents": [ + { + "topicChannelDetailsRenderer": { + "title": {"simpleText": "Sports"} + } + } + ] + } + } + } + JSON + + header = extract_auto_generated_channel_header(initdata, "UCEgdi0XIXXZ-qJOFPf4JSKw") + + expect(header[:author]).to eq("Sports") + expect(header[:author_thumbnail]).to eq("") + end + + it "parses the current pageHeaderRenderer shape" do + # ex: https://www.youtube.com/channel/UCOpNcN46UbXVtpKMrmU4Abg (Gaming) + initdata = JSON.parse(<<-JSON).as_h + { + "header": { + "pageHeaderRenderer": { + "pageTitle": "Gaming", + "content": { + "pageHeaderViewModel": { + "title": { + "dynamicTextViewModel": { + "text": {"content": "Gaming"} + } + }, + "animatedImage": { + "contentPreviewImageViewModel": { + "image": { + "sources": [ + {"url": "//yt3.example/avatar", "width": 48, "height": 48} + ] + } + } + } + } + } + } + } + } + JSON + + header = extract_auto_generated_channel_header(initdata, "UCOpNcN46UbXVtpKMrmU4Abg") + + expect(header[:author]).to eq("Gaming") + expect(header[:author_url]).to eq("https://www.youtube.com/channel/UCOpNcN46UbXVtpKMrmU4Abg") + expect(header[:author_thumbnail]).to eq("//yt3.example/avatar") + end + + it "preserves the legacy interactiveTabbedHeaderRenderer shape" do + initdata = JSON.parse(<<-JSON).as_h + { + "header": { + "interactiveTabbedHeaderRenderer": { + "title": {"simpleText": "Legacy gaming"}, + "boxArt": {"thumbnails": [{"url": "//yt3.example/legacy-avatar"}]}, + "banner": {"thumbnails": [{"url": "//yt3.example/legacy-banner"}]}, + "description": {"simpleText": "A legacy description"}, + "badges": [ + {"metadataBadgeRenderer": {"label": "Gaming"}} + ] + } + }, + "microformat": { + "microformatDataRenderer": { + "urlCanonical": "https://www.youtube.com/channel/UCLegacy" + } + } + } + JSON + + header = extract_auto_generated_channel_header(initdata, "UCLegacy") + + expect(header[:author]).to eq("Legacy gaming") + expect(header[:author_url]).to eq("https://www.youtube.com/channel/UCLegacy") + expect(header[:author_thumbnail]).to eq("//yt3.example/legacy-avatar") + expect(header[:banner]).to eq("//yt3.example/legacy-banner") + expect(header[:tags]).to eq(["Gaming"]) + end + + it "raises an InfoException when the legacy interactive title is missing" do + initdata = JSON.parse(<<-JSON).as_h + { + "header": { + "interactiveTabbedHeaderRenderer": { + "title": {} + } + } + } + JSON + + expect do + extract_auto_generated_channel_header(initdata, "UCMissingTitle") + end.to raise_error(InfoException, /interactive title/) + end + + it "falls back to the channel URL when the canonical URL is missing" do + initdata = JSON.parse(<<-JSON).as_h + { + "header": { + "interactiveTabbedHeaderRenderer": { + "title": {"simpleText": "Legacy gaming"} + } + }, + "microformat": { + "microformatDataRenderer": { + "familySafe": true + } + } + } + JSON + + header = extract_auto_generated_channel_header(initdata, "UCMissingCanonical") + + expect(header[:author_url]).to eq("https://www.youtube.com/channel/UCMissingCanonical") + end + + it "keeps an explicit familySafe: false" do + initdata = JSON.parse(<<-JSON).as_h + { + "header": { + "pageHeaderRenderer": {"pageTitle": "Gaming"} + }, + "microformat": { + "microformatDataRenderer": {"familySafe": false} + } + } + JSON + + header = extract_auto_generated_channel_header(initdata, "UCOpNcN46UbXVtpKMrmU4Abg") + + expect(header[:is_family_friendly]).to be_false + end + + it "defaults an explicit familySafe: null to true" do + initdata = JSON.parse(<<-JSON).as_h + { + "header": { + "pageHeaderRenderer": {"pageTitle": "Gaming"} + }, + "microformat": { + "microformatDataRenderer": {"familySafe": null} + } + } + JSON + + header = extract_auto_generated_channel_header(initdata, "UCNullFamilySafe") + + expect(header[:is_family_friendly]).to be_true + end + + it "raises when the header shape is unknown" do + initdata = JSON.parse(<<-JSON).as_h + { + "header": { + "someFutureHeaderRenderer": {} + } + } + JSON + + expect do + extract_auto_generated_channel_header(initdata, "UCUnknown") + end.to raise_error(InfoException) + end +end + +Spectator.describe "extract_topic_channel_details" do + it "returns nil when the payload carries no carousel header" do + initdata = JSON.parse(<<-JSON).as_h + { + "header": { + "pageHeaderRenderer": {"pageTitle": "Gaming"} + } + } + JSON + + expect(extract_topic_channel_details(initdata)).to be_nil + end + + it "returns nil when the carousel carries no topic details" do + initdata = JSON.parse(<<-JSON).as_h + { + "header": { + "carouselHeaderRenderer": { + "contents": [ + {"carouselItemRenderer": {"carouselItems": []}} + ] + } + } + } + JSON + + expect(extract_topic_channel_details(initdata)).to be_nil + end + + it "returns nil when carousel contents are explicitly null" do + initdata = JSON.parse(<<-JSON).as_h + { + "header": { + "carouselHeaderRenderer": { + "contents": null + } + } + } + JSON + + expect(extract_topic_channel_details(initdata)).to be_nil + end + + it "skips an explicitly null topic renderer before a valid one" do + initdata = JSON.parse(<<-JSON).as_h + { + "header": { + "carouselHeaderRenderer": { + "contents": [ + {"topicChannelDetailsRenderer": null}, + {"topicChannelDetailsRenderer": {"title": {"simpleText": "Sports"}}} + ] + } + } + } + JSON + + details = extract_topic_channel_details(initdata) + + expect(details).not_to be_nil + expect(details.not_nil!.dig("title", "simpleText").as_s).to eq("Sports") + end + + it "exposes the subscriber count carried by the subtitle" do + # `subscriberCountText` is part of the renderer but comes back null, so the + # count is only available as free text in the subtitle. + # ex: https://www.youtube.com/channel/UCEgdi0XIXXZ-qJOFPf4JSKw (Sports) + initdata = JSON.parse(<<-JSON).as_h + { + "header": { + "carouselHeaderRenderer": { + "contents": [ + {"carouselItemRenderer": {"carouselItems": []}}, + { + "topicChannelDetailsRenderer": { + "title": {"simpleText": "Sports"}, + "avatar": {"thumbnails": [{"url": "//yt3.example/topic-avatar"}]}, + "subscriberCountText": null, + "subtitle": {"simpleText": "74.3M subscribers"} + } + } + ] + } + } + } + JSON + + expect(extract_topic_channel_subscriber_count(initdata)).to eq(74_300_000) + end +end diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr index 247f628a0..016908fc4 100644 --- a/src/invidious/channels/about.cr +++ b/src/invidious/channels/about.cr @@ -19,6 +19,113 @@ record AboutChannel, verified : Bool, is_age_gated : Bool +# Topic channels keep the channel details inside one of the carousel entries. +# The position of that entry varies between channels, so it is looked up by key +# rather than by index. +# ex: https://www.youtube.com/channel/UCEgdi0XIXXZ-qJOFPf4JSKw +def extract_topic_channel_details(initdata : Hash(String, JSON::Any)) : JSON::Any? + contents = initdata.dig?("header", "carouselHeaderRenderer", "contents").try &.as_a? + return nil unless contents + + contents.each do |content| + details = content.dig?("topicChannelDetailsRenderer") + return details if details.try &.as_h? + end + + nil +end + +def extract_topic_channel_subscriber_count(initdata : Hash(String, JSON::Any)) : Int32 + details = extract_topic_channel_details(initdata) + return 0 unless details + + sub_text = details.dig?("subscriberCountText", "simpleText").try &.as_s? + unless sub_text.try &.includes?("subscriber") + sub_text = details.dig?("subtitle", "simpleText").try &.as_s? + end + + return 0 unless sub_text.try &.includes?("subscriber") + + short_text_to_number(sub_text.split(" ")[0]).to_i32 +end + +# Auto-generated channels come with one of three header shapes. This is only +# reached when the payload has no `metadata` object, i.e. when the regular +# `channelMetadataRenderer` path is not available. +def extract_auto_generated_channel_header(initdata : Hash(String, JSON::Any), ucid : String) + banner = nil + description_node = nil + tags = [] of String + + if header = initdata.dig?("header", "interactiveTabbedHeaderRenderer") + author = header.dig?("title", "simpleText").try &.as_s + author ||= raise InfoException.new("Could not extract the interactive title of channel #{ucid}") + author_url = initdata.dig?("microformat", "microformatDataRenderer", "urlCanonical").try &.as_s? + author_url ||= "https://www.youtube.com/channel/#{ucid}" + author_thumbnail = header.dig?("boxArt", "thumbnails", 0, "url").try &.as_s || "" + + banner = header.dig?("banner", "thumbnails").try &.[-1]?.try &.["url"].as_s? + + description_base_node = header["description"]? + # some channels have the description in a simpleText + # ex: https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/ + description_node = description_base_node.try do |node| + node.dig?("simpleText") || node + end + + tags = header.dig?("badges") + .try &.as_a.map(&.["metadataBadgeRenderer"]["label"].as_s) || [] of String + elsif header = initdata.dig?("header", "pageHeaderRenderer") + # ex: https://www.youtube.com/channel/UCOpNcN46UbXVtpKMrmU4Abg + view_model = header.dig?("content", "pageHeaderViewModel") + + author = view_model.try &.dig?("title", "dynamicTextViewModel", "text", "content").try &.as_s + author ||= header.dig?("pageTitle").try &.as_s + author ||= ucid + + author_url = "https://www.youtube.com/channel/#{ucid}" + + author_thumbnail = view_model.try &.dig?("image", "decoratedAvatarViewModel", "avatar", "avatarViewModel", "image", "sources", 0, "url").try &.as_s + author_thumbnail ||= view_model.try &.dig?("animatedImage", "contentPreviewImageViewModel", "image", "sources", 0, "url").try &.as_s + author_thumbnail ||= "" + + banner = view_model.try &.dig?("banner", "imageBannerViewModel", "image", "sources") + .try &.[-1]?.try &.["url"].as_s? + elsif initdata.dig?("header", "carouselHeaderRenderer") + # ex: https://www.youtube.com/channel/UCEgdi0XIXXZ-qJOFPf4JSKw + # This shape carries neither a banner nor a description. + details = extract_topic_channel_details(initdata) + + unless details + raise InfoException.new("Could not extract the carousel header of channel #{ucid}") + end + + author = details.dig?("title", "simpleText").try &.as_s + author ||= raise InfoException.new("Could not extract the carousel title of channel #{ucid}") + author_url = "https://www.youtube.com/channel/#{ucid}" + + # A missing avatar is not worth failing the whole page over, and the + # `pageHeaderRenderer` branch above falls back to an empty string too. + author_thumbnail = details.dig?("avatar", "thumbnails", 0, "url").try &.as_s || "" + else + raise InfoException.new("Could not extract the header of channel #{ucid}") + end + + # `microformat` is absent from these payloads, so a missing flag defaults to + # safe. An explicit `false` is still preserved. + family_safe = initdata.dig?("microformat", "microformatDataRenderer", "familySafe").try &.as_bool? + + { + author: author, + author_url: author_url, + author_thumbnail: author_thumbnail, + banner: banner, + description_node: description_node, + tags: tags, + is_family_friendly: family_safe.nil? ? true : family_safe, + } +end + def get_about_info(ucid) : AboutChannel begin # Fetch channel information from channel home page @@ -64,21 +171,14 @@ def get_about_info(ucid) : AboutChannel auto_generated = false else if auto_generated - author = initdata["header"]["interactiveTabbedHeaderRenderer"]["title"]["simpleText"].as_s - author_url = initdata["microformat"]["microformatDataRenderer"]["urlCanonical"].as_s - author_thumbnail = initdata["header"]["interactiveTabbedHeaderRenderer"]["boxArt"]["thumbnails"][0]["url"].as_s - - # Raises a KeyError on failure. - banners = initdata["header"]["interactiveTabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? - banner = banners.try &.[-1]?.try &.["url"].as_s? - - description_base_node = initdata["header"]["interactiveTabbedHeaderRenderer"]["description"] - # some channels have the description in a simpleText - # ex: https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/ - description_node = description_base_node.dig?("simpleText") || description_base_node - - tags = initdata.dig?("header", "interactiveTabbedHeaderRenderer", "badges") - .try &.as_a.map(&.["metadataBadgeRenderer"]["label"].as_s) || [] of String + channel_header = extract_auto_generated_channel_header(initdata, ucid) + author = channel_header[:author] + author_url = channel_header[:author_url] + author_thumbnail = channel_header[:author_thumbnail] + banner = channel_header[:banner] + description_node = channel_header[:description_node] + tags = channel_header[:tags] + is_family_friendly = channel_header[:is_family_friendly] else author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s @@ -103,9 +203,9 @@ def get_about_info(ucid) : AboutChannel description_node = initdata["metadata"]["channelMetadataRenderer"]?.try &.["description"]? tags = initdata.dig?("microformat", "microformatDataRenderer", "tags").try &.as_a.map(&.as_s) || [] of String + is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool end - is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool if tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]? # Get the name of the tabs available on this channel tab_names = tabs_json.as_a.compact_map do |entry| @@ -182,6 +282,8 @@ def get_about_info(ucid) : AboutChannel break if sub_count != 0 && !pronouns.nil? end + elsif initdata.dig?("header", "carouselHeaderRenderer") + sub_count = extract_topic_channel_subscriber_count(initdata) end AboutChannel.new(