diff --git a/spec/invidious/channels/about_spec.cr b/spec/invidious/channels/about_spec.cr new file mode 100644 index 00000000..66f8e029 --- /dev/null +++ b/spec/invidious/channels/about_spec.cr @@ -0,0 +1,251 @@ +require "../../../src/invidious/exceptions" +require "../../spec_helper" + +Spectator.describe "extract_auto_generated_channel_header" do + it "parses the carouselHeaderRenderer shape" do + # ex: https://www.youtube.com/channel/UCEgdi0XIXXZ-qJOFPf4JSKw (Sports) + initdata = JSON.parse(<<-JSON).as_h + { + "header": { + "carouselHeaderRenderer": { + "contents": [ + { + "carouselItemRenderer": { + "carouselItems": [] + } + }, + { + "topicChannelDetailsRenderer": { + "title": {"simpleText": "Sports"}, + "avatar": { + "thumbnails": [ + {"url": "//yt3.example/topic-avatar", "width": 88, "height": 88} + ] + }, + "subtitle": {"simpleText": "74.3M subscribers"} + } + } + ] + } + } + } + JSON + + header = extract_auto_generated_channel_header(initdata, "UCEgdi0XIXXZ-qJOFPf4JSKw") + + expect(header[:author]).to eq("Sports") + expect(header[:author_url]).to eq("https://www.youtube.com/channel/UCEgdi0XIXXZ-qJOFPf4JSKw") + expect(header[:author_thumbnail]).to eq("//yt3.example/topic-avatar") + expect(header[:banner]).to be_nil + expect(header[:description_node]).to be_nil + expect(header[:tags]).to be_empty + expect(header[:is_family_friendly]).to be_true + end + + it "finds the topic details regardless of their position in the carousel" do + initdata = JSON.parse(<<-JSON).as_h + { + "header": { + "carouselHeaderRenderer": { + "contents": [ + { + "topicChannelDetailsRenderer": { + "title": {"simpleText": "Sports"}, + "avatar": {"thumbnails": [{"url": "//yt3.example/first"}]} + } + }, + { + "carouselItemRenderer": {"carouselItems": []} + } + ] + } + } + } + JSON + + header = extract_auto_generated_channel_header(initdata, "UCEgdi0XIXXZ-qJOFPf4JSKw") + + expect(header[:author]).to eq("Sports") + expect(header[:author_thumbnail]).to eq("//yt3.example/first") + end + + it "falls back to the ucid when the carousel carries no topic details" do + initdata = JSON.parse(<<-JSON).as_h + { + "header": { + "carouselHeaderRenderer": { + "contents": [ + {"carouselItemRenderer": {"carouselItems": []}} + ] + } + } + } + JSON + + header = extract_auto_generated_channel_header(initdata, "UCEgdi0XIXXZ-qJOFPf4JSKw") + + expect(header[:author]).to eq("UCEgdi0XIXXZ-qJOFPf4JSKw") + expect(header[:author_thumbnail]).to eq("") + end + + it "parses the current pageHeaderRenderer shape" do + # ex: https://www.youtube.com/channel/UCOpNcN46UbXVtpKMrmU4Abg (Gaming) + initdata = JSON.parse(<<-JSON).as_h + { + "header": { + "pageHeaderRenderer": { + "pageTitle": "Gaming", + "content": { + "pageHeaderViewModel": { + "title": { + "dynamicTextViewModel": { + "text": {"content": "Gaming"} + } + }, + "animatedImage": { + "contentPreviewImageViewModel": { + "image": { + "sources": [ + {"url": "//yt3.example/avatar", "width": 48, "height": 48} + ] + } + } + } + } + } + } + } + } + JSON + + header = extract_auto_generated_channel_header(initdata, "UCOpNcN46UbXVtpKMrmU4Abg") + + expect(header[:author]).to eq("Gaming") + expect(header[:author_url]).to eq("https://www.youtube.com/channel/UCOpNcN46UbXVtpKMrmU4Abg") + expect(header[:author_thumbnail]).to eq("//yt3.example/avatar") + end + + it "preserves the legacy interactiveTabbedHeaderRenderer shape" do + initdata = JSON.parse(<<-JSON).as_h + { + "header": { + "interactiveTabbedHeaderRenderer": { + "title": {"simpleText": "Legacy gaming"}, + "boxArt": {"thumbnails": [{"url": "//yt3.example/legacy-avatar"}]}, + "banner": {"thumbnails": [{"url": "//yt3.example/legacy-banner"}]}, + "description": {"simpleText": "A legacy description"}, + "badges": [ + {"metadataBadgeRenderer": {"label": "Gaming"}} + ] + } + }, + "microformat": { + "microformatDataRenderer": { + "urlCanonical": "https://www.youtube.com/channel/UCLegacy" + } + } + } + JSON + + header = extract_auto_generated_channel_header(initdata, "UCLegacy") + + expect(header[:author]).to eq("Legacy gaming") + expect(header[:author_url]).to eq("https://www.youtube.com/channel/UCLegacy") + expect(header[:author_thumbnail]).to eq("//yt3.example/legacy-avatar") + expect(header[:banner]).to eq("//yt3.example/legacy-banner") + expect(header[:tags]).to eq(["Gaming"]) + end + + it "keeps an explicit familySafe: false" do + initdata = JSON.parse(<<-JSON).as_h + { + "header": { + "pageHeaderRenderer": {"pageTitle": "Gaming"} + }, + "microformat": { + "microformatDataRenderer": {"familySafe": false} + } + } + JSON + + header = extract_auto_generated_channel_header(initdata, "UCOpNcN46UbXVtpKMrmU4Abg") + + expect(header[:is_family_friendly]).to be_false + end + + it "raises when the header shape is unknown" do + initdata = JSON.parse(<<-JSON).as_h + { + "header": { + "someFutureHeaderRenderer": {} + } + } + JSON + + expect do + extract_auto_generated_channel_header(initdata, "UCUnknown") + end.to raise_error(InfoException) + end +end + +Spectator.describe "extract_topic_channel_details" do + it "returns nil when the payload carries no carousel header" do + initdata = JSON.parse(<<-JSON).as_h + { + "header": { + "pageHeaderRenderer": {"pageTitle": "Gaming"} + } + } + JSON + + expect(extract_topic_channel_details(initdata)).to be_nil + end + + it "returns nil when the carousel carries no topic details" do + initdata = JSON.parse(<<-JSON).as_h + { + "header": { + "carouselHeaderRenderer": { + "contents": [ + {"carouselItemRenderer": {"carouselItems": []}} + ] + } + } + } + JSON + + expect(extract_topic_channel_details(initdata)).to be_nil + end + + it "exposes the subscriber count carried by the subtitle" do + # `subscriberCountText` is part of the renderer but comes back null, so the + # count is only available as free text in the subtitle. + # ex: https://www.youtube.com/channel/UCEgdi0XIXXZ-qJOFPf4JSKw (Sports) + initdata = JSON.parse(<<-JSON).as_h + { + "header": { + "carouselHeaderRenderer": { + "contents": [ + {"carouselItemRenderer": {"carouselItems": []}}, + { + "topicChannelDetailsRenderer": { + "title": {"simpleText": "Sports"}, + "avatar": {"thumbnails": [{"url": "//yt3.example/topic-avatar"}]}, + "subscriberCountText": null, + "subtitle": {"simpleText": "74.3M subscribers"} + } + } + ] + } + } + } + JSON + + details = extract_topic_channel_details(initdata) + + expect(details).not_to be_nil + sub_text = details.not_nil!.dig("subtitle", "simpleText").as_s + expect(sub_text).to eq("74.3M subscribers") + expect(short_text_to_number(sub_text.split(" ")[0])).to eq(74_300_000_i64) + end +end diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr index bb55147b..aec8d27d 100644 --- a/src/invidious/channels/about.cr +++ b/src/invidious/channels/about.cr @@ -19,6 +19,84 @@ record AboutChannel, verified : Bool, is_age_gated : Bool +# Topic channels keep the channel details inside one of the carousel entries. +# The position of that entry varies between channels, so it is looked up by key +# rather than by index. +# ex: https://www.youtube.com/channel/UCEgdi0XIXXZ-qJOFPf4JSKw +def extract_topic_channel_details(initdata : Hash(String, JSON::Any)) : JSON::Any? + contents = initdata.dig?("header", "carouselHeaderRenderer", "contents") + return nil if contents.nil? + + contents.as_a + .find { |content| !content.dig?("topicChannelDetailsRenderer").nil? } + .try &.dig?("topicChannelDetailsRenderer") +end + +# Auto-generated channels come with one of three header shapes. This is only +# reached when the payload has no `metadata` object, i.e. when the regular +# `channelMetadataRenderer` path is not available. +def extract_auto_generated_channel_header(initdata : Hash(String, JSON::Any), ucid : String) + banner = nil + description_node = nil + tags = [] of String + + if header = initdata.dig?("header", "interactiveTabbedHeaderRenderer") + author = header.dig("title", "simpleText").as_s + author_url = initdata.dig("microformat", "microformatDataRenderer", "urlCanonical").as_s + author_thumbnail = header.dig("boxArt", "thumbnails", 0, "url").as_s + + banner = header.dig?("banner", "thumbnails").try &.[-1]?.try &.["url"].as_s? + + description_base_node = header["description"] + # some channels have the description in a simpleText + # ex: https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/ + description_node = description_base_node.dig?("simpleText") || description_base_node + + tags = header.dig?("badges") + .try &.as_a.map(&.["metadataBadgeRenderer"]["label"].as_s) || [] of String + elsif header = initdata.dig?("header", "pageHeaderRenderer") + # ex: https://www.youtube.com/channel/UCOpNcN46UbXVtpKMrmU4Abg + view_model = header.dig?("content", "pageHeaderViewModel") + + author = view_model.try &.dig?("title", "dynamicTextViewModel", "text", "content").try &.as_s + author ||= header.dig?("pageTitle").try &.as_s + author ||= ucid + + author_url = "https://www.youtube.com/channel/#{ucid}" + + author_thumbnail = view_model.try &.dig?("image", "decoratedAvatarViewModel", "avatar", "avatarViewModel", "image", "sources", 0, "url").try &.as_s + author_thumbnail ||= view_model.try &.dig?("animatedImage", "contentPreviewImageViewModel", "image", "sources", 0, "url").try &.as_s + author_thumbnail ||= "" + + banner = view_model.try &.dig?("banner", "imageBannerViewModel", "image", "sources") + .try &.[-1]?.try &.["url"].as_s? + elsif initdata.dig?("header", "carouselHeaderRenderer") + # ex: https://www.youtube.com/channel/UCEgdi0XIXXZ-qJOFPf4JSKw + # This shape carries neither a banner nor a description. + details = extract_topic_channel_details(initdata) + + author = details.try &.dig?("title", "simpleText").try &.as_s || ucid + author_url = "https://www.youtube.com/channel/#{ucid}" + author_thumbnail = details.try &.dig?("avatar", "thumbnails", 0, "url").try &.as_s || "" + else + raise InfoException.new("Could not extract the header of channel #{ucid}") + end + + # `microformat` is absent from these payloads, so a missing flag defaults to + # safe. An explicit `false` is still preserved. + family_safe = initdata.dig?("microformat", "microformatDataRenderer", "familySafe").try(&.as_bool) + + { + author: author, + author_url: author_url, + author_thumbnail: author_thumbnail, + banner: banner, + description_node: description_node, + tags: tags, + is_family_friendly: family_safe.nil? ? true : family_safe, + } +end + def get_about_info(ucid, locale) : AboutChannel begin # Fetch channel information from channel home page @@ -64,21 +142,14 @@ def get_about_info(ucid, locale) : AboutChannel auto_generated = false else if auto_generated - author = initdata["header"]["interactiveTabbedHeaderRenderer"]["title"]["simpleText"].as_s - author_url = initdata["microformat"]["microformatDataRenderer"]["urlCanonical"].as_s - author_thumbnail = initdata["header"]["interactiveTabbedHeaderRenderer"]["boxArt"]["thumbnails"][0]["url"].as_s - - # Raises a KeyError on failure. - banners = initdata["header"]["interactiveTabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? - banner = banners.try &.[-1]?.try &.["url"].as_s? - - description_base_node = initdata["header"]["interactiveTabbedHeaderRenderer"]["description"] - # some channels have the description in a simpleText - # ex: https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/ - description_node = description_base_node.dig?("simpleText") || description_base_node - - tags = initdata.dig?("header", "interactiveTabbedHeaderRenderer", "badges") - .try &.as_a.map(&.["metadataBadgeRenderer"]["label"].as_s) || [] of String + channel_header = extract_auto_generated_channel_header(initdata, ucid) + author = channel_header[:author] + author_url = channel_header[:author_url] + author_thumbnail = channel_header[:author_thumbnail] + banner = channel_header[:banner] + description_node = channel_header[:description_node] + tags = channel_header[:tags] + is_family_friendly = channel_header[:is_family_friendly] else author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s @@ -103,9 +174,9 @@ def get_about_info(ucid, locale) : AboutChannel description_node = initdata["metadata"]["channelMetadataRenderer"]?.try &.["description"]? tags = initdata.dig?("microformat", "microformatDataRenderer", "tags").try &.as_a.map(&.as_s) || [] of String + is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool end - is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool if tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]? # Get the name of the tabs available on this channel tab_names = tabs_json.as_a.compact_map do |entry| @@ -182,6 +253,16 @@ def get_about_info(ucid, locale) : AboutChannel break if sub_count != 0 && !pronouns.nil? end + elsif (topic_details = extract_topic_channel_details(initdata)) + # Topic channels carry the subscriber count as free text in the subtitle, + # ex: "74.3M subscribers". `subscriberCountText` is part of the same + # renderer but comes back null, so it is only used as a first choice. + sub_text = topic_details.dig?("subscriberCountText", "simpleText").try &.as_s + sub_text ||= topic_details.dig?("subtitle", "simpleText").try &.as_s + + if sub_text && sub_text.includes?("subscriber") + sub_count = short_text_to_number(sub_text.split(" ")[0]).to_i32 + end end AboutChannel.new(