mirror of
https://github.com/iv-org/invidious.git
synced 2026-08-16 21:10:53 -05:00
Merge 2c6f804d5a20c00dc006900c02f9cc3e5611b008 into d10f2a48021f1768f2d6ff3bd1b9f3de4e0b2d22
This commit is contained in:
EazyHood
GitHub
commit
036c34d675
295
spec/invidious/channels/about_spec.cr
Normal file
295
spec/invidious/channels/about_spec.cr
Normal file
@ -0,0 +1,295 @@
|
||||
require "../../../src/invidious/exceptions"
|
||||
require "../../spec_helper"
|
||||
|
||||
Spectator.describe "extract_auto_generated_channel_header" do
|
||||
it "parses the carouselHeaderRenderer shape" do
|
||||
# ex: https://www.youtube.com/channel/UCEgdi0XIXXZ-qJOFPf4JSKw (Sports)
|
||||
initdata = JSON.parse(<<-JSON).as_h
|
||||
{
|
||||
"header": {
|
||||
"carouselHeaderRenderer": {
|
||||
"contents": [
|
||||
{
|
||||
"carouselItemRenderer": {
|
||||
"carouselItems": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"topicChannelDetailsRenderer": {
|
||||
"title": {"simpleText": "Sports"},
|
||||
"avatar": {
|
||||
"thumbnails": [
|
||||
{"url": "//yt3.example/topic-avatar", "width": 88, "height": 88}
|
||||
]
|
||||
},
|
||||
"subtitle": {"simpleText": "74.3M subscribers"}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
JSON
|
||||
|
||||
header = extract_auto_generated_channel_header(initdata, "UCEgdi0XIXXZ-qJOFPf4JSKw")
|
||||
|
||||
expect(header[:author]).to eq("Sports")
|
||||
expect(header[:author_url]).to eq("https://www.youtube.com/channel/UCEgdi0XIXXZ-qJOFPf4JSKw")
|
||||
expect(header[:author_thumbnail]).to eq("//yt3.example/topic-avatar")
|
||||
expect(header[:banner]).to be_nil
|
||||
expect(header[:description_node]).to be_nil
|
||||
expect(header[:tags]).to be_empty
|
||||
expect(header[:is_family_friendly]).to be_true
|
||||
end
|
||||
|
||||
it "finds the topic details regardless of their position in the carousel" do
|
||||
initdata = JSON.parse(<<-JSON).as_h
|
||||
{
|
||||
"header": {
|
||||
"carouselHeaderRenderer": {
|
||||
"contents": [
|
||||
{
|
||||
"topicChannelDetailsRenderer": {
|
||||
"title": {"simpleText": "Sports"},
|
||||
"avatar": {"thumbnails": [{"url": "//yt3.example/first"}]}
|
||||
}
|
||||
},
|
||||
{
|
||||
"carouselItemRenderer": {"carouselItems": []}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
JSON
|
||||
|
||||
header = extract_auto_generated_channel_header(initdata, "UCEgdi0XIXXZ-qJOFPf4JSKw")
|
||||
|
||||
expect(header[:author]).to eq("Sports")
|
||||
expect(header[:author_thumbnail]).to eq("//yt3.example/first")
|
||||
end
|
||||
|
||||
it "raises when the carousel carries no topic details" do
|
||||
initdata = JSON.parse(<<-JSON).as_h
|
||||
{
|
||||
"header": {
|
||||
"carouselHeaderRenderer": {
|
||||
"contents": [
|
||||
{"carouselItemRenderer": {"carouselItems": []}}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
JSON
|
||||
|
||||
expect do
|
||||
extract_auto_generated_channel_header(initdata, "UCEgdi0XIXXZ-qJOFPf4JSKw")
|
||||
end.to raise_error(InfoException)
|
||||
end
|
||||
|
||||
it "raises when the carousel topic details have no title" do
|
||||
initdata = JSON.parse(<<-JSON).as_h
|
||||
{
|
||||
"header": {
|
||||
"carouselHeaderRenderer": {
|
||||
"contents": [
|
||||
{
|
||||
"topicChannelDetailsRenderer": {
|
||||
"avatar": {"thumbnails": [{"url": "//yt3.example/avatar"}]}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
JSON
|
||||
|
||||
expect do
|
||||
extract_auto_generated_channel_header(initdata, "UCEgdi0XIXXZ-qJOFPf4JSKw")
|
||||
end.to raise_error(InfoException)
|
||||
end
|
||||
|
||||
it "still renders a carousel channel that carries no avatar" do
|
||||
initdata = JSON.parse(<<-JSON).as_h
|
||||
{
|
||||
"header": {
|
||||
"carouselHeaderRenderer": {
|
||||
"contents": [
|
||||
{
|
||||
"topicChannelDetailsRenderer": {
|
||||
"title": {"simpleText": "Sports"}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
JSON
|
||||
|
||||
header = extract_auto_generated_channel_header(initdata, "UCEgdi0XIXXZ-qJOFPf4JSKw")
|
||||
|
||||
expect(header[:author]).to eq("Sports")
|
||||
expect(header[:author_thumbnail]).to eq("")
|
||||
end
|
||||
|
||||
it "parses the current pageHeaderRenderer shape" do
|
||||
# ex: https://www.youtube.com/channel/UCOpNcN46UbXVtpKMrmU4Abg (Gaming)
|
||||
initdata = JSON.parse(<<-JSON).as_h
|
||||
{
|
||||
"header": {
|
||||
"pageHeaderRenderer": {
|
||||
"pageTitle": "Gaming",
|
||||
"content": {
|
||||
"pageHeaderViewModel": {
|
||||
"title": {
|
||||
"dynamicTextViewModel": {
|
||||
"text": {"content": "Gaming"}
|
||||
}
|
||||
},
|
||||
"animatedImage": {
|
||||
"contentPreviewImageViewModel": {
|
||||
"image": {
|
||||
"sources": [
|
||||
{"url": "//yt3.example/avatar", "width": 48, "height": 48}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
JSON
|
||||
|
||||
header = extract_auto_generated_channel_header(initdata, "UCOpNcN46UbXVtpKMrmU4Abg")
|
||||
|
||||
expect(header[:author]).to eq("Gaming")
|
||||
expect(header[:author_url]).to eq("https://www.youtube.com/channel/UCOpNcN46UbXVtpKMrmU4Abg")
|
||||
expect(header[:author_thumbnail]).to eq("//yt3.example/avatar")
|
||||
end
|
||||
|
||||
it "preserves the legacy interactiveTabbedHeaderRenderer shape" do
|
||||
initdata = JSON.parse(<<-JSON).as_h
|
||||
{
|
||||
"header": {
|
||||
"interactiveTabbedHeaderRenderer": {
|
||||
"title": {"simpleText": "Legacy gaming"},
|
||||
"boxArt": {"thumbnails": [{"url": "//yt3.example/legacy-avatar"}]},
|
||||
"banner": {"thumbnails": [{"url": "//yt3.example/legacy-banner"}]},
|
||||
"description": {"simpleText": "A legacy description"},
|
||||
"badges": [
|
||||
{"metadataBadgeRenderer": {"label": "Gaming"}}
|
||||
]
|
||||
}
|
||||
},
|
||||
"microformat": {
|
||||
"microformatDataRenderer": {
|
||||
"urlCanonical": "https://www.youtube.com/channel/UCLegacy"
|
||||
}
|
||||
}
|
||||
}
|
||||
JSON
|
||||
|
||||
header = extract_auto_generated_channel_header(initdata, "UCLegacy")
|
||||
|
||||
expect(header[:author]).to eq("Legacy gaming")
|
||||
expect(header[:author_url]).to eq("https://www.youtube.com/channel/UCLegacy")
|
||||
expect(header[:author_thumbnail]).to eq("//yt3.example/legacy-avatar")
|
||||
expect(header[:banner]).to eq("//yt3.example/legacy-banner")
|
||||
expect(header[:tags]).to eq(["Gaming"])
|
||||
end
|
||||
|
||||
it "keeps an explicit familySafe: false" do
|
||||
initdata = JSON.parse(<<-JSON).as_h
|
||||
{
|
||||
"header": {
|
||||
"pageHeaderRenderer": {"pageTitle": "Gaming"}
|
||||
},
|
||||
"microformat": {
|
||||
"microformatDataRenderer": {"familySafe": false}
|
||||
}
|
||||
}
|
||||
JSON
|
||||
|
||||
header = extract_auto_generated_channel_header(initdata, "UCOpNcN46UbXVtpKMrmU4Abg")
|
||||
|
||||
expect(header[:is_family_friendly]).to be_false
|
||||
end
|
||||
|
||||
it "raises when the header shape is unknown" do
|
||||
initdata = JSON.parse(<<-JSON).as_h
|
||||
{
|
||||
"header": {
|
||||
"someFutureHeaderRenderer": {}
|
||||
}
|
||||
}
|
||||
JSON
|
||||
|
||||
expect do
|
||||
extract_auto_generated_channel_header(initdata, "UCUnknown")
|
||||
end.to raise_error(InfoException)
|
||||
end
|
||||
end
|
||||
|
||||
Spectator.describe "extract_topic_channel_details" do
|
||||
it "returns nil when the payload carries no carousel header" do
|
||||
initdata = JSON.parse(<<-JSON).as_h
|
||||
{
|
||||
"header": {
|
||||
"pageHeaderRenderer": {"pageTitle": "Gaming"}
|
||||
}
|
||||
}
|
||||
JSON
|
||||
|
||||
expect(extract_topic_channel_details(initdata)).to be_nil
|
||||
end
|
||||
|
||||
it "returns nil when the carousel carries no topic details" do
|
||||
initdata = JSON.parse(<<-JSON).as_h
|
||||
{
|
||||
"header": {
|
||||
"carouselHeaderRenderer": {
|
||||
"contents": [
|
||||
{"carouselItemRenderer": {"carouselItems": []}}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
JSON
|
||||
|
||||
expect(extract_topic_channel_details(initdata)).to be_nil
|
||||
end
|
||||
|
||||
it "exposes the subscriber count carried by the subtitle" do
|
||||
# `subscriberCountText` is part of the renderer but comes back null, so the
|
||||
# count is only available as free text in the subtitle.
|
||||
# ex: https://www.youtube.com/channel/UCEgdi0XIXXZ-qJOFPf4JSKw (Sports)
|
||||
initdata = JSON.parse(<<-JSON).as_h
|
||||
{
|
||||
"header": {
|
||||
"carouselHeaderRenderer": {
|
||||
"contents": [
|
||||
{"carouselItemRenderer": {"carouselItems": []}},
|
||||
{
|
||||
"topicChannelDetailsRenderer": {
|
||||
"title": {"simpleText": "Sports"},
|
||||
"avatar": {"thumbnails": [{"url": "//yt3.example/topic-avatar"}]},
|
||||
"subscriberCountText": null,
|
||||
"subtitle": {"simpleText": "74.3M subscribers"}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
JSON
|
||||
|
||||
details = extract_topic_channel_details(initdata)
|
||||
|
||||
expect(details).not_to be_nil
|
||||
sub_text = details.not_nil!.dig("subtitle", "simpleText").as_s
|
||||
expect(sub_text).to eq("74.3M subscribers")
|
||||
expect(short_text_to_number(sub_text.split(" ")[0])).to eq(74_300_000_i64)
|
||||
end
|
||||
end
|
||||
22
spec/invidious/yt_backend/extractors_spec.cr
Normal file
22
spec/invidious/yt_backend/extractors_spec.cr
Normal file
@ -0,0 +1,22 @@
|
||||
require "../../parsers_helper"
|
||||
|
||||
Spectator.describe "YouTubeTabs" do
|
||||
it "treats a selected tab without content as empty" do
|
||||
initdata = JSON.parse(<<-JSON).as_h
|
||||
{
|
||||
"contents": {
|
||||
"twoColumnBrowseResultsRenderer": {
|
||||
"tabs": [
|
||||
{"tabRenderer": {"selected": true}}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
JSON
|
||||
|
||||
items, continuation = extract_items(initdata)
|
||||
|
||||
expect(items).to be_empty
|
||||
expect(continuation).to be_nil
|
||||
end
|
||||
end
|
||||
@ -19,6 +19,92 @@ record AboutChannel,
|
||||
verified : Bool,
|
||||
is_age_gated : Bool
|
||||
|
||||
# Topic channels keep the channel details inside one of the carousel entries.
|
||||
# The position of that entry varies between channels, so it is looked up by key
|
||||
# rather than by index.
|
||||
# ex: https://www.youtube.com/channel/UCEgdi0XIXXZ-qJOFPf4JSKw
|
||||
def extract_topic_channel_details(initdata : Hash(String, JSON::Any)) : JSON::Any?
|
||||
contents = initdata.dig?("header", "carouselHeaderRenderer", "contents")
|
||||
return nil if contents.nil?
|
||||
|
||||
contents.as_a
|
||||
.find { |content| !content.dig?("topicChannelDetailsRenderer").nil? }
|
||||
.try &.dig?("topicChannelDetailsRenderer")
|
||||
end
|
||||
|
||||
# Auto-generated channels come with one of three header shapes. This is only
|
||||
# reached when the payload has no `metadata` object, i.e. when the regular
|
||||
# `channelMetadataRenderer` path is not available.
|
||||
def extract_auto_generated_channel_header(initdata : Hash(String, JSON::Any), ucid : String)
|
||||
banner = nil
|
||||
description_node = nil
|
||||
tags = [] of String
|
||||
|
||||
if header = initdata.dig?("header", "interactiveTabbedHeaderRenderer")
|
||||
author = header.dig("title", "simpleText").as_s
|
||||
author_url = initdata.dig("microformat", "microformatDataRenderer", "urlCanonical").as_s
|
||||
author_thumbnail = header.dig("boxArt", "thumbnails", 0, "url").as_s
|
||||
|
||||
banner = header.dig?("banner", "thumbnails").try &.[-1]?.try &.["url"].as_s?
|
||||
|
||||
description_base_node = header["description"]
|
||||
# some channels have the description in a simpleText
|
||||
# ex: https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/
|
||||
description_node = description_base_node.dig?("simpleText") || description_base_node
|
||||
|
||||
tags = header.dig?("badges")
|
||||
.try &.as_a.map(&.["metadataBadgeRenderer"]["label"].as_s) || [] of String
|
||||
elsif header = initdata.dig?("header", "pageHeaderRenderer")
|
||||
# ex: https://www.youtube.com/channel/UCOpNcN46UbXVtpKMrmU4Abg
|
||||
view_model = header.dig?("content", "pageHeaderViewModel")
|
||||
|
||||
author = view_model.try &.dig?("title", "dynamicTextViewModel", "text", "content").try &.as_s
|
||||
author ||= header.dig?("pageTitle").try &.as_s
|
||||
author ||= ucid
|
||||
|
||||
author_url = "https://www.youtube.com/channel/#{ucid}"
|
||||
|
||||
author_thumbnail = view_model.try &.dig?("image", "decoratedAvatarViewModel", "avatar", "avatarViewModel", "image", "sources", 0, "url").try &.as_s
|
||||
author_thumbnail ||= view_model.try &.dig?("animatedImage", "contentPreviewImageViewModel", "image", "sources", 0, "url").try &.as_s
|
||||
author_thumbnail ||= ""
|
||||
|
||||
banner = view_model.try &.dig?("banner", "imageBannerViewModel", "image", "sources")
|
||||
.try &.[-1]?.try &.["url"].as_s?
|
||||
elsif initdata.dig?("header", "carouselHeaderRenderer")
|
||||
# ex: https://www.youtube.com/channel/UCEgdi0XIXXZ-qJOFPf4JSKw
|
||||
# This shape carries neither a banner nor a description.
|
||||
details = extract_topic_channel_details(initdata)
|
||||
|
||||
unless details
|
||||
raise InfoException.new("Could not extract the carousel header of channel #{ucid}")
|
||||
end
|
||||
|
||||
author = details.dig?("title", "simpleText").try &.as_s
|
||||
author ||= raise InfoException.new("Could not extract the carousel title of channel #{ucid}")
|
||||
author_url = "https://www.youtube.com/channel/#{ucid}"
|
||||
|
||||
# A missing avatar is not worth failing the whole page over, and the
|
||||
# `pageHeaderRenderer` branch above falls back to an empty string too.
|
||||
author_thumbnail = details.dig?("avatar", "thumbnails", 0, "url").try &.as_s || ""
|
||||
else
|
||||
raise InfoException.new("Could not extract the header of channel #{ucid}")
|
||||
end
|
||||
|
||||
# `microformat` is absent from these payloads, so a missing flag defaults to
|
||||
# safe. An explicit `false` is still preserved.
|
||||
family_safe = initdata.dig?("microformat", "microformatDataRenderer", "familySafe").try(&.as_bool)
|
||||
|
||||
{
|
||||
author: author,
|
||||
author_url: author_url,
|
||||
author_thumbnail: author_thumbnail,
|
||||
banner: banner,
|
||||
description_node: description_node,
|
||||
tags: tags,
|
||||
is_family_friendly: family_safe.nil? ? true : family_safe,
|
||||
}
|
||||
end
|
||||
|
||||
def get_about_info(ucid, locale) : AboutChannel
|
||||
begin
|
||||
# Fetch channel information from channel home page
|
||||
@ -64,21 +150,14 @@ def get_about_info(ucid, locale) : AboutChannel
|
||||
auto_generated = false
|
||||
else
|
||||
if auto_generated
|
||||
author = initdata["header"]["interactiveTabbedHeaderRenderer"]["title"]["simpleText"].as_s
|
||||
author_url = initdata["microformat"]["microformatDataRenderer"]["urlCanonical"].as_s
|
||||
author_thumbnail = initdata["header"]["interactiveTabbedHeaderRenderer"]["boxArt"]["thumbnails"][0]["url"].as_s
|
||||
|
||||
# Raises a KeyError on failure.
|
||||
banners = initdata["header"]["interactiveTabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]?
|
||||
banner = banners.try &.[-1]?.try &.["url"].as_s?
|
||||
|
||||
description_base_node = initdata["header"]["interactiveTabbedHeaderRenderer"]["description"]
|
||||
# some channels have the description in a simpleText
|
||||
# ex: https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/
|
||||
description_node = description_base_node.dig?("simpleText") || description_base_node
|
||||
|
||||
tags = initdata.dig?("header", "interactiveTabbedHeaderRenderer", "badges")
|
||||
.try &.as_a.map(&.["metadataBadgeRenderer"]["label"].as_s) || [] of String
|
||||
channel_header = extract_auto_generated_channel_header(initdata, ucid)
|
||||
author = channel_header[:author]
|
||||
author_url = channel_header[:author_url]
|
||||
author_thumbnail = channel_header[:author_thumbnail]
|
||||
banner = channel_header[:banner]
|
||||
description_node = channel_header[:description_node]
|
||||
tags = channel_header[:tags]
|
||||
is_family_friendly = channel_header[:is_family_friendly]
|
||||
else
|
||||
author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s
|
||||
author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s
|
||||
@ -103,9 +182,9 @@ def get_about_info(ucid, locale) : AboutChannel
|
||||
|
||||
description_node = initdata["metadata"]["channelMetadataRenderer"]?.try &.["description"]?
|
||||
tags = initdata.dig?("microformat", "microformatDataRenderer", "tags").try &.as_a.map(&.as_s) || [] of String
|
||||
is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool
|
||||
end
|
||||
|
||||
is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool
|
||||
if tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]?
|
||||
# Get the name of the tabs available on this channel
|
||||
tab_names = tabs_json.as_a.compact_map do |entry|
|
||||
@ -182,6 +261,16 @@ def get_about_info(ucid, locale) : AboutChannel
|
||||
|
||||
break if sub_count != 0 && !pronouns.nil?
|
||||
end
|
||||
elsif (topic_details = extract_topic_channel_details(initdata))
|
||||
# Topic channels carry the subscriber count as free text in the subtitle,
|
||||
# ex: "74.3M subscribers". `subscriberCountText` is part of the same
|
||||
# renderer but comes back null, so it is only used as a first choice.
|
||||
sub_text = topic_details.dig?("subscriberCountText", "simpleText").try &.as_s
|
||||
sub_text ||= topic_details.dig?("subtitle", "simpleText").try &.as_s
|
||||
|
||||
if sub_text && sub_text.includes?("subscriber")
|
||||
sub_count = short_text_to_number(sub_text.split(" ")[0]).to_i32
|
||||
end
|
||||
end
|
||||
|
||||
AboutChannel.new(
|
||||
|
||||
@ -964,7 +964,8 @@ private module Extractors
|
||||
|
||||
private def self.extract(target)
|
||||
raw_items = [] of JSON::Any
|
||||
content = extract_selected_tab(target["tabs"])["content"]
|
||||
content = extract_selected_tab(target["tabs"])["content"]?
|
||||
return raw_items if content.nil?
|
||||
|
||||
if section_list_contents = content.dig?("sectionListRenderer", "contents")
|
||||
raw_items = unpack_section_list(section_list_contents)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user