mirror of
https://github.com/iv-org/invidious.git
synced 2026-09-06 09:02:45 -05:00
videos: parse lockupViewModel-based related videos on watch pages
YouTube migrated the related videos section to lockupViewModel items wrapped in an itemSectionRenderer. Parse them into the same hash as compactVideoRenderer items, including author channel links stored on the avatar (single author) or in an avatar-stack dialog (collabs). Fixes: #5722
This commit is contained in:
soygeitoyt-lgtm
parent
080f87f0f5
commit
9c24985401
96
spec/invidious/videos/related_lockup_extract_spec.cr
Normal file
96
spec/invidious/videos/related_lockup_extract_spec.cr
Normal file
@ -0,0 +1,96 @@
|
||||
require "../../parsers_helper.cr"
|
||||
|
||||
# Wraps raw lockupViewModel items the same way YouTube encapsulates them in
|
||||
# the watch page's related videos section (secondaryResults -> results ->
|
||||
# itemSectionRenderer -> contents).
|
||||
private def wrap_related_lockup(lockups : Array(JSON::Any)) : Hash(String, JSON::Any)
|
||||
contents = lockups.map { |lockup| {"lockupViewModel" => lockup} }
|
||||
|
||||
wrapped = {
|
||||
"secondaryResults": {
|
||||
"secondaryResults": {
|
||||
"results": [{
|
||||
"itemSectionRenderer": {
|
||||
"contents": contents,
|
||||
},
|
||||
}],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
return JSON.parse(wrapped.to_json).as_h
|
||||
end
|
||||
|
||||
Spectator.describe "parse_related_videos" do
|
||||
# Real InnerTube data, captured from a watch page's related videos
|
||||
# section (2026-08).
|
||||
# See: https://github.com/iv-org/invidious/issues/5722
|
||||
COLLAB_RELATED_LOCKUP = JSON.parse(%q({"contentType":"LOCKUP_CONTENT_TYPE_VIDEO","contentId":"ywd-Ve8a8Tc","contentImage":{"thumbnailViewModel":{"image":{"sources":[{"url":"https://i.ytimg.com/vi/ywd-Ve8a8Tc/hqdefault.jpg","width":360,"height":202}]},"overlays":[{"thumbnailBottomOverlayViewModel":{"badges":[{"thumbnailBadgeViewModel":{"text":"2:31:01"}}]}}]}},"metadata":{"lockupMetadataViewModel":{"title":{"content":"World Order Is a Lie: The Next 50 Years Will Change Everything"},"image":{"avatarStackViewModel":{"rendererContext":{"commandContext":{"onTap":{"innertubeCommand":{"showDialogCommand":{"panelLoadingStrategy":{"inlineContent":{"dialogViewModel":{"customContent":{"listViewModel":{"listItems":[{"listItemViewModel":{"title":{"content":"Raj Shamani"},"rendererContext":{"commandContext":{"onTap":{"innertubeCommand":{"browseEndpoint":{"browseId":"UCzwCEE_PchiBULMnAJqhGVg"}}}}}}},{"listItemViewModel":{"title":{"content":"Predictive History"},"rendererContext":{"commandContext":{"onTap":{"innertubeCommand":{"browseEndpoint":{"browseId":"UC11aHtNnc5bEPLI4jf6mnYg"}}}}}}}]}}}}}}}}}}}},"metadata":{"contentMetadataViewModel":{"metadataRows":[{"metadataParts":[{"text":{"content":"Raj Shamani and Predictive History","attachmentRuns":[{"element":{"type":{"imageType":{"image":{"sources":[{"clientResource":{"imageName":"CHECK_CIRCLE_FILLED"},"width":14,"height":14}]}}}}}]}}]},{"metadataParts":[{"text":{"content":"2M"}},{"text":{"content":"10d ago"}}]}]}}}}}))
|
||||
|
||||
SINGLE_AUTHOR_RELATED_LOCKUP = JSON.parse(%q({"contentType":"LOCKUP_CONTENT_TYPE_VIDEO","contentId":"XuoqKYxDHVc","contentImage":{"thumbnailViewModel":{"decoratedAvatarViewModel":{"avatar":{"avatarViewModel":{"size":"AVATAR_SIZE_TYPE_M","image":{"sources":[{"url":"https://yt3.ggpht.com/ytc/example","width":88,"height":88}]}}},"rendererContext":{"commandContext":{"onTap":{"innertubeCommand":{"browseEndpoint":{"browseId":"UC0p5jTq6Xx_DosDFxVXnWaQ"}}}}}},"image":{"sources":[{"url":"https://i.ytimg.com/vi/XuoqKYxDHVc/hqdefault.jpg","width":360,"height":202}]},"overlays":[{"thumbnailBottomOverlayViewModel":{"badges":[{"thumbnailBadgeViewModel":{"text":"12:34"}}]}}]}},"metadata":{"lockupMetadataViewModel":{"title":{"content":"Why Russia Is Winning The Energy War"},"metadata":{"contentMetadataViewModel":{"metadataRows":[{"metadataParts":[{"text":{"content":"The Economist"}}]},{"metadataParts":[{"text":{"content":"1.2K views"}},{"text":{"content":"3w ago"}}]}]}}}}}))
|
||||
|
||||
LEGACY_CVR_ITEM = JSON.parse(%q({"videoId":"aqz-KE-bpKQ","title":{"simpleText":"Big Buck Bunny"},"lengthText":{"simpleText":"9:56"},"shortBylineText":{"runs":[{"text":"Blender","navigationEndpoint":{"browseEndpoint":{"browseId":"UCSMOQeBJ2RAnuFungnYDxuA"}}}]},"ownerText":{"runs":[{"text":"Blender"}]},"shortViewCountText":{"simpleText":"10M views"},"publishedTimeText":{"simpleText":"15 years ago"}}))
|
||||
|
||||
it "parses a collaboration related video with multiple channel links" do
|
||||
results = wrap_related_lockup([COLLAB_RELATED_LOCKUP]).dig?("secondaryResults", "secondaryResults", "results")
|
||||
|
||||
related = Invidious::Videos::Parser.parse_related_videos(results)
|
||||
|
||||
expect(related.size).to eq(1)
|
||||
|
||||
video = related[0].as_h
|
||||
expect(video["id"].as_s).to eq("ywd-Ve8a8Tc")
|
||||
expect(video["ucid"].as_s).to eq("UCzwCEE_PchiBULMnAJqhGVg")
|
||||
expect(video["author"].as_s).to eq("Raj Shamani and Predictive History")
|
||||
expect(video["author_verified"].as_s).to eq("true")
|
||||
expect(video["length_seconds"].as_s).to eq((2*3600 + 31*60 + 1).to_s)
|
||||
expect(video["short_view_count"].as_s).to eq("2M")
|
||||
expect(Time.parse_rfc3339(video["published"].as_s)).to be_close(Time.local - 10.days, 1.days)
|
||||
end
|
||||
|
||||
it "parses a single-author related video (channel link on avatar)" do
|
||||
results = wrap_related_lockup([SINGLE_AUTHOR_RELATED_LOCKUP]).dig?("secondaryResults", "secondaryResults", "results")
|
||||
|
||||
related = Invidious::Videos::Parser.parse_related_videos(results)
|
||||
|
||||
expect(related.size).to eq(1)
|
||||
|
||||
video = related[0].as_h
|
||||
expect(video["id"].as_s).to eq("XuoqKYxDHVc")
|
||||
expect(video["ucid"].as_s).to eq("UC0p5jTq6Xx_DosDFxVXnWaQ")
|
||||
expect(video["author"].as_s).to eq("The Economist")
|
||||
expect(video["author_verified"].as_s).to eq("false")
|
||||
expect(video["length_seconds"].as_s).to eq(754.to_s)
|
||||
expect(video["short_view_count"].as_s).to eq("1.2K")
|
||||
expect(Time.parse_rfc3339(video["published"].as_s)).to be_close(Time.local - 21.days, 1.days)
|
||||
end
|
||||
|
||||
it "ignores non-video lockups" do
|
||||
playlist_lockup = JSON.parse(%q({"contentType":"LOCKUP_CONTENT_TYPE_PLAYLIST","contentId":"PLtest"}))
|
||||
|
||||
results = wrap_related_lockup([playlist_lockup]).dig?("secondaryResults", "secondaryResults", "results")
|
||||
|
||||
related = Invidious::Videos::Parser.parse_related_videos(results)
|
||||
|
||||
expect(related.size).to eq(0)
|
||||
end
|
||||
|
||||
it "still parses legacy compactVideoRenderer items" do
|
||||
results = JSON.parse({
|
||||
"secondaryResults": {
|
||||
"secondaryResults": {
|
||||
"results": [{"compactVideoRenderer" => LEGACY_CVR_ITEM}],
|
||||
},
|
||||
},
|
||||
}.to_json).dig?("secondaryResults", "secondaryResults", "results")
|
||||
|
||||
related = Invidious::Videos::Parser.parse_related_videos(results)
|
||||
|
||||
expect(related.size).to eq(1)
|
||||
|
||||
video = related[0].as_h
|
||||
expect(video["id"].as_s).to eq("aqz-KE-bpKQ")
|
||||
expect(video["ucid"].as_s).to eq("UCSMOQeBJ2RAnuFungnYDxuA")
|
||||
expect(video["length_seconds"].as_s).to eq((9*60 + 56).to_s)
|
||||
end
|
||||
end
|
||||
@ -55,6 +55,145 @@ module Invidious::Videos::Parser
|
||||
}
|
||||
end
|
||||
|
||||
# Use to parse the "lockupViewModel" items from the watch page's related
|
||||
# videos section (new format as of 2026-08). Produces the same hash as
|
||||
# #parse_related_video.
|
||||
#
|
||||
# The author's channel link is stored in different places depending on
|
||||
# whether the video is a collaboration: for regular videos it is attached
|
||||
# to the avatar, while collaborations carry an avatar-stack dialog with one
|
||||
# entry per channel. Only the first channel is kept, since the related
|
||||
# videos hash exposes a single author/ucid pair.
|
||||
def parse_related_lockup(lockup : JSON::Any) : Hash(String, JSON::Any)?
|
||||
return nil if !lockup["contentId"]?
|
||||
return nil if lockup["contentType"]?.try(&.as_s) != "LOCKUP_CONTENT_TYPE_VIDEO"
|
||||
|
||||
metadata = lockup.dig?("metadata", "lockupMetadataViewModel")
|
||||
return nil unless metadata
|
||||
|
||||
title = metadata.dig?("title", "content").try(&.as_s) || ""
|
||||
|
||||
ucids = [] of String
|
||||
|
||||
if browse = lockup.dig?("contentImage", "thumbnailViewModel", "decoratedAvatarViewModel",
|
||||
"rendererContext", "commandContext", "onTap", "innertubeCommand", "browseEndpoint")
|
||||
self.add_channel_browse_id(ucids, browse)
|
||||
end
|
||||
|
||||
avatar_stack_items = metadata.dig?("image", "avatarStackViewModel", "rendererContext",
|
||||
"commandContext", "onTap", "innertubeCommand", "showDialogCommand", "panelLoadingStrategy",
|
||||
"inlineContent", "dialogViewModel", "customContent", "listViewModel", "listItems")
|
||||
|
||||
avatar_stack_items.try &.as_a.each do |item|
|
||||
if browse = item.dig?("listItemViewModel", "rendererContext", "commandContext",
|
||||
"onTap", "innertubeCommand", "browseEndpoint")
|
||||
self.add_channel_browse_id(ucids, browse)
|
||||
end
|
||||
end
|
||||
|
||||
author = ""
|
||||
author_verified = false
|
||||
short_view_count = ""
|
||||
published_text = ""
|
||||
|
||||
metadata_rows = metadata.dig?("metadata", "contentMetadataViewModel", "metadataRows")
|
||||
metadata_rows.try &.as_a.each do |row|
|
||||
row.dig?("metadataParts").try &.as_a.each do |part|
|
||||
text = part.dig?("text", "content").try(&.as_s)
|
||||
next unless text
|
||||
|
||||
if part.dig?("text", "attachmentRuns").try &.as_a.any? { |attachment|
|
||||
attachment.dig?("element", "type", "imageType", "image", "sources", 0,
|
||||
"clientResource", "imageName").try(&.as_s) == "CHECK_CIRCLE_FILLED"
|
||||
}
|
||||
author_verified = true
|
||||
end
|
||||
|
||||
if text.matches?(/ago/i)
|
||||
published_text = text
|
||||
elsif short_view_count.empty? && text.lstrip.matches?(/\d/)
|
||||
short_view_count = text.gsub(/\s*views?$/i, "")
|
||||
else
|
||||
author = text if author.empty?
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
length_text = lockup.dig?("contentImage", "thumbnailViewModel", "overlays", 0,
|
||||
"thumbnailBottomOverlayViewModel", "badges", 0, "thumbnailBadgeViewModel", "text")
|
||||
.try(&.as_s)
|
||||
|
||||
length_seconds = length_text ? decode_length_seconds(length_text.not_nil!).to_s : "0"
|
||||
|
||||
published = published_text.empty? ? "" : self.decode_compact_published(published_text)
|
||||
|
||||
return {
|
||||
"id" => lockup["contentId"],
|
||||
"title" => JSON::Any.new(title),
|
||||
"author" => JSON::Any.new(author),
|
||||
"ucid" => JSON::Any.new(ucids.first? || ""),
|
||||
"length_seconds" => JSON::Any.new(length_seconds),
|
||||
"short_view_count" => JSON::Any.new(short_view_count),
|
||||
"author_verified" => JSON::Any.new(author_verified.to_s),
|
||||
"published" => JSON::Any.new(published),
|
||||
}
|
||||
end
|
||||
|
||||
private def add_channel_browse_id(list : Array(String), browse_endpoint : JSON::Any)
|
||||
browse_id = browse_endpoint.dig?("browseId").try(&.as_s)
|
||||
list << browse_id if browse_id && browse_id.starts_with?("UC") && !list.includes?(browse_id)
|
||||
end
|
||||
|
||||
# Handles the compact relative dates ("10d ago") used by lockups, falling
|
||||
# back to the regular long forms ("10 days ago").
|
||||
private def decode_compact_published(text : String) : String
|
||||
if match = text.match(/(\d+)\s*([smhdwy])\s+ago/i)
|
||||
amount = match[1].to_i64
|
||||
|
||||
time = case match[2].downcase
|
||||
when "s" then Time.local - amount.seconds
|
||||
when "m" then Time.local - amount.minutes
|
||||
when "h" then Time.local - amount.hours
|
||||
when "d" then Time.local - amount.days
|
||||
when "w" then Time.local - amount.weeks
|
||||
when "y" then Time.local - amount.years
|
||||
else Time.local - amount.days * 30
|
||||
end
|
||||
|
||||
return time.to_rfc3339
|
||||
end
|
||||
|
||||
return decode_date(text).to_rfc3339
|
||||
rescue
|
||||
return ""
|
||||
end
|
||||
|
||||
# Parses the related videos section of a watch page. Supports both the
|
||||
# legacy "compactVideoRenderer" items and the new "lockupViewModel"
|
||||
# items wrapped in an item section (as of 2026-08).
|
||||
def parse_related_videos(results : JSON::Any?) : Array(JSON::Any)
|
||||
related = [] of JSON::Any
|
||||
|
||||
results.try &.as_a.each do |element|
|
||||
if item = element["compactVideoRenderer"]?
|
||||
related_video = self.parse_related_video(item)
|
||||
related << JSON::Any.new(related_video) if related_video
|
||||
elsif section = element["itemSectionRenderer"]?
|
||||
section.dig?("contents").try &.as_a.each do |sub_element|
|
||||
if item = sub_element["lockupViewModel"]?
|
||||
related_video = self.parse_related_lockup(item)
|
||||
related << JSON::Any.new(related_video) if related_video
|
||||
elsif item = sub_element["compactVideoRenderer"]?
|
||||
related_video = self.parse_related_video(item)
|
||||
related << JSON::Any.new(related_video) if related_video
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
return related
|
||||
end
|
||||
|
||||
def extract_video_info(video_id : String)
|
||||
# Fetch data from the player endpoint
|
||||
player_response = YoutubeAPI.player(video_id: video_id)
|
||||
@ -240,15 +379,9 @@ module Invidious::Videos::Parser
|
||||
|
||||
related = [] of JSON::Any
|
||||
|
||||
# Parse "compactVideoRenderer" items (under secondary results)
|
||||
secondary_results = main_results
|
||||
.dig?("secondaryResults", "secondaryResults", "results")
|
||||
secondary_results.try &.as_a.each do |element|
|
||||
if item = element["compactVideoRenderer"]?
|
||||
related_video = self.parse_related_video(item)
|
||||
related << JSON::Any.new(related_video) if related_video
|
||||
end
|
||||
end
|
||||
related = self.parse_related_videos(
|
||||
main_results.dig?("secondaryResults", "secondaryResults", "results")
|
||||
)
|
||||
|
||||
# If nothing was found previously, fall back to end screen renderer
|
||||
if related.empty?
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user