diff --git a/spec/invidious/videos/related_lockup_extract_spec.cr b/spec/invidious/videos/related_lockup_extract_spec.cr new file mode 100644 index 000000000..7688ca9d8 --- /dev/null +++ b/spec/invidious/videos/related_lockup_extract_spec.cr @@ -0,0 +1,96 @@ +require "../../parsers_helper.cr" + +# Wraps raw lockupViewModel items the same way YouTube encapsulates them in +# the watch page's related videos section (secondaryResults -> results -> +# itemSectionRenderer -> contents). +private def wrap_related_lockup(lockups : Array(JSON::Any)) : Hash(String, JSON::Any) + contents = lockups.map { |lockup| {"lockupViewModel" => lockup} } + + wrapped = { + "secondaryResults": { + "secondaryResults": { + "results": [{ + "itemSectionRenderer": { + "contents": contents, + }, + }], + }, + }, + } + + return JSON.parse(wrapped.to_json).as_h +end + +Spectator.describe "parse_related_videos" do + # Real InnerTube data, captured from a watch page's related videos + # section (2026-08). + # See: https://github.com/iv-org/invidious/issues/5722 + COLLAB_RELATED_LOCKUP = JSON.parse(%q({"contentType":"LOCKUP_CONTENT_TYPE_VIDEO","contentId":"ywd-Ve8a8Tc","contentImage":{"thumbnailViewModel":{"image":{"sources":[{"url":"https://i.ytimg.com/vi/ywd-Ve8a8Tc/hqdefault.jpg","width":360,"height":202}]},"overlays":[{"thumbnailBottomOverlayViewModel":{"badges":[{"thumbnailBadgeViewModel":{"text":"2:31:01"}}]}}]}},"metadata":{"lockupMetadataViewModel":{"title":{"content":"World Order Is a Lie: The Next 50 Years Will Change Everything"},"image":{"avatarStackViewModel":{"rendererContext":{"commandContext":{"onTap":{"innertubeCommand":{"showDialogCommand":{"panelLoadingStrategy":{"inlineContent":{"dialogViewModel":{"customContent":{"listViewModel":{"listItems":[{"listItemViewModel":{"title":{"content":"Raj Shamani"},"rendererContext":{"commandContext":{"onTap":{"innertubeCommand":{"browseEndpoint":{"browseId":"UCzwCEE_PchiBULMnAJqhGVg"}}}}}}},{"listItemViewModel":{"title":{"content":"Predictive History"},"rendererContext":{"commandContext":{"onTap":{"innertubeCommand":{"browseEndpoint":{"browseId":"UC11aHtNnc5bEPLI4jf6mnYg"}}}}}}}]}}}}}}}}}}}},"metadata":{"contentMetadataViewModel":{"metadataRows":[{"metadataParts":[{"text":{"content":"Raj Shamani and Predictive History","attachmentRuns":[{"element":{"type":{"imageType":{"image":{"sources":[{"clientResource":{"imageName":"CHECK_CIRCLE_FILLED"},"width":14,"height":14}]}}}}}]}}]},{"metadataParts":[{"text":{"content":"2M"}},{"text":{"content":"10d ago"}}]}]}}}}})) + + SINGLE_AUTHOR_RELATED_LOCKUP = JSON.parse(%q({"contentType":"LOCKUP_CONTENT_TYPE_VIDEO","contentId":"XuoqKYxDHVc","contentImage":{"thumbnailViewModel":{"decoratedAvatarViewModel":{"avatar":{"avatarViewModel":{"size":"AVATAR_SIZE_TYPE_M","image":{"sources":[{"url":"https://yt3.ggpht.com/ytc/example","width":88,"height":88}]}}},"rendererContext":{"commandContext":{"onTap":{"innertubeCommand":{"browseEndpoint":{"browseId":"UC0p5jTq6Xx_DosDFxVXnWaQ"}}}}}},"image":{"sources":[{"url":"https://i.ytimg.com/vi/XuoqKYxDHVc/hqdefault.jpg","width":360,"height":202}]},"overlays":[{"thumbnailBottomOverlayViewModel":{"badges":[{"thumbnailBadgeViewModel":{"text":"12:34"}}]}}]}},"metadata":{"lockupMetadataViewModel":{"title":{"content":"Why Russia Is Winning The Energy War"},"metadata":{"contentMetadataViewModel":{"metadataRows":[{"metadataParts":[{"text":{"content":"The Economist"}}]},{"metadataParts":[{"text":{"content":"1.2K views"}},{"text":{"content":"3w ago"}}]}]}}}}})) + + LEGACY_CVR_ITEM = JSON.parse(%q({"videoId":"aqz-KE-bpKQ","title":{"simpleText":"Big Buck Bunny"},"lengthText":{"simpleText":"9:56"},"shortBylineText":{"runs":[{"text":"Blender","navigationEndpoint":{"browseEndpoint":{"browseId":"UCSMOQeBJ2RAnuFungnYDxuA"}}}]},"ownerText":{"runs":[{"text":"Blender"}]},"shortViewCountText":{"simpleText":"10M views"},"publishedTimeText":{"simpleText":"15 years ago"}})) + + it "parses a collaboration related video with multiple channel links" do + results = wrap_related_lockup([COLLAB_RELATED_LOCKUP]).dig?("secondaryResults", "secondaryResults", "results") + + related = Invidious::Videos::Parser.parse_related_videos(results) + + expect(related.size).to eq(1) + + video = related[0].as_h + expect(video["id"].as_s).to eq("ywd-Ve8a8Tc") + expect(video["ucid"].as_s).to eq("UCzwCEE_PchiBULMnAJqhGVg") + expect(video["author"].as_s).to eq("Raj Shamani and Predictive History") + expect(video["author_verified"].as_s).to eq("true") + expect(video["length_seconds"].as_s).to eq((2*3600 + 31*60 + 1).to_s) + expect(video["short_view_count"].as_s).to eq("2M") + expect(Time.parse_rfc3339(video["published"].as_s)).to be_close(Time.local - 10.days, 1.days) + end + + it "parses a single-author related video (channel link on avatar)" do + results = wrap_related_lockup([SINGLE_AUTHOR_RELATED_LOCKUP]).dig?("secondaryResults", "secondaryResults", "results") + + related = Invidious::Videos::Parser.parse_related_videos(results) + + expect(related.size).to eq(1) + + video = related[0].as_h + expect(video["id"].as_s).to eq("XuoqKYxDHVc") + expect(video["ucid"].as_s).to eq("UC0p5jTq6Xx_DosDFxVXnWaQ") + expect(video["author"].as_s).to eq("The Economist") + expect(video["author_verified"].as_s).to eq("false") + expect(video["length_seconds"].as_s).to eq(754.to_s) + expect(video["short_view_count"].as_s).to eq("1.2K") + expect(Time.parse_rfc3339(video["published"].as_s)).to be_close(Time.local - 21.days, 1.days) + end + + it "ignores non-video lockups" do + playlist_lockup = JSON.parse(%q({"contentType":"LOCKUP_CONTENT_TYPE_PLAYLIST","contentId":"PLtest"})) + + results = wrap_related_lockup([playlist_lockup]).dig?("secondaryResults", "secondaryResults", "results") + + related = Invidious::Videos::Parser.parse_related_videos(results) + + expect(related.size).to eq(0) + end + + it "still parses legacy compactVideoRenderer items" do + results = JSON.parse({ + "secondaryResults": { + "secondaryResults": { + "results": [{"compactVideoRenderer" => LEGACY_CVR_ITEM}], + }, + }, + }.to_json).dig?("secondaryResults", "secondaryResults", "results") + + related = Invidious::Videos::Parser.parse_related_videos(results) + + expect(related.size).to eq(1) + + video = related[0].as_h + expect(video["id"].as_s).to eq("aqz-KE-bpKQ") + expect(video["ucid"].as_s).to eq("UCSMOQeBJ2RAnuFungnYDxuA") + expect(video["length_seconds"].as_s).to eq((9*60 + 56).to_s) + end +end diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 8367fcd78..ea6ace5ac 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -55,6 +55,145 @@ module Invidious::Videos::Parser } end + # Use to parse the "lockupViewModel" items from the watch page's related + # videos section (new format as of 2026-08). Produces the same hash as + # #parse_related_video. + # + # The author's channel link is stored in different places depending on + # whether the video is a collaboration: for regular videos it is attached + # to the avatar, while collaborations carry an avatar-stack dialog with one + # entry per channel. Only the first channel is kept, since the related + # videos hash exposes a single author/ucid pair. + def parse_related_lockup(lockup : JSON::Any) : Hash(String, JSON::Any)? + return nil if !lockup["contentId"]? + return nil if lockup["contentType"]?.try(&.as_s) != "LOCKUP_CONTENT_TYPE_VIDEO" + + metadata = lockup.dig?("metadata", "lockupMetadataViewModel") + return nil unless metadata + + title = metadata.dig?("title", "content").try(&.as_s) || "" + + ucids = [] of String + + if browse = lockup.dig?("contentImage", "thumbnailViewModel", "decoratedAvatarViewModel", + "rendererContext", "commandContext", "onTap", "innertubeCommand", "browseEndpoint") + self.add_channel_browse_id(ucids, browse) + end + + avatar_stack_items = metadata.dig?("image", "avatarStackViewModel", "rendererContext", + "commandContext", "onTap", "innertubeCommand", "showDialogCommand", "panelLoadingStrategy", + "inlineContent", "dialogViewModel", "customContent", "listViewModel", "listItems") + + avatar_stack_items.try &.as_a.each do |item| + if browse = item.dig?("listItemViewModel", "rendererContext", "commandContext", + "onTap", "innertubeCommand", "browseEndpoint") + self.add_channel_browse_id(ucids, browse) + end + end + + author = "" + author_verified = false + short_view_count = "" + published_text = "" + + metadata_rows = metadata.dig?("metadata", "contentMetadataViewModel", "metadataRows") + metadata_rows.try &.as_a.each do |row| + row.dig?("metadataParts").try &.as_a.each do |part| + text = part.dig?("text", "content").try(&.as_s) + next unless text + + if part.dig?("text", "attachmentRuns").try &.as_a.any? { |attachment| + attachment.dig?("element", "type", "imageType", "image", "sources", 0, + "clientResource", "imageName").try(&.as_s) == "CHECK_CIRCLE_FILLED" + } + author_verified = true + end + + if text.matches?(/ago/i) + published_text = text + elsif short_view_count.empty? && text.lstrip.matches?(/\d/) + short_view_count = text.gsub(/\s*views?$/i, "") + else + author = text if author.empty? + end + end + end + + length_text = lockup.dig?("contentImage", "thumbnailViewModel", "overlays", 0, + "thumbnailBottomOverlayViewModel", "badges", 0, "thumbnailBadgeViewModel", "text") + .try(&.as_s) + + length_seconds = length_text ? decode_length_seconds(length_text.not_nil!).to_s : "0" + + published = published_text.empty? ? "" : self.decode_compact_published(published_text) + + return { + "id" => lockup["contentId"], + "title" => JSON::Any.new(title), + "author" => JSON::Any.new(author), + "ucid" => JSON::Any.new(ucids.first? || ""), + "length_seconds" => JSON::Any.new(length_seconds), + "short_view_count" => JSON::Any.new(short_view_count), + "author_verified" => JSON::Any.new(author_verified.to_s), + "published" => JSON::Any.new(published), + } + end + + private def add_channel_browse_id(list : Array(String), browse_endpoint : JSON::Any) + browse_id = browse_endpoint.dig?("browseId").try(&.as_s) + list << browse_id if browse_id && browse_id.starts_with?("UC") && !list.includes?(browse_id) + end + + # Handles the compact relative dates ("10d ago") used by lockups, falling + # back to the regular long forms ("10 days ago"). + private def decode_compact_published(text : String) : String + if match = text.match(/(\d+)\s*([smhdwy])\s+ago/i) + amount = match[1].to_i64 + + time = case match[2].downcase + when "s" then Time.local - amount.seconds + when "m" then Time.local - amount.minutes + when "h" then Time.local - amount.hours + when "d" then Time.local - amount.days + when "w" then Time.local - amount.weeks + when "y" then Time.local - amount.years + else Time.local - amount.days * 30 + end + + return time.to_rfc3339 + end + + return decode_date(text).to_rfc3339 + rescue + return "" + end + + # Parses the related videos section of a watch page. Supports both the + # legacy "compactVideoRenderer" items and the new "lockupViewModel" + # items wrapped in an item section (as of 2026-08). + def parse_related_videos(results : JSON::Any?) : Array(JSON::Any) + related = [] of JSON::Any + + results.try &.as_a.each do |element| + if item = element["compactVideoRenderer"]? + related_video = self.parse_related_video(item) + related << JSON::Any.new(related_video) if related_video + elsif section = element["itemSectionRenderer"]? + section.dig?("contents").try &.as_a.each do |sub_element| + if item = sub_element["lockupViewModel"]? + related_video = self.parse_related_lockup(item) + related << JSON::Any.new(related_video) if related_video + elsif item = sub_element["compactVideoRenderer"]? + related_video = self.parse_related_video(item) + related << JSON::Any.new(related_video) if related_video + end + end + end + end + + return related + end + def extract_video_info(video_id : String) # Fetch data from the player endpoint player_response = YoutubeAPI.player(video_id: video_id) @@ -240,15 +379,9 @@ module Invidious::Videos::Parser related = [] of JSON::Any - # Parse "compactVideoRenderer" items (under secondary results) - secondary_results = main_results - .dig?("secondaryResults", "secondaryResults", "results") - secondary_results.try &.as_a.each do |element| - if item = element["compactVideoRenderer"]? - related_video = self.parse_related_video(item) - related << JSON::Any.new(related_video) if related_video - end - end + related = self.parse_related_videos( + main_results.dig?("secondaryResults", "secondaryResults", "results") + ) # If nothing was found previously, fall back to end screen renderer if related.empty?