From 8ed07a58d43ddc9299de13e915b4672521648c62 Mon Sep 17 00:00:00 2001 From: naoNao89 <90588855+naoNao89@users.noreply.github.com> Date: Mon, 18 Aug 2025 22:58:05 +0700 Subject: [PATCH] [Videos] Patch missing streamingData sections from fallback (formats + adaptive) - YouTube increasingly serves SABR/modified DASH. When the primary player response lacks usable stream URLs, Invidious falls back to alternate clients. Historically only adaptiveFormats were patched in fallback. This change patches whichever sections are actually missing (adaptiveFormats and/or formats), but only when the fallback contains usable entries (url or signatureCipher/cipher), avoiding overwriting valid primary data. - Adds a debug log (fallback_patched) to indicate which client patched which sections and a playback_404 triage log with stream counts to ease diagnosis. - Fixes #5420 --- spec/streaming_fallback_spec.cr | 94 ++++++++++++++++++++++++++ src/invidious/routes/video_playback.cr | 2 + src/invidious/videos/parser.cr | 73 ++++++++++++++++++-- 3 files changed, 164 insertions(+), 5 deletions(-) create mode 100644 spec/streaming_fallback_spec.cr diff --git a/spec/streaming_fallback_spec.cr b/spec/streaming_fallback_spec.cr new file mode 100644 index 00000000..1bf144a4 --- /dev/null +++ b/spec/streaming_fallback_spec.cr @@ -0,0 +1,94 @@ +require "spectator" + +# Bring in the helper under test +require "../src/invidious/videos/parser.cr" + +Spectator.describe Invidious::Videos::ParserHelpers do + def json_any_hash(h : Hash(String, JSON::Any)) + h + end + + def json_any_array(a : Array(JSON::Any)) + JSON::Any.new(a) + end + + def json_any_str(s : String) + JSON::Any.new(s) + end + + def json_any_obj(h : Hash(String, JSON::Any)) + JSON::Any.new(h) + end + + it "patches formats when primary missing and fallback has usable formats" do + primary_sd = { + "formats" => JSON::Any.new([] of JSON::Any), + "adaptiveFormats" => JSON::Any.new([] of JSON::Any), + } of String => JSON::Any + + fallback_sd = { + "formats" => JSON::Any.new([ + JSON::Any.new({"url" => json_any_str("https://example.com/video.mp4")}), + ] of JSON::Any), + "adaptiveFormats" => JSON::Any.new([ + JSON::Any.new({"url" => json_any_str("https://example.com/audio.m4a")}), + ] of JSON::Any), + } of String => JSON::Any + + res = Invidious::Videos::ParserHelpers.patch_streaming_data_if_missing!(primary_sd, fallback_sd) + + expect(res[:patched_formats]).to be_true + expect(res[:patched_adaptive]).to be_true + + # Ensure formats now have a non-empty URL + first_fmt = primary_sd["formats"].as_a[0].as_h + expect(first_fmt["url"].as_s).to_not be_empty + end + + it "does not overwrite valid primary data" do + primary_sd = { + "formats" => JSON::Any.new([ + JSON::Any.new({"url" => json_any_str("https://primary/video.mp4")}), + ] of JSON::Any), + "adaptiveFormats" => JSON::Any.new([ + JSON::Any.new({"url" => json_any_str("https://primary/audio.m4a")}), + ] of JSON::Any), + } of String => JSON::Any + + fallback_sd = { + "formats" => JSON::Any.new([ + JSON::Any.new({"url" => json_any_str("https://fallback/video.mp4")}), + ] of JSON::Any), + "adaptiveFormats" => JSON::Any.new([ + JSON::Any.new({"url" => json_any_str("https://fallback/audio.m4a")}), + ] of JSON::Any), + } of String => JSON::Any + + res = Invidious::Videos::ParserHelpers.patch_streaming_data_if_missing!(primary_sd, fallback_sd) + + expect(res[:patched_formats]).to be_false + expect(res[:patched_adaptive]).to be_false + + # Primary values should remain + expect(primary_sd["formats"].as_a[0].as_h["url"].as_s).to eq("https://primary/video.mp4") + expect(primary_sd["adaptiveFormats"].as_a[0].as_h["url"].as_s).to eq("https://primary/audio.m4a") + end + + it "handles fallback without formats gracefully" do + primary_sd = { + "formats" => JSON::Any.new([] of JSON::Any), + "adaptiveFormats" => JSON::Any.new([] of JSON::Any), + } of String => JSON::Any + + fallback_sd = { + "adaptiveFormats" => JSON::Any.new([ + JSON::Any.new({"url" => json_any_str("https://example.com/audio.m4a")}), + ] of JSON::Any), + } of String => JSON::Any + + res = Invidious::Videos::ParserHelpers.patch_streaming_data_if_missing!(primary_sd, fallback_sd) + + expect(res[:patched_adaptive]).to be_true + expect(res[:patched_formats]).to be_false + end +end diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index 083087a9..4bb7c892 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -299,6 +299,8 @@ module Invidious::Routes::VideoPlayback url = fmt.try &.["url"]?.try &.as_s if !url + # Extra context for debugging playback errors + LOGGER.warn("playback_404: no URL for id=#{id} itag=#{itag.inspect} fmt=#{video.fmt_stream.size} adaptive=#{video.adaptive_fmts.size}") haltf env, status_code: 404 end diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index feb58440..8e844471 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -58,6 +58,59 @@ def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)? } end +# Small helper utilities extracted for testability and clarity (per LLVM policy) +module Invidious::Videos::ParserHelpers + extend self + + # True if any entry has a usable URL or signatureCipher/cipher that can be converted to a URL + def has_usable_stream?(container : JSON::Any?) : Bool + return false unless container + arr = container.as_a? + return false unless arr && arr.size > 0 + arr.each do |entry| + obj = entry.as_h? + next unless obj + # Accept direct URL + if (u = obj["url"]?.try &.as_s?) && !u.empty? + return true + end + # Accept ciphered URL that convert_url can handle + if (sc = obj["signatureCipher"]?.try &.as_s?) && !sc.empty? + return true + end + if (c = obj["cipher"]?.try &.as_s?) && !c.empty? + return true + end + end + false + end + + # Mutates streaming_data by patching only the sections missing a usable URL + # Returns flags indicating what was patched. + def patch_streaming_data_if_missing!(streaming_data : Hash(String, JSON::Any), fallback_sd : Hash(String, JSON::Any)) : NamedTuple(patched_formats: Bool, patched_adaptive: Bool) + patched_formats = false + patched_adaptive = false + + # Adaptive formats + unless has_usable_stream?(streaming_data["adaptiveFormats"]?) + if has_usable_stream?(fallback_sd["adaptiveFormats"]?) + streaming_data["adaptiveFormats"] = fallback_sd["adaptiveFormats"] + patched_adaptive = true + end + end + + # Progressive formats + unless has_usable_stream?(streaming_data["formats"]?) + if has_usable_stream?(fallback_sd["formats"]?) + streaming_data["formats"] = fallback_sd["formats"] + patched_formats = true + end + end + + {patched_formats: patched_formats, patched_adaptive: patched_adaptive} + end +end + def extract_video_info(video_id : String) # Init client config for the API client_config = YoutubeAPI::ClientConfig.new @@ -109,8 +162,14 @@ def extract_video_info(video_id : String) params["reason"] = JSON::Any.new(reason) if reason if !CONFIG.invidious_companion.present? - if player_response.dig?("streamingData", "adaptiveFormats", 0, "url").nil? - LOGGER.warn("Missing URLs for adaptive formats, falling back to other YT clients.") + # Fix for issue #5420: /api/v1/videos/ endpoint has formatStreams blank + # Determine if we are missing URLs for either adaptive or progressive formats + need_adaptive = player_response.dig?("streamingData", "adaptiveFormats", 0, "url").nil? + need_formats = player_response.dig?("streamingData", "formats", 0, "url").nil? + + if need_adaptive || need_formats + missing = [need_adaptive ? "adaptiveFormats" : nil, need_formats ? "formats" : nil].compact.join(", ") + LOGGER.warn("Missing URLs for #{missing}, falling back to other YT clients.") players_fallback = {YoutubeAPI::ClientType::TvHtml5, YoutubeAPI::ClientType::WebMobile} players_fallback.each do |player_fallback| @@ -118,9 +177,13 @@ def extract_video_info(video_id : String) next if !(player_fallback_response = try_fetch_streaming_data(video_id, client_config)) - if player_fallback_response.dig?("streamingData", "adaptiveFormats", 0, "url") - streaming_data = player_response["streamingData"].as_h - streaming_data["adaptiveFormats"] = player_fallback_response["streamingData"]["adaptiveFormats"] + # Only patch what is actually missing to avoid downgrading good data + streaming_data = player_response["streamingData"].as_h + fallback_sd = player_fallback_response["streamingData"].as_h + patched = Invidious::Videos::ParserHelpers.patch_streaming_data_if_missing!(streaming_data, fallback_sd) + + if patched[:patched_adaptive] || patched[:patched_formats] + LOGGER.debug("fallback_patched: client=#{player_fallback} video=#{video_id} patched_adaptive=#{patched[:patched_adaptive]} patched_formats=#{patched[:patched_formats]}") player_response["streamingData"] = JSON::Any.new(streaming_data) break end