[Videos] Patch missing streamingData sections from fallback (formats + adaptive)

- YouTube increasingly serves SABR/modified DASH. When the primary player response lacks usable stream URLs, Invidious falls back to alternate clients. Historically only adaptiveFormats were patched in fallback. This change patches whichever sections are actually missing (adaptiveFormats and/or formats), but only when the fallback contains usable entries (url or signatureCipher/cipher), avoiding overwriting valid primary data.
- Adds a debug log (fallback_patched) to indicate which client patched which sections and a playback_404 triage log with stream counts to ease diagnosis.
- Fixes #5420

spec/streaming_fallback_spec.cr

@@ -0,0 +1,94 @@
+require "spectator"
+
+# Bring in the helper under test
+require "../src/invidious/videos/parser.cr"
+
+Spectator.describe Invidious::Videos::ParserHelpers do
+  def json_any_hash(h : Hash(String, JSON::Any))
+    h
+  end
+
+  def json_any_array(a : Array(JSON::Any))
+    JSON::Any.new(a)
+  end
+
+  def json_any_str(s : String)
+    JSON::Any.new(s)
+  end
+
+  def json_any_obj(h : Hash(String, JSON::Any))
+    JSON::Any.new(h)
+  end
+
+  it "patches formats when primary missing and fallback has usable formats" do
+    primary_sd = {
+      "formats"         => JSON::Any.new([] of JSON::Any),
+      "adaptiveFormats" => JSON::Any.new([] of JSON::Any),
+    } of String => JSON::Any
+
+    fallback_sd = {
+      "formats" => JSON::Any.new([
+        JSON::Any.new({"url" => json_any_str("https://example.com/video.mp4")}),
+      ] of JSON::Any),
+      "adaptiveFormats" => JSON::Any.new([
+        JSON::Any.new({"url" => json_any_str("https://example.com/audio.m4a")}),
+      ] of JSON::Any),
+    } of String => JSON::Any
+
+    res = Invidious::Videos::ParserHelpers.patch_streaming_data_if_missing!(primary_sd, fallback_sd)
+
+    expect(res[:patched_formats]).to be_true
+    expect(res[:patched_adaptive]).to be_true
+
+    # Ensure formats now have a non-empty URL
+    first_fmt = primary_sd["formats"].as_a[0].as_h
+    expect(first_fmt["url"].as_s).to_not be_empty
+  end
+
+  it "does not overwrite valid primary data" do
+    primary_sd = {
+      "formats" => JSON::Any.new([
+        JSON::Any.new({"url" => json_any_str("https://primary/video.mp4")}),
+      ] of JSON::Any),
+      "adaptiveFormats" => JSON::Any.new([
+        JSON::Any.new({"url" => json_any_str("https://primary/audio.m4a")}),
+      ] of JSON::Any),
+    } of String => JSON::Any
+
+    fallback_sd = {
+      "formats" => JSON::Any.new([
+        JSON::Any.new({"url" => json_any_str("https://fallback/video.mp4")}),
+      ] of JSON::Any),
+      "adaptiveFormats" => JSON::Any.new([
+        JSON::Any.new({"url" => json_any_str("https://fallback/audio.m4a")}),
+      ] of JSON::Any),
+    } of String => JSON::Any
+
+    res = Invidious::Videos::ParserHelpers.patch_streaming_data_if_missing!(primary_sd, fallback_sd)
+
+    expect(res[:patched_formats]).to be_false
+    expect(res[:patched_adaptive]).to be_false
+
+    # Primary values should remain
+    expect(primary_sd["formats"].as_a[0].as_h["url"].as_s).to eq("https://primary/video.mp4")
+    expect(primary_sd["adaptiveFormats"].as_a[0].as_h["url"].as_s).to eq("https://primary/audio.m4a")
+  end
+
+  it "handles fallback without formats gracefully" do
+    primary_sd = {
+      "formats"         => JSON::Any.new([] of JSON::Any),
+      "adaptiveFormats" => JSON::Any.new([] of JSON::Any),
+    } of String => JSON::Any
+
+    fallback_sd = {
+      "adaptiveFormats" => JSON::Any.new([
+        JSON::Any.new({"url" => json_any_str("https://example.com/audio.m4a")}),
+      ] of JSON::Any),
+    } of String => JSON::Any
+
+    res = Invidious::Videos::ParserHelpers.patch_streaming_data_if_missing!(primary_sd, fallback_sd)
+
+    expect(res[:patched_adaptive]).to be_true
+    expect(res[:patched_formats]).to be_false
+  end
+end

src/invidious/routes/video_playback.cr

@@ -299,6 +299,8 @@ module Invidious::Routes::VideoPlayback
     url = fmt.try &.["url"]?.try &.as_s
 
     if !url
+      # Extra context for debugging playback errors
+      LOGGER.warn("playback_404: no URL for id=#{id} itag=#{itag.inspect} fmt=#{video.fmt_stream.size} adaptive=#{video.adaptive_fmts.size}")
       haltf env, status_code: 404
     end
 

src/invidious/videos/parser.cr

@@ -58,6 +58,59 @@ def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)?
   }
 end
 
+# Small helper utilities extracted for testability and clarity (per LLVM policy)
+module Invidious::Videos::ParserHelpers
+  extend self
+
+  # True if any entry has a usable URL or signatureCipher/cipher that can be converted to a URL
+  def has_usable_stream?(container : JSON::Any?) : Bool
+    return false unless container
+    arr = container.as_a?
+    return false unless arr && arr.size > 0
+    arr.each do |entry|
+      obj = entry.as_h?
+      next unless obj
+      # Accept direct URL
+      if (u = obj["url"]?.try &.as_s?) && !u.empty?
+        return true
+      end
+      # Accept ciphered URL that convert_url can handle
+      if (sc = obj["signatureCipher"]?.try &.as_s?) && !sc.empty?
+        return true
+      end
+      if (c = obj["cipher"]?.try &.as_s?) && !c.empty?
+        return true
+      end
+    end
+    false
+  end
+
+  # Mutates streaming_data by patching only the sections missing a usable URL
+  # Returns flags indicating what was patched.
+  def patch_streaming_data_if_missing!(streaming_data : Hash(String, JSON::Any), fallback_sd : Hash(String, JSON::Any)) : NamedTuple(patched_formats: Bool, patched_adaptive: Bool)
+    patched_formats = false
+    patched_adaptive = false
+
+    # Adaptive formats
+    unless has_usable_stream?(streaming_data["adaptiveFormats"]?)
+      if has_usable_stream?(fallback_sd["adaptiveFormats"]?)
+        streaming_data["adaptiveFormats"] = fallback_sd["adaptiveFormats"]
+        patched_adaptive = true
+      end
+    end
+
+    # Progressive formats
+    unless has_usable_stream?(streaming_data["formats"]?)
+      if has_usable_stream?(fallback_sd["formats"]?)
+        streaming_data["formats"] = fallback_sd["formats"]
+        patched_formats = true
+      end
+    end
+
+    {patched_formats: patched_formats, patched_adaptive: patched_adaptive}
+  end
+end
+
 def extract_video_info(video_id : String)
   # Init client config for the API
   client_config = YoutubeAPI::ClientConfig.new
@@ -109,8 +162,14 @@ def extract_video_info(video_id : String)
   params["reason"] = JSON::Any.new(reason) if reason
 
   if !CONFIG.invidious_companion.present?
-    if player_response.dig?("streamingData", "adaptiveFormats", 0, "url").nil?
-      LOGGER.warn("Missing URLs for adaptive formats, falling back to other YT clients.")
+    # Fix for issue #5420: /api/v1/videos/<video_id> endpoint has formatStreams blank
+    # Determine if we are missing URLs for either adaptive or progressive formats
+    need_adaptive = player_response.dig?("streamingData", "adaptiveFormats", 0, "url").nil?
+    need_formats = player_response.dig?("streamingData", "formats", 0, "url").nil?
+
+    if need_adaptive || need_formats
+      missing = [need_adaptive ? "adaptiveFormats" : nil, need_formats ? "formats" : nil].compact.join(", ")
+      LOGGER.warn("Missing URLs for #{missing}, falling back to other YT clients.")
       players_fallback = {YoutubeAPI::ClientType::TvHtml5, YoutubeAPI::ClientType::WebMobile}
 
       players_fallback.each do |player_fallback|
@@ -118,9 +177,13 @@ def extract_video_info(video_id : String)
 
         next if !(player_fallback_response = try_fetch_streaming_data(video_id, client_config))
 
-        if player_fallback_response.dig?("streamingData", "adaptiveFormats", 0, "url")
-          streaming_data = player_response["streamingData"].as_h
-          streaming_data["adaptiveFormats"] = player_fallback_response["streamingData"]["adaptiveFormats"]
+        # Only patch what is actually missing to avoid downgrading good data
+        streaming_data = player_response["streamingData"].as_h
+        fallback_sd = player_fallback_response["streamingData"].as_h
+        patched = Invidious::Videos::ParserHelpers.patch_streaming_data_if_missing!(streaming_data, fallback_sd)
+
+        if patched[:patched_adaptive] || patched[:patched_formats]
+          LOGGER.debug("fallback_patched: client=#{player_fallback} video=#{video_id} patched_adaptive=#{patched[:patched_adaptive]} patched_formats=#{patched[:patched_formats]}")
           player_response["streamingData"] = JSON::Any.new(streaming_data)
           break
         end