Harden /next fallback: fix null subreason, escaping, caching

Review fixes on top of the rebased /next-fallback work:

- Fix TypeCastError when subreason is stored as JSON null: accessors use
  as_s?, and error returns go through a helper that omits nil subreason.
- Restore graceful no-companion path by reverting player() to return nil,
  and drop the now-dead info.nil? branch in fetch_video.
- Re-enable video caching when Invidious Companion is not configured.
- Detect genuinely unavailable videos via playabilityStatus == "ERROR"
  instead of the localized "Video unavailable" string.
- HTML-escape reason/subreason coming from YouTube to avoid HTML injection,
  and guard against a missing "runs" key.
- Tidy published-date fallback and add a regression test.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

spec/invidious/videos/regular_videos_extract_spec.cr

@@ -205,3 +205,19 @@ Spectator.describe "parse_video_info" do
     expect(info["relatedVideos"].as_a.size).to eq(20)
   end
 end
+
+Spectator.describe "Video error accessors" do
+  it "returns nil for a subreason stored as JSON null" do
+    # Some error paths store `subreason` as a JSON `null`; the accessor must
+    # not raise a TypeCastError on it.
+    info = {
+      "reason"    => JSON::Any.new("Video unavailable"),
+      "subreason" => JSON::Any.new(nil),
+    } of String => JSON::Any
+
+    video = Video.new({id: "dQw4w9WgXcQ", info: info, updated: Time.utc})
+
+    expect(video.reason).to eq("Video unavailable")
+    expect(video.subreason).to be_nil
+  end
+end

src/invidious/videos.cr

@@ -179,11 +179,13 @@ struct Video
   end
 
   def reason : String?
-    info["reason"]?.try &.as_s
+    # Use `as_s?` so a JSON `null` (stored by some error paths) yields `nil`
+    # instead of raising a TypeCastError.
+    info["reason"]?.try &.as_s?
   end
 
   def subreason : String?
-    info["subreason"]?.try &.as_s
+    info["subreason"]?.try &.as_s?
   end
 
   def music : Array(VideoMusic)
@@ -310,7 +312,7 @@ def get_video(id, refresh = true, region = nil, force_refresh = false)
        video.schema_version != Video::SCHEMA_VERSION # cache control
       begin
         video = fetch_video(id, region)
-        Invidious::Database::Videos.update(video) if CONFIG.invidious_companion.present?
+        Invidious::Database::Videos.update(video)
       rescue ex
         Invidious::Database::Videos.delete(id)
         raise ex
@@ -318,7 +320,7 @@ def get_video(id, refresh = true, region = nil, force_refresh = false)
     end
   else
     video = fetch_video(id, region)
-    Invidious::Database::Videos.insert(video) if !region && CONFIG.invidious_companion.present?
+    Invidious::Database::Videos.insert(video) if !region
   end
 
   return video
@@ -329,22 +331,10 @@ rescue DB::Error
 end
 
 def fetch_video(id, region)
+  # `extract_video_info` always returns a hash (it raises `NotFoundException`
+  # for genuinely unavailable videos), so there is no nil case to handle here.
   info = Invidious::Videos::Parser.extract_video_info(video_id: id)
 
-  if info.nil?
-    raise InfoException.new("Invidious companion is not available. \
-    Video playback cannot continue. \
-    If you are the administrator of this instance, install Invidious companion \
-    following the installation instructions \
-    <a href=\"https://docs.invidious.io/installation/\">https://docs.invidious.io/installation/</a>")
-  end
-
-  if reason = info["reason"]?
-    if reason == "Video unavailable" && !info["title"]?
-      raise NotFoundException.new(reason.as_s || "")
-    end
-  end
-
   video = Video.new({
     id:      id,
     info:    info,

src/invidious/videos/parser.cr

@@ -55,6 +55,18 @@ module Invidious::Videos::Parser
     }
   end
 
+  # Builds the minimal "error" info hash returned when a video can't be
+  # played but we still want to render the watch page. `subreason` is only
+  # included when present, so it never gets stored as a JSON `null`.
+  private def self.error_video_info(reason : String?, subreason : String?) : Hash(String, JSON::Any)
+    params = {
+      "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),
+    } of String => JSON::Any
+    params["reason"] = JSON::Any.new(reason) if reason
+    params["subreason"] = JSON::Any.new(subreason) if subreason
+    params
+  end
+
   def extract_video_info(video_id : String)
     # Fetch data from the player endpoint
     player_response = nil
@@ -93,18 +105,21 @@ If you are the administrator, install Invidious Companion: \
       reason ||= player_response.dig?("playabilityStatus", "errorScreen", "playerErrorMessageRenderer", "reason", "simpleText").try &.as_s
 
       subreason_main = player_response.dig?("playabilityStatus", "errorScreen", "playerErrorMessageRenderer", "subreason")
-      subreason = subreason_main.try &.[]?("simpleText").try &.as_s
+      subreason = subreason_main.try &.["simpleText"]?.try &.as_s
-      subreason ||= subreason_main.try &.[]("runs").as_a.map(&.[]("text")).join("")
+      subreason ||= subreason_main.try &.["runs"]?.try &.as_a.map(&.["text"]).join("")
 
-      # Stop here if video is not found, or private with no further data.
+      # These strings come straight from YouTube and are rendered as raw HTML
-      # For other statuses (UNPLAYABLE, etc.), continue to extract as much
+      # on the watch/embed pages, so escape them to avoid HTML injection.
-      # data as possible from the /next endpoint.
+      reason = HTML.escape(reason) if reason
-      if reason == "Video unavailable" && playability_status != "UNPLAYABLE"
+      subreason = HTML.escape(subreason) if subreason
-        return {
+
-          "version"   => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),
+      # Stop here if the video is genuinely unavailable (deleted, removed or
-          "reason"    => JSON::Any.new(reason),
+      # nonexistent): YouTube returns status "ERROR" for these regardless of
-          "subreason" => JSON::Any.new(subreason),
+      # the (localized) reason text, and there is nothing to extract from the
-        }
+      # /next endpoint. For other statuses (UNPLAYABLE, LOGIN_REQUIRED,
+      # LIVE_STREAM_OFFLINE, ...) we continue and extract as much as possible.
+      if playability_status == "ERROR"
+        raise NotFoundException.new(reason || "Video unavailable")
       end
     elsif video_id != player_response.dig?("videoDetails", "videoId")
       # YouTube may return a different video player response than expected.
@@ -115,10 +130,7 @@ If you are the administrator, install Invidious Companion: \
       # wrong video means that we should count it as a failure.
       Helpers.get_playback_statistic["totalRequests"] += 1
 
-      return {
+      return error_video_info("Can't load the video on this Invidious instance. YouTube is currently trying to block Invidious instances. <a href=\"https://github.com/iv-org/invidious/issues/3822\">Click here for more info about the issue.</a>", nil)
-        "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),
-        "reason"  => JSON::Any.new("Can't load the video on this Invidious instance. YouTube is currently trying to block Invidious instances. <a href=\"https://github.com/iv-org/invidious/issues/3822\">Click here for more info about the issue.</a>"),
-      }
     else
       reason = nil
     end
@@ -134,11 +146,7 @@ If you are the administrator, install Invidious Companion: \
       # If we're in an error state and /next fails, return what we have
       if reason
         LOGGER.debug("extract_video_info: /next failed for #{video_id}: #{ex.message}")
-        return {
+        return error_video_info(reason, subreason)
-          "version"   => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),
-          "reason"    => JSON::Any.new(reason),
-          "subreason" => JSON::Any.new(subreason),
-        }
       end
       raise ex
     end
@@ -147,11 +155,7 @@ If you are the administrator, install Invidious Companion: \
       params = self.parse_video_info(video_id, player_response)
     rescue ex : BrokenTubeException
       if reason
-        return {
+        return error_video_info(reason, subreason)
-          "version"   => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),
-          "reason"    => JSON::Any.new(reason),
-          "subreason" => JSON::Any.new(subreason),
-        }
       end
       raise ex
     end
@@ -257,15 +261,22 @@ If you are the administrator, install Invidious Companion: \
       .try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) }
 
     if published.nil?
+      # Fall back to the human-readable date shown on the watch page (e.g.
+      # "Started streaming 3 hours ago" or "Premiered Jan 1, 2024").
       published_txt = video_primary_renderer
-        .try &.dig?("dateText", "simpleText")
+        .try &.dig?("dateText", "simpleText").try &.as_s
 
-      if published_txt.try &.as_s.includes?("ago") && !published_txt.nil?
+      date_regex = /(\w{3} \d{1,2}, \d{4})$/
-        published = decode_date(published_txt.as_s.lchop("Started streaming "))
+
-      elsif published_txt && published_txt.try &.as_s.matches?(/(\w{3} \d{1,2}, \d{4})$/)
+      published =
-        published = Time.parse(published_txt.as_s.match!(/(\w{3} \d{1,2}, \d{4})$/)[0], "%b %-d, %Y", Time::Location::UTC)
+        if published_txt.nil?
+          Time.utc
+        elsif published_txt.includes?("ago")
+          decode_date(published_txt.lchop("Started streaming "))
+        elsif match = published_txt.match(date_regex)
+          Time.parse(match[0], "%b %-d, %Y", Time::Location::UTC)
         else
-        published = Time.utc
+          Time.utc
         end
     end
 
@@ -295,12 +306,13 @@ If you are the administrator, install Invidious Companion: \
       .try &.as_a.map &.as_s || [] of String
 
     allow_ratings = video_details["allowRatings"]?.try &.as_bool
+    # When the microformat is missing (e.g. /next-only data) we can't know for
+    # sure, so optimistically assume the video is family safe.
     family_friendly = microformat["isFamilySafe"]?.try &.as_bool
-    if family_friendly.nil?
+    family_friendly = true if family_friendly.nil?
-      family_friendly = true
-    end
     is_listed = video_details["isCrawlable"]?.try &.as_bool
     if is_listed.nil?
+      # Without videoDetails, infer from the "Unlisted" badge if present.
       if video_badges = video_primary_renderer.try &.dig?("badges")
         is_listed = !has_unlisted_badge?(video_badges)
       else

src/invidious/yt_backend/youtube_api.cr

@@ -463,7 +463,9 @@ module YoutubeAPI
     if CONFIG.invidious_companion.present?
       return self._post_invidious_companion("/youtubei/v1/player", data)
     else
-      return self._post_json("/youtubei/v1/player", data, nil)
+      # Without Invidious Companion the player endpoint is unavailable.
+      # Returning nil lets the caller fall back to the /next endpoint.
+      return nil
     end
   end