Rebase onto master: graceful fallback to /next endpoint for major watch page data

- Display watch page even when player endpoint fails, using data from /next
- Show error message in player container placeholder (like YouTube does)
- Extract title, description, views, likes, comments etc. from /next
- Handle missing videoDetails gracefully with fallback to renderer data
- Fix is_listed detection from video badges when videoDetails unavailable
- Add subreason display for better error messages
- Add has_unlisted_badge? helper function
- Bump SCHEMA_VERSION to 4
- Integrate embed page with the same error handling
- Guard against empty streams when video has errors
- Fix JSON::Any.new(nil) crash for shortDescription
- Add locale key: error_from_youtube_unplayable

PR TODO status:
[x] Integrate the embed page with the changes
[x] Display the subreason on the watch page when there is an error
[x] Fix description (nil safety)
[ ] Add mocks for a video without videoDetails

locales/en-US.json

@@ -508,5 +508,6 @@
     "timeline_parse_error_placeholder_heading": "Unable to parse item",
     "timeline_parse_error_placeholder_message": "Invidious encountered an error while trying to parse this item. For more information see below:",
     "timeline_parse_error_show_technical_details": "Show technical details",
-    "dmca_content": "This video cannot be downloaded on this instance due to a DMCA/copyright infringement letter sent to the instance administrator."
+    "dmca_content": "This video cannot be downloaded on this instance due to a DMCA/copyright infringement letter sent to the instance administrator.",
+    "error_from_youtube_unplayable": "Video unplayable due to an error from YouTube:"
 }

src/invidious/routes/watch.cr

@@ -130,7 +130,7 @@ module Invidious::Routes::Watch
     audio_streams = video.audio_streams
 
     # Videos that are a premiere do not have audio streams.
-    if video.premiere_timestamp.nil?
+    if video.premiere_timestamp.nil? && video.reason.nil?
       # Older videos may not have audio sources available.
       # We redirect here so they're not unplayable
       if audio_streams.empty? && !video.live_now
@@ -162,7 +162,7 @@ module Invidious::Routes::Watch
 
     thumbnail = "/vi/#{video.id}/maxres.jpg"
 
-    if params.raw
+    if params.raw && video.reason.nil?
       if params.listen
         url = audio_streams[0]["url"].as_s
 

src/invidious/videos.cr

@@ -15,7 +15,7 @@ struct Video
   # NOTE: don't forget to bump this number if any change is made to
   # the `params` structure in videos/parser.cr!!!
   #
-  SCHEMA_VERSION = 3
+  SCHEMA_VERSION = 4
 
   property id : String
 
@@ -182,6 +182,10 @@ struct Video
     info["reason"]?.try &.as_s
   end
 
+  def subreason : String?
+    info["subreason"]?.try &.as_s
+  end
+
   def music : Array(VideoMusic)
     info["music"].as_a.map { |music_json|
       VideoMusic.new(
@@ -314,7 +318,7 @@ def get_video(id, refresh = true, region = nil, force_refresh = false)
     end
   else
     video = fetch_video(id, region)
-    Invidious::Database::Videos.insert(video) if !region
+    Invidious::Database::Videos.insert(video) if !region && video.reason.nil?
   end
 
   return video
@@ -336,12 +340,8 @@ def fetch_video(id, region)
   end
 
   if reason = info["reason"]?
-    if reason == "Video unavailable"
+    if reason == "Video unavailable" && !info["title"]?
       raise NotFoundException.new(reason.as_s || "")
-    elsif !reason.as_s.starts_with? "Premieres"
-      # dont error when it's a premiere.
-      # we already parsed most of the data and display the premiere date
-      raise InfoException.new(reason.as_s || "")
     end
   end
 

src/invidious/videos/parser.cr

@@ -66,18 +66,21 @@ module Invidious::Videos::Parser
     playability_status = player_response.dig?("playabilityStatus", "status").try &.as_s
 
     if playability_status != "OK"
-      subreason = player_response.dig?("playabilityStatus", "errorScreen", "playerErrorMessageRenderer", "subreason")
+      reason = player_response.dig?("playabilityStatus", "reason").try &.as_s
-      reason = subreason.try &.[]?("simpleText").try &.as_s
+      reason ||= player_response.dig?("playabilityStatus", "errorScreen", "playerErrorMessageRenderer", "reason", "simpleText").try &.as_s
-      reason ||= subreason.try &.[]("runs").as_a.map(&.[]("text")).join("")
-      reason ||= player_response.dig("playabilityStatus", "reason").as_s
 
-      # Stop here if video is not a scheduled livestream or
+      subreason_main = player_response.dig?("playabilityStatus", "errorScreen", "playerErrorMessageRenderer", "subreason")
-      # for LOGIN_REQUIRED when videoDetails element is not found because retrying won't help
+      subreason = subreason_main.try &.[]?("simpleText").try &.as_s
-      if !{"LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status) ||
+      subreason ||= subreason_main.try &.[]("runs").as_a.map(&.[]("text")).join("")
-         playability_status == "LOGIN_REQUIRED" && !player_response.dig?("videoDetails")
+
+      # Stop here if video is not found, or private with no further data.
+      # For other statuses (UNPLAYABLE, etc.), continue to extract as much
+      # data as possible from the /next endpoint.
+      if reason == "Video unavailable" && !{"UNPLAYABLE"}.any?(playability_status)
         return {
-          "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),
+          "version"   => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),
-          "reason"  => JSON::Any.new(reason),
+          "reason"    => JSON::Any.new(reason),
+          "subreason" => JSON::Any.new(subreason),
         }
       end
     elsif video_id != player_response.dig?("videoDetails", "videoId")
@@ -97,17 +100,40 @@ module Invidious::Videos::Parser
       reason = nil
     end
 
-    # Don't fetch the next endpoint if the video is unavailable.
+    # Fetch the /next endpoint for additional data.
-    if {"OK", "LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status)
+    begin
-      next_response = YoutubeAPI.next({"videoId": video_id, "params": ""})
+        next_response = YoutubeAPI.next({"videoId": video_id, "params": ""})
-      # Remove the microformat returned by the /next endpoint on some videos
+        # Remove the microformat returned by the /next endpoint on some videos
-      # to prevent player_response microformat from being overwritten.
+        # to prevent player_response microformat from being overwritten.
-      next_response.delete("microformat")
+        next_response.delete("microformat")
-      player_response = player_response.merge(next_response)
+        player_response = player_response.merge(next_response)
-    end
+      rescue ex
+        # If we're in an error state and /next fails, return what we have
+        if reason
+          LOGGER.debug("extract_video_info: /next failed for #{video_id}: #{ex.message}")
+          return {
+            "version"   => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),
+            "reason"    => JSON::Any.new(reason),
+            "subreason" => JSON::Any.new(subreason),
+          }
+        end
+        raise ex
+      end
 
-    params = self.parse_video_info(video_id, player_response)
+    begin
+      params = self.parse_video_info(video_id, player_response)
+    rescue ex : BrokenTubeException
+      if reason
+        return {
+          "version"   => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),
+          "reason"    => JSON::Any.new(reason),
+          "subreason" => JSON::Any.new(subreason),
+        }
+      end
+      raise ex
+    end
     params["reason"] = JSON::Any.new(reason) if reason
+    params["subreason"] = JSON::Any.new(subreason) if subreason
 
     {"captions", "playabilityStatus", "playerConfig", "storyboards"}.each do |f|
       params[f] = player_response[f] if player_response[f]?
@@ -177,16 +203,20 @@ module Invidious::Videos::Parser
     end
 
     video_details = player_response.dig?("videoDetails")
+    video_details ||= {} of String => JSON::Any
     if !(microformat = player_response.dig?("microformat", "playerMicroformatRenderer"))
       microformat = {} of String => JSON::Any
     end
 
-    raise BrokenTubeException.new("videoDetails") if !video_details
-
     # Basic video infos
 
     title = video_details["title"]?.try &.as_s
 
+    title ||= extract_text(
+      video_primary_renderer
+        .try &.dig?("title")
+    )
+
     # We have to try to extract viewCount from videoPrimaryInfoRenderer first,
     # then from videoDetails, as the latter is "0" for livestreams (we want
     # to get the amount of viewers watching).
@@ -197,11 +227,24 @@ module Invidious::Videos::Parser
     views_txt ||= video_details["viewCount"]?.try &.as_s || ""
     views = views_txt.gsub(/\D/, "").to_i64?
 
-    length_txt = (microformat["lengthSeconds"]? || video_details["lengthSeconds"])
+    length_txt = (microformat["lengthSeconds"]? || video_details["lengthSeconds"]?)
       .try &.as_s.to_i64
 
     published = microformat["publishDate"]?
-      .try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc
+      .try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) }
+
+    if published.nil?
+      published_txt = video_primary_renderer
+        .try &.dig?("dateText", "simpleText")
+
+      if published_txt.try &.as_s.includes?("ago") && !published_txt.nil?
+        published = decode_date(published_txt.as_s.lchop("Started streaming "))
+      elsif published_txt && published_txt.try &.as_s.matches?(/(\w{3} \d{1,2}, \d{4})$/)
+        published = Time.parse(published_txt.as_s.match!(/(\w{3} \d{1,2}, \d{4})$/)[0], "%b %-d, %Y", Time::Location::UTC)
+      else
+        published = Time.utc
+      end
+    end
 
     premiere_timestamp = microformat.dig?("liveBroadcastDetails", "startTimestamp")
       .try { |t| Time.parse_rfc3339(t.as_s) }
@@ -228,7 +271,17 @@ module Invidious::Videos::Parser
 
     allow_ratings = video_details["allowRatings"]?.try &.as_bool
     family_friendly = microformat["isFamilySafe"]?.try &.as_bool
+    if family_friendly.nil?
+      family_friendly = true
+    end
     is_listed = video_details["isCrawlable"]?.try &.as_bool
+    if is_listed.nil?
+      if video_badges = video_primary_renderer.try &.dig?("badges")
+        is_listed = !has_unlisted_badge?(video_badges)
+      else
+        is_listed = true
+      end
+    end
     is_upcoming = video_details["isUpcoming"]?.try &.as_bool
 
     keywords = video_details["keywords"]?
@@ -428,7 +481,7 @@ module Invidious::Videos::Parser
       # Description
       "description"      => JSON::Any.new(description || ""),
       "descriptionHtml"  => JSON::Any.new(description_html || "<p></p>"),
-      "shortDescription" => JSON::Any.new(short_description.try &.as_s || nil),
+      "shortDescription" => JSON::Any.new(short_description.try &.as_s || ""),
       # Video metadata
       "genre"     => JSON::Any.new(genre.try &.as_s || ""),
       "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s?),

src/invidious/views/embed.ecr

@@ -31,7 +31,19 @@
 %>
 </script>
 
+<% if video.reason.nil? %>
 <%= rendered "components/player" %>
+<% else %>
+<div id="player-container" class="h-box">
+    <div style="display:flex;align-items:center;justify-content:center;background-color:#000;width:100%;aspect-ratio:16/9">
+        <p style="color:#fff;font-size:1.2em;text-align:center">
+            <%= I18n.translate(preferences.locale, "error_from_youtube_unplayable") %><br/>
+            <strong><%= video.reason %></strong>
+            <% if video.subreason %><br/><%= video.subreason %><% end %>
+        </p>
+    </div>
+</div>
+<% end %>
 <script src="/js/embed.js?v=<%= ASSET_COMMIT %>"></script>
 </body>
 </html>

src/invidious/views/watch.ecr

@@ -71,9 +71,21 @@ we're going to need to do it here in order to allow for translations.
 %>
 </script>
 
+<% if video.reason.nil? %>
 <div id="player-container" class="h-box">
     <%= rendered "components/player" %>
 </div>
+<% else %>
+<div id="player-container" class="h-box">
+    <div style="display:flex;align-items:center;justify-content:center;background-color:#000;width:100%;aspect-ratio:16/9">
+        <p style="color:#fff;font-size:1.2em;text-align:center">
+            <%= I18n.translate(locale, "error_from_youtube_unplayable") %><br/>
+            <strong><%= video.reason %></strong>
+            <% if video.subreason %><br/><%= video.subreason %><% end %>
+        </p>
+    </div>
+</div>
+<% end %>
 
 <div class="h-box">
     <h1>
@@ -97,8 +109,12 @@ we're going to need to do it here in order to allow for translations.
 
     <% if video.reason %>
         <h3>
-            <%= video.reason %>
+            <%= I18n.translate(locale, "error_from_youtube_unplayable") %> <%= video.reason %>
         </h3>
+        <% if video.subreason %>
+        <p><%= video.subreason %></p>
+        <% end %>
+        <%= error_redirect_helper(env) %>
     <% elsif video.premiere_timestamp.try &.> Time.utc %>
         <h3>
             <%= video.premiere_timestamp.try { |t| I18n.translate(locale, "Premieres in `x`", recode_date((t - Time.utc).ago, locale)) } %>

src/invidious/yt_backend/extractors_utils.cr

@@ -68,6 +68,23 @@ rescue ex
   return false
 end
 
+def has_unlisted_badge?(badges : JSON::Any?)
+  return false if badges.nil?
+
+  badges.as_a.each do |badge|
+    icon_type = badge.dig("metadataBadgeRenderer", "icon", "iconType").as_s
+
+    return true if icon_type == "PRIVACY_UNLISTED"
+  end
+
+  return false
+rescue ex
+  LOGGER.debug("Unable to parse badges for unlisted check. Got exception: #{ex.message}")
+  LOGGER.trace("Badges data: #{badges.to_json}")
+
+  return false
+end
+
 # This function extracts SearchVideo items from a Category.
 # Categories are commonly returned in search results and trending pages.
 def extract_category(category : Category) : Array(SearchVideo)