Merge 8864b41b0f4d19b3c90c049954033f3f4e54237e into 9c62c8b6603d8cecde61a446627a408e7fd57969

locales/en-US.json

@@ -508,5 +508,6 @@
     "timeline_parse_error_placeholder_heading": "Unable to parse item",
     "timeline_parse_error_placeholder_message": "Invidious encountered an error while trying to parse this item. For more information see below:",
     "timeline_parse_error_show_technical_details": "Show technical details",
-    "dmca_content": "This video cannot be downloaded on this instance due to a DMCA/copyright infringement letter sent to the instance administrator."
+    "dmca_content": "This video cannot be downloaded on this instance due to a DMCA/copyright infringement letter sent to the instance administrator.",
+    "error_from_youtube_unplayable": "Video unplayable due to an error from YouTube:"
 }

mocks

@@ -1 +1 @@
-Subproject commit b55d58dea94f7144ff0205857dfa70ec14eaa872
+Subproject commit 562192243e0348bd53bb85a5bf6d404109b21da0

spec/invidious/videos/regular_videos_extract_spec.cr

@@ -163,4 +163,61 @@ Spectator.describe "parse_video_info" do
     expect(info["authorVerified"].as_bool).to be_false
     expect(info["subCountText"].as_s).to eq("3.11K")
   end
+
+  it "parses a video without videoDetails (fallback to /next data)" do
+    _player = load_mock("video/no-videodetails.player")
+    _next = load_mock("video/regular_no-description.next")
+
+    raw_data = _player.merge!(_next)
+    info = Invidious::Videos::Parser.parse_video_info("no-videodetails-id", raw_data)
+
+    expect(typeof(info)).to eq(Hash(String, JSON::Any))
+
+    # Title falls back to videoPrimaryInfoRenderer
+    expect(info["title"].as_s).to eq("Chris Rea - Auberge")
+
+    # Views from videoPrimaryInfoRenderer
+    expect(info["views"].as_i).to eq(14_324_584)
+
+    # Likes still work (from videoPrimaryInfoRenderer buttons)
+    expect(info["likes"].as_i).to eq(35_870)
+
+    # Length from microformat
+    expect(info["lengthSeconds"].as_i).to eq(283_i64)
+
+    # Published from microformat, or dateText fallback
+    expect(info["published"].as_s).to eq("2012-05-21T00:00:00Z")
+
+    # Author from videoOwnerRenderer
+    expect(info["author"].as_s).to eq("ChrisReaVideos")
+    expect(info["ucid"].as_s).to eq("UC_5q6nWPbD30-y6oiWF_oNA")
+
+    # Family friendly defaults to true when missing
+    expect(info["isFamilyFriendly"].as_bool).to be_true
+
+    # isListed defaults to true when no badges
+    expect(info["isListed"].as_bool).to be_true
+
+    # Description
+    expect(info["descriptionHtml"].as_s).to eq("")
+
+    # Related videos still work from /next
+    expect(info["relatedVideos"].as_a.size).to eq(20)
+  end
+end
+
+Spectator.describe "Video error accessors" do
+  it "returns nil for a subreason stored as JSON null" do
+    # Some error paths store `subreason` as a JSON `null`; the accessor must
+    # not raise a TypeCastError on it.
+    info = {
+      "reason"    => JSON::Any.new("Video unavailable"),
+      "subreason" => JSON::Any.new(nil),
+    } of String => JSON::Any
+
+    video = Video.new({id: "dQw4w9WgXcQ", info: info, updated: Time.utc})
+
+    expect(video.reason).to eq("Video unavailable")
+    expect(video.subreason).to be_nil
+  end
 end

src/invidious/routes/watch.cr

@@ -130,7 +130,7 @@ module Invidious::Routes::Watch
     audio_streams = video.audio_streams
 
     # Videos that are a premiere do not have audio streams.
-    if video.premiere_timestamp.nil?
+    if video.premiere_timestamp.nil? && video.reason.nil?
       # Older videos may not have audio sources available.
       # We redirect here so they're not unplayable
       if audio_streams.empty? && !video.live_now
@@ -162,7 +162,7 @@ module Invidious::Routes::Watch
 
     thumbnail = "/vi/#{video.id}/maxres.jpg"
 
-    if params.raw
+    if params.raw && video.reason.nil?
       if params.listen
         url = audio_streams[0]["url"].as_s
 

src/invidious/videos.cr

@@ -15,7 +15,7 @@ struct Video
   # NOTE: don't forget to bump this number if any change is made to
   # the `params` structure in videos/parser.cr!!!
   #
-  SCHEMA_VERSION = 3
+  SCHEMA_VERSION = 4
 
   property id : String
 
@@ -179,7 +179,13 @@ struct Video
   end
 
   def reason : String?
-    info["reason"]?.try &.as_s
+    # Use `as_s?` so a JSON `null` (stored by some error paths) yields `nil`
+    # instead of raising a TypeCastError.
+    info["reason"]?.try &.as_s?
+  end
+
+  def subreason : String?
+    info["subreason"]?.try &.as_s?
   end
 
   def music : Array(VideoMusic)
@@ -325,26 +331,10 @@ rescue DB::Error
 end
 
 def fetch_video(id, region)
+  # `extract_video_info` always returns a hash (it raises `NotFoundException`
+  # for genuinely unavailable videos), so there is no nil case to handle here.
   info = Invidious::Videos::Parser.extract_video_info(video_id: id)
 
-  if info.nil?
-    raise InfoException.new("Invidious companion is not available. \
-    Video playback cannot continue. \
-    If you are the administrator of this instance, install Invidious companion \
-    following the installation instructions \
-    <a href=\"https://docs.invidious.io/installation/\">https://docs.invidious.io/installation/</a>")
-  end
-
-  if reason = info["reason"]?
-    if reason == "Video unavailable"
-      raise NotFoundException.new(reason.as_s || "")
-    elsif !reason.as_s.starts_with? "Premieres"
-      # dont error when it's a premiere.
-      # we already parsed most of the data and display the premiere date
-      raise InfoException.new(reason.as_s || "")
-    end
-  end
-
   video = Video.new({
     id:      id,
     info:    info,

src/invidious/videos/parser.cr

@@ -55,30 +55,71 @@ module Invidious::Videos::Parser
     }
   end
 
+  # Builds the minimal "error" info hash returned when a video can't be
+  # played but we still want to render the watch page. `subreason` is only
+  # included when present, so it never gets stored as a JSON `null`.
+  private def self.error_video_info(reason : String?, subreason : String?) : Hash(String, JSON::Any)
+    params = {
+      "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),
+    } of String => JSON::Any
+    params["reason"] = JSON::Any.new(reason) if reason
+    params["subreason"] = JSON::Any.new(subreason) if subreason
+    params
+  end
+
   def extract_video_info(video_id : String)
     # Fetch data from the player endpoint
+    player_response = nil
+    begin
       player_response = YoutubeAPI.player(video_id: video_id)
+    rescue ex
+      LOGGER.debug("extract_video_info: player endpoint failed for #{video_id}: #{ex.message}")
+    end
 
     if player_response.nil?
-      return nil
+      # Player endpoint failed (e.g. no companion). Try to get data from /next only.
+      begin
+        player_response = YoutubeAPI.next({"videoId": video_id, "params": ""})
+      rescue ex
+        LOGGER.debug("extract_video_info: /next also failed for #{video_id}: #{ex.message}")
+        raise NotFoundException.new("Video unavailable")
+      end
+      begin
+        params = self.parse_video_info(video_id, player_response)
+        params["version"] = JSON::Any.new(Video::SCHEMA_VERSION.to_i64)
+        params["reason"] = JSON::Any.new("Video unavailable")
+        params["subreason"] = JSON::Any.new("Invidious Companion is not available. Video playback is not possible. \
+If you are the administrator, install Invidious Companion: \
+<a href=\"https://docs.invidious.io/installation/\">https://docs.invidious.io/installation/</a>")
+        return params
+      rescue ex
+        LOGGER.debug("extract_video_info: parse from /next failed for #{video_id}: #{ex.message}")
+        raise NotFoundException.new("Video unavailable")
+      end
     end
 
     playability_status = player_response.dig?("playabilityStatus", "status").try &.as_s
 
     if playability_status != "OK"
-      subreason = player_response.dig?("playabilityStatus", "errorScreen", "playerErrorMessageRenderer", "subreason")
+      reason = player_response.dig?("playabilityStatus", "reason").try &.as_s
-      reason = subreason.try &.[]?("simpleText").try &.as_s
+      reason ||= player_response.dig?("playabilityStatus", "errorScreen", "playerErrorMessageRenderer", "reason", "simpleText").try &.as_s
-      reason ||= subreason.try &.[]("runs").as_a.map(&.[]("text")).join("")
-      reason ||= player_response.dig("playabilityStatus", "reason").as_s
 
-      # Stop here if video is not a scheduled livestream or
+      subreason_main = player_response.dig?("playabilityStatus", "errorScreen", "playerErrorMessageRenderer", "subreason")
-      # for LOGIN_REQUIRED when videoDetails element is not found because retrying won't help
+      subreason = subreason_main.try &.["simpleText"]?.try &.as_s
-      if !{"LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status) ||
+      subreason ||= subreason_main.try &.["runs"]?.try &.as_a.map(&.["text"]).join("")
-         playability_status == "LOGIN_REQUIRED" && !player_response.dig?("videoDetails")
+
-        return {
+      # These strings come straight from YouTube and are rendered as raw HTML
-          "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),
+      # on the watch/embed pages, so escape them to avoid HTML injection.
-          "reason"  => JSON::Any.new(reason),
+      reason = HTML.escape(reason) if reason
-        }
+      subreason = HTML.escape(subreason) if subreason
+
+      # Stop here if the video is genuinely unavailable (deleted, removed or
+      # nonexistent): YouTube returns status "ERROR" for these regardless of
+      # the (localized) reason text, and there is nothing to extract from the
+      # /next endpoint. For other statuses (UNPLAYABLE, LOGIN_REQUIRED,
+      # LIVE_STREAM_OFFLINE, ...) we continue and extract as much as possible.
+      if playability_status == "ERROR"
+        raise NotFoundException.new(reason || "Video unavailable")
       end
     elsif video_id != player_response.dig?("videoDetails", "videoId")
       # YouTube may return a different video player response than expected.
@@ -89,25 +130,37 @@ module Invidious::Videos::Parser
       # wrong video means that we should count it as a failure.
       Helpers.get_playback_statistic["totalRequests"] += 1
 
-      return {
+      return error_video_info("Can't load the video on this Invidious instance. YouTube is currently trying to block Invidious instances. <a href=\"https://github.com/iv-org/invidious/issues/3822\">Click here for more info about the issue.</a>", nil)
-        "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),
-        "reason"  => JSON::Any.new("Can't load the video on this Invidious instance. YouTube is currently trying to block Invidious instances. <a href=\"https://github.com/iv-org/invidious/issues/3822\">Click here for more info about the issue.</a>"),
-      }
     else
       reason = nil
     end
 
-    # Don't fetch the next endpoint if the video is unavailable.
+    # Fetch the /next endpoint for additional data.
-    if {"OK", "LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status)
+    begin
       next_response = YoutubeAPI.next({"videoId": video_id, "params": ""})
       # Remove the microformat returned by the /next endpoint on some videos
       # to prevent player_response microformat from being overwritten.
       next_response.delete("microformat")
       player_response = player_response.merge(next_response)
+    rescue ex
+      # If we're in an error state and /next fails, return what we have
+      if reason
+        LOGGER.debug("extract_video_info: /next failed for #{video_id}: #{ex.message}")
+        return error_video_info(reason, subreason)
+      end
+      raise ex
     end
 
+    begin
       params = self.parse_video_info(video_id, player_response)
+    rescue ex : BrokenTubeException
+      if reason
+        return error_video_info(reason, subreason)
+      end
+      raise ex
+    end
     params["reason"] = JSON::Any.new(reason) if reason
+    params["subreason"] = JSON::Any.new(subreason) if subreason
 
     {"captions", "playabilityStatus", "playerConfig", "storyboards"}.each do |f|
       params[f] = player_response[f] if player_response[f]?
@@ -177,16 +230,20 @@ module Invidious::Videos::Parser
     end
 
     video_details = player_response.dig?("videoDetails")
+    video_details ||= {} of String => JSON::Any
     if !(microformat = player_response.dig?("microformat", "playerMicroformatRenderer"))
       microformat = {} of String => JSON::Any
     end
 
-    raise BrokenTubeException.new("videoDetails") if !video_details
-
     # Basic video infos
 
     title = video_details["title"]?.try &.as_s
 
+    title ||= extract_text(
+      video_primary_renderer
+        .try &.dig?("title")
+    )
+
     # We have to try to extract viewCount from videoPrimaryInfoRenderer first,
     # then from videoDetails, as the latter is "0" for livestreams (we want
     # to get the amount of viewers watching).
@@ -197,11 +254,31 @@ module Invidious::Videos::Parser
     views_txt ||= video_details["viewCount"]?.try &.as_s || ""
     views = views_txt.gsub(/\D/, "").to_i64?
 
-    length_txt = (microformat["lengthSeconds"]? || video_details["lengthSeconds"])
+    length_txt = (microformat["lengthSeconds"]? || video_details["lengthSeconds"]?)
       .try &.as_s.to_i64
 
     published = microformat["publishDate"]?
-      .try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc
+      .try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) }
+
+    if published.nil?
+      # Fall back to the human-readable date shown on the watch page (e.g.
+      # "Started streaming 3 hours ago" or "Premiered Jan 1, 2024").
+      published_txt = video_primary_renderer
+        .try &.dig?("dateText", "simpleText").try &.as_s
+
+      date_regex = /(\w{3} \d{1,2}, \d{4})$/
+
+      published =
+        if published_txt.nil?
+          Time.utc
+        elsif published_txt.includes?("ago")
+          decode_date(published_txt.lchop("Started streaming "))
+        elsif match = published_txt.match(date_regex)
+          Time.parse(match[0], "%b %-d, %Y", Time::Location::UTC)
+        else
+          Time.utc
+        end
+    end
 
     premiere_timestamp = microformat.dig?("liveBroadcastDetails", "startTimestamp")
       .try { |t| Time.parse_rfc3339(t.as_s) }
@@ -216,7 +293,9 @@ module Invidious::Videos::Parser
 
     live_now = microformat.dig?("liveBroadcastDetails", "isLiveNow")
       .try &.as_bool
-    live_now ||= video_details.dig?("isLive").try &.as_bool || false
+    live_now = video_details.dig?("isLive").try &.as_bool if live_now.nil?
+    live_now = video_primary_renderer.try &.dig?("viewCount", "videoViewCountRenderer", "isLive").try &.as_bool if live_now.nil?
+    live_now ||= false
 
     post_live_dvr = video_details.dig?("isPostLiveDvr")
       .try &.as_bool || false
@@ -227,8 +306,19 @@ module Invidious::Videos::Parser
       .try &.as_a.map &.as_s || [] of String
 
     allow_ratings = video_details["allowRatings"]?.try &.as_bool
+    # When the microformat is missing (e.g. /next-only data) we can't know for
+    # sure, so optimistically assume the video is family safe.
     family_friendly = microformat["isFamilySafe"]?.try &.as_bool
+    family_friendly = true if family_friendly.nil?
     is_listed = video_details["isCrawlable"]?.try &.as_bool
+    if is_listed.nil?
+      # Without videoDetails, infer from the "Unlisted" badge if present.
+      if video_badges = video_primary_renderer.try &.dig?("badges")
+        is_listed = !has_unlisted_badge?(video_badges)
+      else
+        is_listed = true
+      end
+    end
     is_upcoming = video_details["isUpcoming"]?.try &.as_bool
 
     keywords = video_details["keywords"]?
@@ -394,6 +484,9 @@ module Invidious::Videos::Parser
       subs_text = author_info["subscriberCountText"]?
         .try { |t| t["simpleText"]? || t.dig?("runs", 0, "text") }
         .try &.as_s.split(" ", 2)[0]
+
+      author ||= extract_text(author_info.dig?("title"))
+      ucid ||= author_info.dig?("title", "runs", 0, "navigationEndpoint", "browseEndpoint", "browseId").try &.as_s
     end
 
     # Return data
@@ -428,7 +521,7 @@ module Invidious::Videos::Parser
       # Description
       "description"      => JSON::Any.new(description || ""),
       "descriptionHtml"  => JSON::Any.new(description_html || "<p></p>"),
-      "shortDescription" => JSON::Any.new(short_description.try &.as_s || nil),
+      "shortDescription" => JSON::Any.new(short_description.try &.as_s || ""),
       # Video metadata
       "genre"     => JSON::Any.new(genre.try &.as_s || ""),
       "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s?),

src/invidious/views/embed.ecr

@@ -31,7 +31,15 @@
 %>
 </script>
 
+<% if video.reason.nil? %>
 <%= rendered "components/player" %>
+<% else %>
+<p>
+    <%= I18n.translate(preferences.locale, "error_from_youtube_unplayable") %>
+    <strong><%= video.reason %></strong>
+    <% if video.subreason %><br/><%= video.subreason %><% end %>
+</p>
+<% end %>
 <script src="/js/embed.js?v=<%= ASSET_COMMIT %>"></script>
 </body>
 </html>

src/invidious/views/watch.ecr

@@ -71,11 +71,22 @@ we're going to need to do it here in order to allow for translations.
 %>
 </script>
 
+<% if video.reason.nil? %>
 <div id="player-container" class="h-box">
     <%= rendered "components/player" %>
 </div>
+<% end %>
 
 <div class="h-box">
+    <% if video.reason %>
+        <h3>
+            <%= I18n.translate(locale, "error_from_youtube_unplayable") %> <%= video.reason %>
+        </h3>
+        <% if video.subreason %>
+        <p><%= video.subreason %></p>
+        <% end %>
+        <%= error_redirect_helper(env) %>
+    <% end %>
     <h1>
         <%= title %>
         <% if params.listen %>
@@ -95,11 +106,7 @@ we're going to need to do it here in order to allow for translations.
         </h3>
     <% end %>
 
-    <% if video.reason %>
+    <% if video.premiere_timestamp.try &.> Time.utc %>
-        <h3>
-            <%= video.reason %>
-        </h3>
-    <% elsif video.premiere_timestamp.try &.> Time.utc %>
         <h3>
             <%= video.premiere_timestamp.try { |t| I18n.translate(locale, "Premieres in `x`", recode_date((t - Time.utc).ago, locale)) } %>
         </h3>

src/invidious/yt_backend/extractors_utils.cr

@@ -68,6 +68,12 @@ rescue ex
   return false
 end
 
+def has_unlisted_badge?(badges : JSON::Any?)
+  return badges.try &.as_a.any? { |badge|
+    badge.dig?("metadataBadgeRenderer", "icon", "iconType").try &.as_s == "PRIVACY_UNLISTED"
+  } || false
+end
+
 # This function extracts SearchVideo items from a Category.
 # Categories are commonly returned in search results and trending pages.
 def extract_category(category : Category) : Array(SearchVideo)

src/invidious/yt_backend/youtube_api.cr

@@ -463,6 +463,8 @@ module YoutubeAPI
     if CONFIG.invidious_companion.present?
       return self._post_invidious_companion("/youtubei/v1/player", data)
     else
+      # Without Invidious Companion the player endpoint is unavailable.
+      # Returning nil lets the caller fall back to the /next endpoint.
       return nil
     end
   end