Community: Fix live video + parse playlist attachments (#3767)

Due to recent changes to libxml2 (between 2.9.14 and 2.10.4,
See https://gitlab.gnome.org/GNOME/libxml2/-/issues/508), the
HTML parser doesn't take into account the namespaces (xmlns).

Because HTML shouldn't contain namespaces anyway, there is no
reason for use to keep using it. But switching to the XML
parser means that we have to pass the namespaces to every
single 'xpath_node(s)' method for it to be able to properly
navigate the XML structure.

assets/css/default.css

@@ -321,6 +321,30 @@ p.channel-name { margin: 0; }
 p.video-data   { margin: 0; font-weight: bold; font-size: 80%; }
 
 
+/*
+ * Comments & community posts
+ */
+
+#comments {
+  max-width: 800px;
+  margin: auto;
+}
+
+.video-iframe-wrapper {
+  position: relative;
+  height: 0;
+  padding-bottom: 56.25%;
+}
+
+.video-iframe {
+  position: absolute;
+  top: 0;
+  left: 0;
+  width: 100%;
+  height: 100%;
+  border: none;
+}
+
 /*
  * Footer
  */

locales/cs.json

@@ -13,7 +13,7 @@
     "Previous page": "Předchozí strana",
     "Clear watch history?": "Smazat historii?",
     "New password": "Nové heslo",
-    "New passwords must match": "Hesla se musí schodovat",
+    "New passwords must match": "Hesla se musí shodovat",
     "Cannot change password for Google accounts": "Nelze změnit heslo pro účty Google",
     "Authorize token?": "Autorizovat token?",
     "Authorize token for `x`?": "Autorizovat token pro `x`?",

locales/es.json

@@ -398,25 +398,24 @@
     "search_filters_features_option_three_sixty": "360°",
     "videoinfo_watch_on_youTube": "Ver en YouTube",
     "preferences_save_player_pos_label": "Guardar posición de reproducción: ",
-    "generic_views_count": "{{count}} visualización",
+    "generic_views_count": "{{count}} vista",
-    "generic_views_count_plural": "{{count}} visualizaciones",
+    "generic_views_count_plural": "{{count}} vistas",
     "generic_subscribers_count": "{{count}} suscriptor",
     "generic_subscribers_count_plural": "{{count}} suscriptores",
     "generic_subscriptions_count": "{{count}} suscripción",
     "generic_subscriptions_count_plural": "{{count}} suscripciones",
-    "subscriptions_unseen_notifs_count": "{{count}} notificación no vista",
+    "subscriptions_unseen_notifs_count": "{{count}} notificación sin ver",
-    "subscriptions_unseen_notifs_count_plural": "{{count}} notificaciones no vistas",
+    "subscriptions_unseen_notifs_count_plural": "{{count}} notificaciones sin ver",
     "generic_count_days": "{{count}} día",
     "generic_count_days_plural": "{{count}} días",
     "comments_view_x_replies": "Ver {{count}} respuesta",
     "comments_view_x_replies_plural": "Ver {{count}} respuestas",
     "generic_count_weeks": "{{count}} semana",
     "generic_count_weeks_plural": "{{count}} semanas",
-    "generic_playlists_count": "{{count}} lista de reproducción",
+    "generic_playlists_count": "{{count}} reproducción",
-    "generic_playlists_count_plural": "{{count}} listas de reproducción",
+    "generic_playlists_count_plural": "{{count}} reproducciones",
-    "generic_videos_count_0": "{{count}} video",
+    "generic_videos_count": "{{count}} video",
-    "generic_videos_count_1": "{{count}} videos",
+    "generic_videos_count_plural": "{{count}} videos",
-    "generic_videos_count_2": "{{count}} videos",
     "generic_count_months": "{{count}} mes",
     "generic_count_months_plural": "{{count}} meses",
     "comments_points_count": "{{count}} punto",
@@ -469,8 +468,8 @@
     "search_filters_duration_option_none": "Cualquier duración",
     "search_filters_features_option_vr180": "VR180",
     "search_filters_apply_button": "Aplicar filtros",
-    "tokens_count": "{{count}} ficha",
+    "tokens_count": "{{count}} token",
-    "tokens_count_plural": "{{count}} fichas",
+    "tokens_count_plural": "{{count}} tokens",
     "search_message_use_another_instance": " También puede <a href=\"`x`\">buscar en otra instancia</a>.",
     "Popular enabled: ": "¿Habilitar la sección popular? ",
     "error_video_not_in_playlist": "El video que solicitaste no existe en esta lista de reproducción. <a href=\"`x`\">Haz clic aquí para acceder a la página de inicio de la lista de reproducción.</a>",

locales/ko.json

@@ -46,7 +46,7 @@
     "Log in/register": "로그인/회원가입",
     "Log in": "로그인",
     "source": "출처",
-    "JavaScript license information": "자바스크립트 라이센스 정보",
+    "JavaScript license information": "자바스크립트 라이선스 정보",
     "An alternative front-end to YouTube": "유튜브의 프론트엔드 대안",
     "History": "역사",
     "Delete account?": "계정을 삭제 하시겠습니까?",
@@ -116,7 +116,7 @@
     "Show replies": "댓글 보기",
     "Hide replies": "댓글 숨기기",
     "Incorrect password": "잘못된 비밀번호",
-    "License: ": "라이센스: ",
+    "License: ": "라이선스: ",
     "Genre: ": "장르: ",
     "Editing playlist `x`": "재생목록 `x` 수정하기",
     "Playlist privacy": "재생목록 공개 범위",
@@ -135,7 +135,7 @@
     "Unlisted": "목록에 없음",
     "Public": "공개",
     "View privacy policy.": "개인정보 처리방침 보기.",
-    "View JavaScript license information.": "자바스크립트 라이센스 정보 보기.",
+    "View JavaScript license information.": "자바스크립트 라이선스 정보 보기.",
     "Source available here.": "소스는 여기에서 사용할 수 있습니다.",
     "Log out": "로그아웃",
     "search": "검색",
@@ -460,5 +460,12 @@
     "channel_tab_shorts_label": "쇼츠",
     "channel_tab_streams_label": "실시간 스트리밍",
     "channel_tab_channels_label": "채널",
-    "channel_tab_playlists_label": "재생목록"
+    "channel_tab_playlists_label": "재생목록",
+    "Standard YouTube license": "표준 유튜브 라이선스",
+    "Song: ": "제목: ",
+    "Channel Sponsor": "채널 스폰서",
+    "Album: ": "앨범: ",
+    "Music in this video": "동영상 속 음악",
+    "Artist: ": "아티스트: ",
+    "Download is disabled": "다운로드가 비활성화 되어있음"
 }

src/invidious/channels/channels.cr

@@ -159,12 +159,18 @@ def fetch_channel(ucid, pull_all_videos : Bool)
   LOGGER.debug("fetch_channel: #{ucid}")
   LOGGER.trace("fetch_channel: #{ucid} : pull_all_videos = #{pull_all_videos}")
 
+  namespaces = {
+    "yt"      => "http://www.youtube.com/xml/schemas/2015",
+    "media"   => "http://search.yahoo.com/mrss/",
+    "default" => "http://www.w3.org/2005/Atom",
+  }
+
   LOGGER.trace("fetch_channel: #{ucid} : Downloading RSS feed")
   rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}").body
   LOGGER.trace("fetch_channel: #{ucid} : Parsing RSS feed")
-  rss = XML.parse_html(rss)
+  rss = XML.parse(rss)
 
-  author = rss.xpath_node(%q(//feed/title))
+  author = rss.xpath_node("//default:feed/default:title", namespaces)
   if !author
     raise InfoException.new("Deleted or invalid channel")
   end
@@ -192,15 +198,23 @@ def fetch_channel(ucid, pull_all_videos : Bool)
   videos, continuation = IV::Channel::Tabs.get_videos(channel)
 
   LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel RSS feed")
-  rss.xpath_nodes("//feed/entry").each do |entry|
+  rss.xpath_nodes("//default:feed/default:entry", namespaces).each do |entry|
-    video_id = entry.xpath_node("videoid").not_nil!.content
+    video_id = entry.xpath_node("yt:videoId", namespaces).not_nil!.content
-    title = entry.xpath_node("title").not_nil!.content
+    title = entry.xpath_node("default:title", namespaces).not_nil!.content
-    published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content)
+
-    updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content)
+    published = Time.parse_rfc3339(
-    author = entry.xpath_node("author/name").not_nil!.content
+      entry.xpath_node("default:published", namespaces).not_nil!.content
-    ucid = entry.xpath_node("channelid").not_nil!.content
+    )
-    views = entry.xpath_node("group/community/statistics").try &.["views"]?.try &.to_i64?
+    updated = Time.parse_rfc3339(
-    views ||= 0_i64
+      entry.xpath_node("default:updated", namespaces).not_nil!.content
+    )
+
+    author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content
+    ucid = entry.xpath_node("yt:channelId", namespaces).not_nil!.content
+
+    views = entry
+      .xpath_node("media:group/media:community/media:statistics", namespaces)
+      .try &.["views"]?.try &.to_i64? || 0_i64
 
     channel_video = videos
       .select(SearchVideo)

src/invidious/channels/community.cr

@@ -123,49 +123,13 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode)
 
               if attachment = post["backstageAttachment"]?
                 json.field "attachment" do
-                  json.object do
                   case attachment.as_h
                   when .has_key?("videoRenderer")
-                      attachment = attachment["videoRenderer"]
+                    parse_item(attachment)
-                      json.field "type", "video"
+                      .as(SearchVideo)
-
+                      .to_json(locale, json)
-                      if !attachment["videoId"]?
-                        error_message = (attachment["title"]["simpleText"]? ||
-                                         attachment["title"]["runs"]?.try &.[0]?.try &.["text"]?)
-
-                        json.field "error", error_message
-                      else
-                        video_id = attachment["videoId"].as_s
-
-                        video_title = attachment["title"]["simpleText"]? || attachment["title"]["runs"]?.try &.[0]?.try &.["text"]?
-                        json.field "title", video_title
-                        json.field "videoId", video_id
-                        json.field "videoThumbnails" do
-                          Invidious::JSONify::APIv1.thumbnails(json, video_id)
-                        end
-
-                        json.field "lengthSeconds", decode_length_seconds(attachment["lengthText"]["simpleText"].as_s)
-
-                        author_info = attachment["ownerText"]["runs"][0].as_h
-
-                        json.field "author", author_info["text"].as_s
-                        json.field "authorId", author_info["navigationEndpoint"]["browseEndpoint"]["browseId"]
-                        json.field "authorUrl", author_info["navigationEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"]
-
-                        # TODO: json.field "authorThumbnails", "channelThumbnailSupportedRenderers"
-                        # TODO: json.field "authorVerified", "ownerBadges"
-
-                        published = decode_date(attachment["publishedTimeText"]["simpleText"].as_s)
-
-                        json.field "published", published.to_unix
-                        json.field "publishedText", translate(locale, "`x` ago", recode_date(published, locale))
-
-                        view_count = attachment["viewCountText"]?.try &.["simpleText"].as_s.gsub(/\D/, "").to_i64? || 0_i64
-
-                        json.field "viewCount", view_count
-                        json.field "viewCountText", translate_count(locale, "generic_views_count", view_count, NumberFormatting::Short)
-                      end
                   when .has_key?("backstageImageRenderer")
+                    json.object do
                       attachment = attachment["backstageImageRenderer"]
                       json.field "type", "image"
 
@@ -186,7 +150,9 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode)
                           end
                         end
                       end
+                    end
                   when .has_key?("pollRenderer")
+                    json.object do
                       attachment = attachment["pollRenderer"]
                       json.field "type", "poll"
                       json.field "totalVotes", short_text_to_number(attachment["totalVotes"]["simpleText"].as_s.split(" ")[0])
@@ -219,7 +185,9 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode)
                           end
                         end
                       end
+                    end
                   when .has_key?("postMultiImageRenderer")
+                    json.object do
                       attachment = attachment["postMultiImageRenderer"]
                       json.field "type", "multiImage"
                       json.field "images" do
@@ -243,7 +211,13 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode)
                           end
                         end
                       end
+                    end
+                  when .has_key?("playlistRenderer")
+                    parse_item(attachment)
+                      .as(SearchPlaylist)
+                      .to_json(locale, json)
                   else
+                    json.object do
                       json.field "type", "unknown"
                       json.field "error", "Unrecognized attachment type."
                     end

src/invidious/comments.cr

@@ -372,32 +372,25 @@ def template_youtube_comments(comments, locale, thin_mode, is_replies = false)
           </div>
           END_HTML
         when "video"
-          html << <<-END_HTML
-            <div class="pure-g">
-              <div class="pure-u-1 pure-u-md-1-2">
-                <div style="position:relative;width:100%;height:0;padding-bottom:56.25%;margin-bottom:5px">
-          END_HTML
-
           if attachment["error"]?
             html << <<-END_HTML
+            <div class="pure-g video-iframe-wrapper">
               <p>#{attachment["error"]}</p>
+            </div>
             END_HTML
           else
             html << <<-END_HTML
-              <iframe id='ivplayer' style='position:absolute;width:100%;height:100%;left:0;top:0' src='/embed/#{attachment["videoId"]?}?autoplay=0' style='border:none;'></iframe>
+            <div class="pure-g video-iframe-wrapper">
+              <iframe class="video-iframe" src='/embed/#{attachment["videoId"]?}?autoplay=0'></iframe>
+            </div>
             END_HTML
           end
-
-          html << <<-END_HTML
-                </div>
-              </div>
-            </div>
-          END_HTML
         else nil # Ignore
         end
       end
 
       html << <<-END_HTML
+      <p>
         <span title="#{Time.unix(child["published"].as_i64).to_s(translate(locale, "%A %B %-d, %Y"))}">#{translate(locale, "`x` ago", recode_date(Time.unix(child["published"].as_i64), locale))} #{child["isEdited"] == true ? translate(locale, "(edited)") : ""}</span>
         |
       END_HTML
@@ -416,6 +409,7 @@ def template_youtube_comments(comments, locale, thin_mode, is_replies = false)
 
       html << <<-END_HTML
         <i class="icon ion-ios-thumbs-up"></i> #{number_with_separator(child["likeCount"])}
+      </p>
       END_HTML
 
       if child["creatorHeart"]?

src/invidious/helpers/serialized_yt_data.cr

@@ -84,6 +84,7 @@ struct SearchVideo
       json.field "descriptionHtml", self.description_html
 
       json.field "viewCount", self.views
+      json.field "viewCountText", translate_count(locale, "generic_views_count", self.views, NumberFormatting::Short)
       json.field "published", self.published.to_unix
       json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale))
       json.field "lengthSeconds", self.length_seconds

src/invidious/yt_backend/extractors.cr

@@ -268,7 +268,7 @@ private module Parsers
     end
 
     private def self.parse(item_contents, author_fallback)
-      title = item_contents["title"]["simpleText"]?.try &.as_s || ""
+      title = extract_text(item_contents["title"]) || ""
       plid = item_contents["playlistId"]?.try &.as_s || ""
 
       video_count = HelperExtractors.get_video_count(item_contents)