User: Add support for importing Youtube watch history (#4171)

Co-Authored-By: Samantaz Fox <coding@samantaz.fr>

README.md

@@ -82,7 +82,7 @@
 
 **Data import/export**
 - Import subscriptions from YouTube, NewPipe and Freetube
-- Import watch history from NewPipe
+- Import watch history from YouTube and NewPipe
 - Export subscriptions to NewPipe and Freetube
 - Import/Export Invidious user data
 

assets/css/default.css

@@ -441,16 +441,26 @@ p.video-data   { margin: 0; font-weight: bold; font-size: 80%; }
  */
 
 footer {
-  color: #919191;
   margin-top: auto;
   padding: 1.5em 0;
   text-align: center;
   max-height: 30vh;
 }
 
-footer a {
+.light-theme footer {
-  color: #919191 !important;
+  color: #7c7c7c;
-  text-decoration: underline;
+}
+
+.dark-theme footer {
+  color: #adadad;
+}
+
+.light-theme footer a {
+  color: #7c7c7c !important;
+}
+
+.dark-theme footer a {
+  color: #adadad !important;
 }
 
 footer span {
@@ -556,6 +566,14 @@ span > select {
     color: #303030;
   }
 
+  .no-theme footer {
+    color: #7c7c7c;
+  }
+
+  .no-theme footer a {
+    color: #7c7c7c !important;
+  }
+
   .light-theme .pure-menu-heading {
     color: #565d64;
   }
@@ -589,7 +607,7 @@ span > select {
 }
 
 .dark-theme a {
-  color: #a0a0a0;
+  color: #adadad;
   text-decoration: none;
 }
 
@@ -643,7 +661,7 @@ body.dark-theme {
   }
 
   .no-theme a {
-    color: #a0a0a0;
+    color: #adadad;
     text-decoration: none;
   }
 
@@ -674,6 +692,14 @@ body.dark-theme {
     background-color: inherit;
     color: inherit;
   }
+
+  .no-theme footer {
+    color: #adadad;
+  }
+
+  .no-theme footer a {
+    color: #adadad !important;
+  }
 }
 
 
@@ -767,3 +793,7 @@ h1, h2, h3, h4, h5, p,
 .channel-emoji {
   margin: 0 2px;
 }
+
+#download_widget {
+    width: 100%;
+}

assets/js/player.js

@@ -98,11 +98,13 @@ if (video_data.params.quality === 'dash') {
 
 /**
  * Function for add time argument to url
+ *
  * @param {String} url
+ * @param {String} [base]
  * @returns {URL} urlWithTimeArg
  */
-function addCurrentTimeToURL(url) {
+function addCurrentTimeToURL(url, base) {
-    var urlUsed = new URL(url);
+    var urlUsed = new URL(url, base);
     urlUsed.searchParams.delete('start');
     var currentTime = Math.ceil(player.currentTime());
     if (currentTime > 0)
@@ -112,6 +114,50 @@ function addCurrentTimeToURL(url) {
     return urlUsed;
 }
 
+/**
+ * Global variable to save the last timestamp (in full seconds) at which the external
+ * links were updated by the 'timeupdate' callback below.
+ *
+ * It is initialized to 5s so that the video will always restart from the beginning
+ * if the user hasn't really started watching before switching to the other website.
+ */
+var timeupdate_last_ts = 5;
+
+/**
+ * Callback that updates the timestamp on all external links
+ */
+player.on('timeupdate', function () {
+    // Only update once every second
+    let current_ts = Math.floor(player.currentTime());
+    if (current_ts > timeupdate_last_ts) timeupdate_last_ts = current_ts;
+    else return;
+
+    // YouTube links
+
+    let elem_yt_watch = document.getElementById('link-yt-watch');
+    let elem_yt_embed = document.getElementById('link-yt-embed');
+
+    let base_url_yt_watch = elem_yt_watch.getAttribute('data-base-url');
+    let base_url_yt_embed = elem_yt_embed.getAttribute('data-base-url');
+
+    elem_yt_watch.href = addCurrentTimeToURL(base_url_yt_watch);
+    elem_yt_embed.href = addCurrentTimeToURL(base_url_yt_embed);
+
+    // Invidious links
+
+    let domain = window.location.origin;
+
+    let elem_iv_embed = document.getElementById('link-iv-embed');
+    let elem_iv_other = document.getElementById('link-iv-other');
+
+    let base_url_iv_embed = elem_iv_embed.getAttribute('data-base-url');
+    let base_url_iv_other = elem_iv_other.getAttribute('data-base-url');
+
+    elem_iv_embed.href = addCurrentTimeToURL(base_url_iv_embed, domain);
+    elem_iv_other.href = addCurrentTimeToURL(base_url_iv_other, domain);
+});
+
+
 var shareOptions = {
     socials: ['fbFeed', 'tw', 'reddit', 'email'],
 

locales/en-US.json

@@ -40,6 +40,7 @@
     "Import Invidious data": "Import Invidious JSON data",
     "Import YouTube subscriptions": "Import YouTube/OPML subscriptions",
     "Import YouTube playlist (.csv)": "Import YouTube playlist (.csv)",
+    "Import YouTube watch history (.json)": "Import YouTube watch history (.json)",
     "Import FreeTube subscriptions (.db)": "Import FreeTube subscriptions (.db)",
     "Import NewPipe subscriptions (.json)": "Import NewPipe subscriptions (.json)",
     "Import NewPipe data (.zip)": "Import NewPipe data (.zip)",

locales/zh-CN.json

@@ -461,6 +461,7 @@
     "Standard YouTube license": "标准 YouTube 许可证",
     "Download is disabled": "已禁用下载",
     "Import YouTube playlist (.csv)": "导入 YouTube 播放列表（.csv）",
+    "Import YouTube watch history (.json)": "导入 YouTube 观看历史（.json）",
     "generic_button_cancel": "取消",
     "playlist_button_add_items": "添加视频",
     "generic_button_delete": "删除",

locales/zh-TW.json

@@ -461,6 +461,7 @@
     "Standard YouTube license": "標準 YouTube 授權條款",
     "Download is disabled": "已停用下載",
     "Import YouTube playlist (.csv)": "匯入 YouTube 播放清單 (.csv)",
+    "Import YouTube watch history (.json)": "匯入 YouTube 觀看歷史 (.json)",
     "generic_button_cancel": "取消",
     "generic_button_edit": "編輯",
     "generic_button_save": "儲存",

src/invidious/frontend/watch_page.cr

@@ -42,8 +42,7 @@ module Invidious::Frontend::WatchPage
       str << translate(locale, "Download as: ")
       str << "</label>\n"
 
-      # TODO: remove inline style
+      str << "\t\t<select name='download_widget' id='download_widget'>\n"
-      str << "\t\t<select style=\"width:100%\" name='download_widget' id='download_widget'>\n"
 
       # Non-DASH videos (audio+video)
 

src/invidious/jsonify/api_v1/video_json.cr

@@ -39,6 +39,7 @@ module Invidious::JSONify::APIv1
       json.field "author", video.author
       json.field "authorId", video.ucid
       json.field "authorUrl", "/channel/#{video.ucid}"
+      json.field "authorVerified", video.author_verified
 
       json.field "authorThumbnails" do
         json.array do

src/invidious/routes/api/v1/videos.cr

@@ -136,17 +136,17 @@ module Invidious::Routes::API::V1::Videos
           end
         end
       else
-        # Some captions have "align:[start/end]" and "position:[num]%"
+        webvtt = YT_POOL.client &.get("#{url}&fmt=vtt").body
-        # attributes. Those are causing issues with VideoJS, which is unable
+
-        # to properly align the captions on the video, so we remove them.
-        #
-        # See: https://github.com/iv-org/invidious/issues/2391
-        webvtt = YT_POOL.client &.get("#{url}&format=vtt").body
         if webvtt.starts_with?("<?xml")
           webvtt = caption.timedtext_to_vtt(webvtt)
         else
-          webvtt = YT_POOL.client &.get("#{url}&format=vtt").body
+          # Some captions have "align:[start/end]" and "position:[num]%"
-            .gsub(/([0-9:.]{12} --> [0-9:.]{12}).+/, "\\1")
+          # attributes. Those are causing issues with VideoJS, which is unable
+          # to properly align the captions on the video, so we remove them.
+          #
+          # See: https://github.com/iv-org/invidious/issues/2391
+          webvtt = webvtt.gsub(/([0-9:.]{12} --> [0-9:.]{12}).+/, "\\1")
         end
       end
     end

src/invidious/routes/preferences.cr

@@ -319,6 +319,15 @@ module Invidious::Routes::PreferencesRoute
               response: error_template(415, "Invalid playlist file uploaded")
             )
           end
+        when "import_youtube_wh"
+          filename = part.filename || ""
+          success = Invidious::User::Import.from_youtube_wh(user, body, filename, type)
+
+          if !success
+            haltf(env, status_code: 415,
+              response: error_template(415, "Invalid watch history file uploaded")
+            )
+          end
         when "import_freetube"
           Invidious::User::Import.from_freetube(user, body)
         when "import_newpipe_subscriptions"

src/invidious/routes/watch.cr

@@ -30,14 +30,6 @@ module Invidious::Routes::Watch
       return env.redirect "/"
     end
 
-    embed_link = "/embed/#{id}"
-    if env.params.query.size > 1
-      embed_params = HTTP::Params.parse(env.params.query.to_s)
-      embed_params.delete_all("v")
-      embed_link += "?"
-      embed_link += embed_params.to_s
-    end
-
     plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "")
     continuation = process_continuation(env.params.query, plid, id)
 

src/invidious/user/imports.cr

@@ -218,6 +218,26 @@ struct Invidious::User
       end
     end
 
+    def from_youtube_wh(user : User, body : String, filename : String, type : String) : Bool
+      extension = filename.split(".").last
+
+      if extension == "json" || type == "application/json"
+        data = JSON.parse(body)
+        watched = data.as_a.compact_map do |item|
+          next unless url = item["titleUrl"]?
+          next unless match = url.as_s.match(/\?v=(?<video_id>[a-zA-Z0-9_-]+)$/)
+          match["video_id"]
+        end
+        watched.reverse! # YouTube have newest first
+        user.watched += watched
+        user.watched.uniq!
+        Invidious::Database::Users.update_watch_history(user)
+        return true
+      else
+        return false
+      end
+    end
+
     # -------------------
     #  Freetube
     # -------------------
@@ -228,8 +248,12 @@ struct Invidious::User
       subs = matches.map(&.["channel_id"])
 
       if subs.empty?
-        data = JSON.parse(body)["subscriptions"]
+        profiles = body.split('\n', remove_empty: true)
-        subs = data.as_a.map(&.["id"].as_s)
+        profiles.each do |profile|
+          if data = JSON.parse(profile)["subscriptions"]?
+            subs += data.as_a.map(&.["id"].as_s)
+          end
+        end
       end
 
       user.subscriptions += subs

src/invidious/views/user/data_control.ecr

@@ -26,6 +26,11 @@
                 <input type="file" id="import_youtube_pl" name="import_youtube_pl">
             </div>
 
+            <div class="pure-control-group">
+                <label for="import_youtube_wh"><%= translate(locale, "Import YouTube watch history (.json)") %></label>
+                <input type="file" id="import_youtube_wh" name="import_youtube_wh">
+            </div>
+
             <div class="pure-control-group">
                 <label for="import_freetube"><%= translate(locale, "Import FreeTube subscriptions (.db)") %></label>
                 <input type="file" id="import_freetube" name="import_freetube">

src/invidious/views/watch.ecr

@@ -113,19 +113,36 @@ we're going to need to do it here in order to allow for translations.
     <div class="pure-u-1 pure-u-lg-1-5">
         <div class="h-box">
             <span id="watch-on-youtube">
-                <a href="https://www.youtube.com/watch?v=<%= video.id %>"><%= translate(locale, "videoinfo_watch_on_youTube") %></a>
+                <%-
-                (<a href="https://www.youtube.com/embed/<%= video.id %>"><%= translate(locale, "videoinfo_youTube_embed_link") %></a>)
+                    link_yt_watch = URI.new(scheme: "https", host: "www.youtube.com", path: "/watch", query: "v=#{video.id}")
+                    link_yt_embed = URI.new(scheme: "https", host: "www.youtube.com", path: "/embed/#{video.id}")
+
+                    if !plid.nil? && !continuation.nil?
+                        link_yt_param = URI::Params{"plid" => [plid], "index" => [continuation.to_s]}
+                        link_yt_watch = IV::HttpServer::Utils.add_params_to_url(link_yt_watch, link_yt_param)
+                        link_yt_embed = IV::HttpServer::Utils.add_params_to_url(link_yt_embed, link_yt_param)
+                    end
+                -%>
+                <a id="link-yt-watch" data-base-url="<%= link_yt_watch %>" href="<%= link_yt_watch %>"><%= translate(locale, "videoinfo_watch_on_youTube") %></a>
+                (<a id="link-yt-embed" data-base-url="<%= link_yt_embed %>" href="<%= link_yt_embed %>"><%= translate(locale, "videoinfo_youTube_embed_link") %></a>)
             </span>
+
             <p id="watch-on-another-invidious-instance">
-            <% if env.get("preferences").as(Preferences).automatic_instance_redirect%>
+                <%- link_iv_other = IV::Frontend::Misc.redirect_url(env) -%>
-                <a href="/redirect?referer=<%= env.get?("current_page") %>"><%= translate(locale, "Switch Invidious Instance") %></a>
+                <a id="link-iv-other" data-base-url="<%= link_iv_other %>" href="<%= link_iv_other %>"><%= translate(locale, "Switch Invidious Instance") %></a>
-            <% else %>
-                <a href="https://redirect.invidious.io<%= env.request.resource %>"><%= translate(locale, "Switch Invidious Instance") %></a>
-            <% end %>
             </p>
+
             <p id="embed-link">
-                <a href="<%= embed_link %>"><%= translate(locale, "videoinfo_invidious_embed_link") %></a>
+                <%-
+                    params_iv_embed = env.params.query.dup
+                    params_iv_embed.delete_all("v")
+
+                    link_iv_embed = URI.new(path: "/embed/#{id}")
+                    link_iv_embed = IV::HttpServer::Utils.add_params_to_url(link_iv_embed, params_iv_embed)
+                -%>
+                <a id="link-iv-embed" data-base-url="<%= link_iv_embed %>" href="<%= link_iv_embed %>"><%= translate(locale, "videoinfo_invidious_embed_link") %></a>
             </p>
+
             <p id="annotations">
                 <% if params.annotations %>
                     <a href="/watch?<%= env.params.query %>&iv_load_policy=3">