Merge branch 'iv-org:master' into fix-comment-with-emoji

README.md

@@ -129,7 +129,7 @@ You can read more here: https://docs.invidious.io/applications/
 1.  Fork it ( https://github.com/iv-org/invidious/fork ).
 1.  Create your feature branch (`git checkout -b my-new-feature`).
 1.  Stage your files (`git add .`).
-1.  Commit your changes (`git commit -am 'Add some feature'`).
+1.  Commit your changes (`git commit -m 'Add some feature'`).
 1.  Push to the branch (`git push origin my-new-feature`).
 1.  Create a new pull request ( https://github.com/iv-org/invidious/compare ).
 

assets/css/default.css

@@ -75,6 +75,16 @@ body {
   height: auto;
 }
 
+.channel-profile > .channel-name-pronouns {
+    display: inline-block;
+}
+
+.channel-profile > .channel-name-pronouns > .channel-pronouns {
+  font-style: italic;
+  font-size: .8em;
+  font-weight: lighter;
+}
+
 body a.channel-owner {
   background-color: #008bec;
   color: #fff;
@@ -406,7 +416,12 @@ input[type="search"]::-webkit-search-cancel-button {
 
 p.channel-name { margin: 0; overflow-wrap: anywhere;}
 p.video-data   { margin: 0; font-weight: bold; font-size: 80%; }
-.channel-profile > .channel-name { overflow-wrap: anywhere;}
+
+.channel-profile > .channel-name,
+.channel-profile > .channel-name-pronouns >  .channel-name 
+{
+  overflow-wrap: anywhere;
+}
 
 
 /*

config/config.example.yml

@@ -8,6 +8,13 @@
 ## Database configuration with separate parameters.
 ## This setting is MANDATORY, unless 'database_url' is used.
 ##
+## Note: The 'db' setting allows the use of UNIX
+## sockets. To do so, set 'host' to ""
+## E.g:
+##    password: kemal
+##    host: ""
+##    port: 5432
+##
 db:
   user: kemal
   password: kemal
@@ -223,8 +230,12 @@ https_only: false
 
 ##
 ## Configuration for using a HTTP proxy
-##
 ## If unset, then no HTTP proxy will be used.
+## Proxy type supported: HTTP, HTTPS
+## 
+## This is not used for loading the video streams from YouTube servers (circumvent YouTube restrictions)
+## Please instead configure the proxy in Invidious companion:
+## https://github.com/iv-org/invidious-companion/blob/master/config/config.example.toml
 ##
 #http_proxy:
 #  user:

shard.yml

@@ -5,6 +5,10 @@ authors:
   - Invidious team <contact@invidious.io>
   - Contributors!
 
+targets:
+  invidious:
+    main: src/invidious.cr
+
 description: |
    Invidious is an alternative front-end to YouTube 
 

src/invidious.cr

@@ -67,20 +67,9 @@ rescue ex
   puts "Check your 'config.yml' database settings or PostgreSQL settings."
   exit(1)
 end
-ARCHIVE_URL = URI.parse("https://archive.org")
+HOST_URL           = make_host_url(Kemal.config)
-PUBSUB_URL  = URI.parse("https://pubsubhubbub.appspot.com")
-REDDIT_URL  = URI.parse("https://www.reddit.com")
-YT_URL      = URI.parse("https://www.youtube.com")
-HOST_URL    = make_host_url(Kemal.config)
-
-CHARS_SAFE         = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
-TEST_IDS           = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"}
 MAX_ITEMS_PER_PAGE = 1500
 
-REQUEST_HEADERS_WHITELIST  = {"accept", "accept-encoding", "cache-control", "content-length", "if-none-match", "range"}
-RESPONSE_HEADERS_BLACKLIST = {"access-control-allow-origin", "alt-svc", "server"}
-HTTP_CHUNK_SIZE            = 10485760 # ~10MB
-
 CURRENT_BRANCH  = {{ "#{`git branch | sed -n '/* /s///p'`.strip}" }}
 CURRENT_COMMIT  = {{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit`.strip}" }}
 CURRENT_VERSION = {{ "#{`git log -1 --format=%ci | awk '{print $1}' | sed s/-/./g`.strip}" }}
@@ -97,7 +86,7 @@ SOFTWARE = {
   "branch"  => "#{CURRENT_BRANCH}",
 }
 
-YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size)
+YT_POOL = YoutubeConnectionPool.new(URI.parse("https://www.youtube.com"), capacity: CONFIG.pool_size)
 
 # Image request pool
 

src/invidious/channels/about.cr

@@ -12,6 +12,7 @@ record AboutChannel,
   sub_count : Int32,
   joined : Time,
   is_family_friendly : Bool,
+  pronouns : String?,
   allowed_regions : Array(String),
   tabs : Array(String),
   tags : Array(String),
@@ -160,14 +161,21 @@ def get_about_info(ucid, locale) : AboutChannel
   end
 
   sub_count = 0
+  pronouns = nil
 
   if (metadata_rows = initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "metadata", "contentMetadataViewModel", "metadataRows").try &.as_a)
     metadata_rows.each do |row|
-      metadata_part = row.dig?("metadataParts").try &.as_a.find { |i| i.dig?("text", "content").try &.as_s.includes?("subscribers") }
+      subscribe_metadata_part = row.dig?("metadataParts").try &.as_a.find { |i| i.dig?("text", "content").try &.as_s.includes?("subscribers") }
-      if !metadata_part.nil?
+      if !subscribe_metadata_part.nil?
-        sub_count = short_text_to_number(metadata_part.dig("text", "content").as_s.split(" ")[0]).to_i32
+        sub_count = short_text_to_number(subscribe_metadata_part.dig("text", "content").as_s.split(" ")[0]).to_i32
       end
-      break if sub_count != 0
+
+      pronoun_metadata_part = row.dig?("metadataParts").try &.as_a.find { |i| i.dig?("tooltip").try &.as_s.includes?("Pronouns") }
+      if !pronoun_metadata_part.nil?
+        pronouns = pronoun_metadata_part.dig("text", "content").as_s
+      end
+
+      break if sub_count != 0 && !pronouns.nil?
     end
   end
 
@@ -184,6 +192,7 @@ def get_about_info(ucid, locale) : AboutChannel
     sub_count: sub_count,
     joined: joined,
     is_family_friendly: is_family_friendly,
+    pronouns: pronouns,
     allowed_regions: allowed_regions,
     tabs: tab_names,
     tags: tags,

src/invidious/channels/videos.cr

@@ -114,7 +114,11 @@ module Invidious::Channel::Tabs
         "2:embedded" => {
           "1:string" => "00000000-0000-0000-0000-000000000000",
         },
-        "4:varint" => sort_options_videos_short(sort_by),
+        "4:varint"   => sort_options_videos_short(sort_by),
+        "8:embedded" => {
+          "1:string" => "00000000-0000-0000-0000-000000000000",
+          "3:varint" => sort_options_videos_short(sort_by),
+        },
       },
     }
 
@@ -130,7 +134,11 @@ module Invidious::Channel::Tabs
         "2:embedded" => {
           "1:string" => "00000000-0000-0000-0000-000000000000",
         },
-        "4:varint" => sort_options_videos_short(sort_by),
+        "4:varint"   => sort_options_videos_short(sort_by),
+        "7:embedded" => {
+          "1:string" => "00000000-0000-0000-0000-000000000000",
+          "3:varint" => sort_options_videos_short(sort_by),
+        },
       },
     }
 
@@ -154,7 +162,11 @@ module Invidious::Channel::Tabs
         "2:embedded" => {
           "1:string" => "00000000-0000-0000-0000-000000000000",
         },
-        "5:varint" => sort_by_numerical,
+        "5:varint"   => sort_by_numerical,
+        "8:embedded" => {
+          "1:string" => "00000000-0000-0000-0000-000000000000",
+          "3:varint" => sort_by_numerical,
+        },
       },
     }
 

src/invidious/comments/reddit.cr

@@ -1,5 +1,6 @@
 module Invidious::Comments
   extend self
+  private REDDIT_URL = URI.parse("https://www.reddit.com")
 
   def fetch_reddit(id, sort_by = "confidence")
     client = make_client(REDDIT_URL)

src/invidious/frontend/watch_page.cr

@@ -36,7 +36,7 @@ module Invidious::Frontend::WatchPage
     return String.build(4000) do |str|
       str << "<form"
       str << " class=\"pure-form pure-form-stacked\""
-      str << " action='#{url}'"
+      str << " action='" << HTML.escape(url) << "'"
       str << " method='post'"
       str << " rel='noopener noreferrer'"
       str << " target='_blank'>"

src/invidious/helpers/helpers.cr

@@ -1,5 +1,7 @@
 require "./macros"
 
+TEST_IDS = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"}
+
 struct Nonce
   include DB::Serializable
 

src/invidious/helpers/utils.cr

@@ -1,3 +1,5 @@
+PUBSUB_URL = URI.parse("https://pubsubhubbub.appspot.com")
+
 # See http://www.evanmiller.org/how-not-to-sort-by-average-rating.html
 def ci_lower_bound(pos, n)
   if n == 0

src/invidious/playlists.cr

@@ -107,7 +107,11 @@ struct Playlist
 
       json.field "author", self.author
       json.field "authorId", self.ucid
-      json.field "authorUrl", "/channel/#{self.ucid}"
+      if !self.ucid.empty?
+        json.field "authorUrl", "/channel/#{self.ucid}"
+      else
+        json.field "authorUrl", ""
+      end
       json.field "subtitle", self.subtitle
 
       json.field "authorThumbnails" do
@@ -359,6 +363,9 @@ def fetch_playlist(plid : String)
   thumbnail = playlist_info.dig?(
     "thumbnailRenderer", "playlistVideoThumbnailRenderer",
     "thumbnail", "thumbnails", 0, "url"
+  ).try &.as_s || playlist_info.dig?(
+    "thumbnailRenderer", "playlistCustomThumbnailRenderer",
+    "thumbnail", "thumbnails", 0, "url"
   ).try &.as_s
 
   views = 0_i64

src/invidious/routes/api/v1/channels.cr

@@ -104,6 +104,7 @@ module Invidious::Routes::API::V1::Channels
         json.field "tabs", channel.tabs
         json.field "tags", channel.tags
         json.field "authorVerified", channel.verified
+        json.field "pronouns", channel.pronouns
 
         json.field "latestVideos" do
           json.array do

src/invidious/routes/api/v1/videos.cr

@@ -1,6 +1,9 @@
 require "html"
 
 module Invidious::Routes::API::V1::Videos
+  private INTERNET_ARCHIVE_URL = URI.parse("https://archive.org")
+  private CHARS_SAFE           = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"
+
   def self.videos(env)
     locale = env.get("preferences").as(Preferences).locale
 
@@ -279,7 +282,7 @@ module Invidious::Routes::API::V1::Videos
 
         file = URI.encode_www_form("#{id[0, 3]}/#{id}.xml")
 
-        location = make_client(ARCHIVE_URL, &.get("/download/youtubeannotations_#{index}/#{id[0, 2]}.tar/#{file}"))
+        location = make_client(INTERNET_ARCHIVE_URL, &.get("/download/youtubeannotations_#{index}/#{id[0, 2]}.tar/#{file}"))
 
         if !location.headers["Location"]?
           env.response.status_code = location.status_code

src/invidious/routes/before_all.cr

@@ -1,4 +1,16 @@
 module Invidious::Routes::BeforeAll
+  struct CompanionCSP
+    property companion_urls : String = ""
+
+    def initialize
+      self.companion_urls = CONFIG.invidious_companion.reject(&.builtin_proxy).map do |companion|
+        "#{companion.public_url.scheme}://#{companion.public_url.host}#{companion.public_url.port ? ":#{companion.public_url.port}" : ""}"
+      end.join(" ")
+    end
+  end
+
+  private COMPANION_CSP = CompanionCSP.new
+
   def self.handle(env)
     preferences = Preferences.from_json("{}")
 
@@ -35,9 +47,9 @@ module Invidious::Routes::BeforeAll
       "style-src 'self' 'unsafe-inline'",
       "img-src 'self' data:",
       "font-src 'self' data:",
-      "connect-src 'self'",
+      "connect-src 'self' " + COMPANION_CSP.companion_urls,
       "manifest-src 'self'",
-      "media-src 'self' blob:",
+      "media-src 'self' blob: " + COMPANION_CSP.companion_urls,
       "child-src 'self' blob:",
       "frame-src 'self'",
       "frame-ancestors " + frame_ancestors,
@@ -94,8 +106,8 @@ module Invidious::Routes::BeforeAll
     end
 
     dark_mode = convert_theme(env.params.query["dark_mode"]?) || preferences.dark_mode.to_s
-    thin_mode = env.params.query["thin_mode"]? || preferences.thin_mode.to_s
+    thin_mode = env.params.query["thin_mode"]?
-    thin_mode = thin_mode == "true"
+    thin_mode = (thin_mode == "true") || preferences.thin_mode
     locale = env.params.query["hl"]? || preferences.locale
 
     preferences.dark_mode = dark_mode

src/invidious/routes/channels.cr

@@ -231,8 +231,10 @@ module Invidious::Routes::Channels
       env.redirect "/post/#{URI.encode_www_form(lb)}?ucid=#{URI.encode_www_form(ucid)}"
     end
 
-    thin_mode = env.params.query["thin_mode"]? || env.get("preferences").as(Preferences).thin_mode
+    preferences = env.get("preferences").as(Preferences)
-    thin_mode = thin_mode == "true"
+
+    thin_mode = env.params.query["thin_mode"]?
+    thin_mode = (thin_mode == "true") || preferences.thin_mode
 
     continuation = env.params.query["continuation"]?
 

src/invidious/routes/embed.cr

@@ -208,17 +208,6 @@ module Invidious::Routes::Embed
 
     if CONFIG.invidious_companion.present?
       invidious_companion = CONFIG.invidious_companion.sample
-      invidious_companion_urls = CONFIG.invidious_companion.reject(&.builtin_proxy).map do |companion|
-        uri =
-          "#{companion.public_url.scheme}://#{companion.public_url.host}#{companion.public_url.port ? ":#{companion.public_url.port}" : ""}"
-      end.join(" ")
-
-      if !invidious_companion_urls.empty?
-        env.response.headers["Content-Security-Policy"] =
-          env.response.headers["Content-Security-Policy"]
-            .gsub("media-src", "media-src #{invidious_companion_urls}")
-            .gsub("connect-src", "connect-src #{invidious_companion_urls}")
-      end
     end
 
     rendered "embed"

src/invidious/routes/routes.cr

@@ -0,0 +1,20 @@
+module Invidious::Routes
+  private REQUEST_HEADERS_WHITELIST = {
+    "accept",
+    "accept-encoding",
+    "cache-control",
+    "content-length",
+    "if-none-match",
+    "range",
+  }
+  private RESPONSE_HEADERS_BLACKLIST = {
+    "access-control-allow-origin",
+    "alt-svc",
+    "server",
+    "cross-origin-opener-policy-report-only",
+    "report-to",
+    "cross-origin",
+    "timing-allow-origin",
+    "cross-origin-resource-policy",
+  }
+end

src/invidious/routes/video_playback.cr

@@ -1,4 +1,6 @@
 module Invidious::Routes::VideoPlayback
+  private HTTP_CHUNK_SIZE = 10485760 # ~10MB
+
   # /videoplayback
   def self.get_video_playback(env)
     locale = env.get("preferences").as(Preferences).locale

src/invidious/routes/watch.cr

@@ -193,17 +193,6 @@ module Invidious::Routes::Watch
 
     if CONFIG.invidious_companion.present?
       invidious_companion = CONFIG.invidious_companion.sample
-      invidious_companion_urls = CONFIG.invidious_companion.reject(&.builtin_proxy).map do |companion|
-        uri =
-          "#{companion.public_url.scheme}://#{companion.public_url.host}#{companion.public_url.port ? ":#{companion.public_url.port}" : ""}"
-      end.join(" ")
-
-      if !invidious_companion_urls.empty?
-        env.response.headers["Content-Security-Policy"] =
-          env.response.headers["Content-Security-Policy"]
-            .gsub("media-src", "media-src #{invidious_companion_urls}")
-            .gsub("connect-src", "connect-src #{invidious_companion_urls}")
-      end
     end
 
     templated "watch"

src/invidious/user/imports.cr

@@ -30,28 +30,24 @@ struct Invidious::User
       return subscriptions
     end
 
-    def parse_playlist_export_csv(user : User, raw_input : String)
+    # Parse a CSV Google Takeout - Youtube Playlist file
+    def parse_playlist_export_csv(user : User, playlist_name : String, raw_input : String)
       # Split the input into head and body content
-      raw_head, raw_body = raw_input.strip('\n').split("\n\n", limit: 2, remove_empty: true)
+      raw_head, raw_body = raw_input.split("\n", limit: 2, remove_empty: true)
 
       # Create the playlist from the head content
       csv_head = CSV.new(raw_head.strip('\n'), headers: true)
       csv_head.next
-      title = csv_head[4]
+      title = playlist_name
-      description = csv_head[5]
-      visibility = csv_head[6]
 
-      if visibility.compare("Public", case_insensitive: true) == 0
+      description = "This is the default description of an imported playlist. Feel Free to change it as you see fit."
-        privacy = PlaylistPrivacy::Public
+      privacy = PlaylistPrivacy::Private
-      else
-        privacy = PlaylistPrivacy::Private
-      end
 
       playlist = create_playlist(title, privacy, user)
       Invidious::Database::Playlists.update_description(playlist.id, description)
 
       # Add each video to the playlist from the body content
-      csv_body = CSV.new(raw_body.strip('\n'), headers: true)
+      csv_body = CSV.new(raw_body.strip('\n'), headers: false)
       csv_body.each do |row|
         video_id = row[0]
         if playlist
@@ -204,10 +200,12 @@ struct Invidious::User
     end
 
     def from_youtube_pl(user : User, body : String, filename : String, type : String) : Bool
-      extension = filename.split(".").last
+      filename_array = filename.split(".")
+      playlist_name = filename_array.first
+      extension = filename_array.last
 
       if extension == "csv" || type == "text/csv"
-        playlist = parse_playlist_export_csv(user, body)
+        playlist = parse_playlist_export_csv(user, playlist_name, body)
         if playlist
           return true
         else

src/invidious/views/components/channel_info.ecr

@@ -12,7 +12,10 @@
     <div class="pure-u-1-2 flex-left flexible">
         <div class="channel-profile">
             <img src="/ggpht<%= channel_profile_pic %>" alt="" />
-            <span class="channel-name"><%= author %></span><% if !channel.verified.nil? && channel.verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end %>
+            <div class="channel-name-pronouns">
+                <span class="channel-name"><%= author %></span><% if !channel.verified.nil? && channel.verified %>&nbsp;<i class="icon ion ion-md-checkmark-circle"></i><% end %>
+                <% if !channel.pronouns.nil? %><br /><span class="channel-pronouns"><%= channel.pronouns %></span><% end %>
+            </div>
         </div>
     </div>