diff --git a/README.md b/README.md index 97d2109b..5b789a50 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,7 @@ You can read more here: https://docs.invidious.io/applications/ 1. Fork it ( https://github.com/iv-org/invidious/fork ). 1. Create your feature branch (`git checkout -b my-new-feature`). 1. Stage your files (`git add .`). -1. Commit your changes (`git commit -am 'Add some feature'`). +1. Commit your changes (`git commit -m 'Add some feature'`). 1. Push to the branch (`git push origin my-new-feature`). 1. Create a new pull request ( https://github.com/iv-org/invidious/compare ). diff --git a/assets/css/default.css b/assets/css/default.css index 78ef7a60..ff07bdb4 100644 --- a/assets/css/default.css +++ b/assets/css/default.css @@ -75,6 +75,16 @@ body { height: auto; } +.channel-profile > .channel-name-pronouns { + display: inline-block; +} + +.channel-profile > .channel-name-pronouns > .channel-pronouns { + font-style: italic; + font-size: .8em; + font-weight: lighter; +} + body a.channel-owner { background-color: #008bec; color: #fff; @@ -406,7 +416,12 @@ input[type="search"]::-webkit-search-cancel-button { p.channel-name { margin: 0; overflow-wrap: anywhere;} p.video-data { margin: 0; font-weight: bold; font-size: 80%; } -.channel-profile > .channel-name { overflow-wrap: anywhere;} + +.channel-profile > .channel-name, +.channel-profile > .channel-name-pronouns > .channel-name +{ + overflow-wrap: anywhere; +} /* diff --git a/config/config.example.yml b/config/config.example.yml index eedd9539..f3f43bba 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -8,6 +8,13 @@ ## Database configuration with separate parameters. ## This setting is MANDATORY, unless 'database_url' is used. ## +## Note: The 'db' setting allows the use of UNIX +## sockets. To do so, set 'host' to "" +## E.g: +## password: kemal +## host: "" +## port: 5432 +## db: user: kemal password: kemal @@ -223,9 +230,13 @@ https_only: false ## ## Configuration for using a HTTP proxy -## ## If unset, then no HTTP proxy will be used. +## Proxy type supported: HTTP, HTTPS ## +## This is not used for loading the video streams from YouTube servers (circumvent YouTube restrictions) +## Please instead configure the proxy in Invidious companion: +## https://github.com/iv-org/invidious-companion/blob/master/config/config.example.toml +## #http_proxy: # user: # password: diff --git a/shard.yml b/shard.yml index bc6c4bf4..dde1851e 100644 --- a/shard.yml +++ b/shard.yml @@ -5,6 +5,10 @@ authors: - Invidious team - Contributors! +targets: + invidious: + main: src/invidious.cr + description: | Invidious is an alternative front-end to YouTube diff --git a/src/invidious.cr b/src/invidious.cr index a61f91a9..d7c5b80b 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -67,20 +67,9 @@ rescue ex puts "Check your 'config.yml' database settings or PostgreSQL settings." exit(1) end -ARCHIVE_URL = URI.parse("https://archive.org") -PUBSUB_URL = URI.parse("https://pubsubhubbub.appspot.com") -REDDIT_URL = URI.parse("https://www.reddit.com") -YT_URL = URI.parse("https://www.youtube.com") -HOST_URL = make_host_url(Kemal.config) - -CHARS_SAFE = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" -TEST_IDS = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"} +HOST_URL = make_host_url(Kemal.config) MAX_ITEMS_PER_PAGE = 1500 -REQUEST_HEADERS_WHITELIST = {"accept", "accept-encoding", "cache-control", "content-length", "if-none-match", "range"} -RESPONSE_HEADERS_BLACKLIST = {"access-control-allow-origin", "alt-svc", "server"} -HTTP_CHUNK_SIZE = 10485760 # ~10MB - CURRENT_BRANCH = {{ "#{`git branch | sed -n '/* /s///p'`.strip}" }} CURRENT_COMMIT = {{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit`.strip}" }} CURRENT_VERSION = {{ "#{`git log -1 --format=%ci | awk '{print $1}' | sed s/-/./g`.strip}" }} @@ -97,7 +86,7 @@ SOFTWARE = { "branch" => "#{CURRENT_BRANCH}", } -YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size) +YT_POOL = YoutubeConnectionPool.new(URI.parse("https://www.youtube.com"), capacity: CONFIG.pool_size) # Image request pool diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr index 13909527..537aa034 100644 --- a/src/invidious/channels/about.cr +++ b/src/invidious/channels/about.cr @@ -12,6 +12,7 @@ record AboutChannel, sub_count : Int32, joined : Time, is_family_friendly : Bool, + pronouns : String?, allowed_regions : Array(String), tabs : Array(String), tags : Array(String), @@ -160,14 +161,21 @@ def get_about_info(ucid, locale) : AboutChannel end sub_count = 0 + pronouns = nil if (metadata_rows = initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "metadata", "contentMetadataViewModel", "metadataRows").try &.as_a) metadata_rows.each do |row| - metadata_part = row.dig?("metadataParts").try &.as_a.find { |i| i.dig?("text", "content").try &.as_s.includes?("subscribers") } - if !metadata_part.nil? - sub_count = short_text_to_number(metadata_part.dig("text", "content").as_s.split(" ")[0]).to_i32 + subscribe_metadata_part = row.dig?("metadataParts").try &.as_a.find { |i| i.dig?("text", "content").try &.as_s.includes?("subscribers") } + if !subscribe_metadata_part.nil? + sub_count = short_text_to_number(subscribe_metadata_part.dig("text", "content").as_s.split(" ")[0]).to_i32 end - break if sub_count != 0 + + pronoun_metadata_part = row.dig?("metadataParts").try &.as_a.find { |i| i.dig?("tooltip").try &.as_s.includes?("Pronouns") } + if !pronoun_metadata_part.nil? + pronouns = pronoun_metadata_part.dig("text", "content").as_s + end + + break if sub_count != 0 && !pronouns.nil? end end @@ -184,6 +192,7 @@ def get_about_info(ucid, locale) : AboutChannel sub_count: sub_count, joined: joined, is_family_friendly: is_family_friendly, + pronouns: pronouns, allowed_regions: allowed_regions, tabs: tab_names, tags: tags, diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index 96400f47..e2cc8305 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -114,7 +114,11 @@ module Invidious::Channel::Tabs "2:embedded" => { "1:string" => "00000000-0000-0000-0000-000000000000", }, - "4:varint" => sort_options_videos_short(sort_by), + "4:varint" => sort_options_videos_short(sort_by), + "8:embedded" => { + "1:string" => "00000000-0000-0000-0000-000000000000", + "3:varint" => sort_options_videos_short(sort_by), + }, }, } @@ -130,7 +134,11 @@ module Invidious::Channel::Tabs "2:embedded" => { "1:string" => "00000000-0000-0000-0000-000000000000", }, - "4:varint" => sort_options_videos_short(sort_by), + "4:varint" => sort_options_videos_short(sort_by), + "7:embedded" => { + "1:string" => "00000000-0000-0000-0000-000000000000", + "3:varint" => sort_options_videos_short(sort_by), + }, }, } @@ -154,7 +162,11 @@ module Invidious::Channel::Tabs "2:embedded" => { "1:string" => "00000000-0000-0000-0000-000000000000", }, - "5:varint" => sort_by_numerical, + "5:varint" => sort_by_numerical, + "8:embedded" => { + "1:string" => "00000000-0000-0000-0000-000000000000", + "3:varint" => sort_by_numerical, + }, }, } diff --git a/src/invidious/comments/reddit.cr b/src/invidious/comments/reddit.cr index ba9c19f1..e128350c 100644 --- a/src/invidious/comments/reddit.cr +++ b/src/invidious/comments/reddit.cr @@ -1,5 +1,6 @@ module Invidious::Comments extend self + private REDDIT_URL = URI.parse("https://www.reddit.com") def fetch_reddit(id, sort_by = "confidence") client = make_client(REDDIT_URL) diff --git a/src/invidious/frontend/watch_page.cr b/src/invidious/frontend/watch_page.cr index 14e169e8..642ab4cc 100644 --- a/src/invidious/frontend/watch_page.cr +++ b/src/invidious/frontend/watch_page.cr @@ -36,7 +36,7 @@ module Invidious::Frontend::WatchPage return String.build(4000) do |str| str << "" diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 6add0237..ab694b1f 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -1,5 +1,7 @@ require "./macros" +TEST_IDS = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"} + struct Nonce include DB::Serializable diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index 5637e533..24b20ed9 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -1,3 +1,5 @@ +PUBSUB_URL = URI.parse("https://pubsubhubbub.appspot.com") + # See http://www.evanmiller.org/how-not-to-sort-by-average-rating.html def ci_lower_bound(pos, n) if n == 0 diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index 7c584d15..eb084331 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -107,7 +107,11 @@ struct Playlist json.field "author", self.author json.field "authorId", self.ucid - json.field "authorUrl", "/channel/#{self.ucid}" + if !self.ucid.empty? + json.field "authorUrl", "/channel/#{self.ucid}" + else + json.field "authorUrl", "" + end json.field "subtitle", self.subtitle json.field "authorThumbnails" do @@ -359,6 +363,9 @@ def fetch_playlist(plid : String) thumbnail = playlist_info.dig?( "thumbnailRenderer", "playlistVideoThumbnailRenderer", "thumbnail", "thumbnails", 0, "url" + ).try &.as_s || playlist_info.dig?( + "thumbnailRenderer", "playlistCustomThumbnailRenderer", + "thumbnail", "thumbnails", 0, "url" ).try &.as_s views = 0_i64 diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index 503b8c05..f8060342 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -104,6 +104,7 @@ module Invidious::Routes::API::V1::Channels json.field "tabs", channel.tabs json.field "tags", channel.tags json.field "authorVerified", channel.verified + json.field "pronouns", channel.pronouns json.field "latestVideos" do json.array do diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index 6a3eb8ae..fc3de695 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -1,6 +1,9 @@ require "html" module Invidious::Routes::API::V1::Videos + private INTERNET_ARCHIVE_URL = URI.parse("https://archive.org") + private CHARS_SAFE = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" + def self.videos(env) locale = env.get("preferences").as(Preferences).locale @@ -279,7 +282,7 @@ module Invidious::Routes::API::V1::Videos file = URI.encode_www_form("#{id[0, 3]}/#{id}.xml") - location = make_client(ARCHIVE_URL, &.get("/download/youtubeannotations_#{index}/#{id[0, 2]}.tar/#{file}")) + location = make_client(INTERNET_ARCHIVE_URL, &.get("/download/youtubeannotations_#{index}/#{id[0, 2]}.tar/#{file}")) if !location.headers["Location"]? env.response.status_code = location.status_code diff --git a/src/invidious/routes/before_all.cr b/src/invidious/routes/before_all.cr index 63b935ec..6d374fff 100644 --- a/src/invidious/routes/before_all.cr +++ b/src/invidious/routes/before_all.cr @@ -1,4 +1,16 @@ module Invidious::Routes::BeforeAll + struct CompanionCSP + property companion_urls : String = "" + + def initialize + self.companion_urls = CONFIG.invidious_companion.reject(&.builtin_proxy).map do |companion| + "#{companion.public_url.scheme}://#{companion.public_url.host}#{companion.public_url.port ? ":#{companion.public_url.port}" : ""}" + end.join(" ") + end + end + + private COMPANION_CSP = CompanionCSP.new + def self.handle(env) preferences = Preferences.from_json("{}") @@ -35,9 +47,9 @@ module Invidious::Routes::BeforeAll "style-src 'self' 'unsafe-inline'", "img-src 'self' data:", "font-src 'self' data:", - "connect-src 'self'", + "connect-src 'self' " + COMPANION_CSP.companion_urls, "manifest-src 'self'", - "media-src 'self' blob:", + "media-src 'self' blob: " + COMPANION_CSP.companion_urls, "child-src 'self' blob:", "frame-src 'self'", "frame-ancestors " + frame_ancestors, @@ -94,8 +106,8 @@ module Invidious::Routes::BeforeAll end dark_mode = convert_theme(env.params.query["dark_mode"]?) || preferences.dark_mode.to_s - thin_mode = env.params.query["thin_mode"]? || preferences.thin_mode.to_s - thin_mode = thin_mode == "true" + thin_mode = env.params.query["thin_mode"]? + thin_mode = (thin_mode == "true") || preferences.thin_mode locale = env.params.query["hl"]? || preferences.locale preferences.dark_mode = dark_mode diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index f785de18..968d38dc 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -231,8 +231,10 @@ module Invidious::Routes::Channels env.redirect "/post/#{URI.encode_www_form(lb)}?ucid=#{URI.encode_www_form(ucid)}" end - thin_mode = env.params.query["thin_mode"]? || env.get("preferences").as(Preferences).thin_mode - thin_mode = thin_mode == "true" + preferences = env.get("preferences").as(Preferences) + + thin_mode = env.params.query["thin_mode"]? + thin_mode = (thin_mode == "true") || preferences.thin_mode continuation = env.params.query["continuation"]? diff --git a/src/invidious/routes/embed.cr b/src/invidious/routes/embed.cr index d0a3b5c1..ec5a5804 100644 --- a/src/invidious/routes/embed.cr +++ b/src/invidious/routes/embed.cr @@ -208,17 +208,6 @@ module Invidious::Routes::Embed if CONFIG.invidious_companion.present? invidious_companion = CONFIG.invidious_companion.sample - invidious_companion_urls = CONFIG.invidious_companion.reject(&.builtin_proxy).map do |companion| - uri = - "#{companion.public_url.scheme}://#{companion.public_url.host}#{companion.public_url.port ? ":#{companion.public_url.port}" : ""}" - end.join(" ") - - if !invidious_companion_urls.empty? - env.response.headers["Content-Security-Policy"] = - env.response.headers["Content-Security-Policy"] - .gsub("media-src", "media-src #{invidious_companion_urls}") - .gsub("connect-src", "connect-src #{invidious_companion_urls}") - end end rendered "embed" diff --git a/src/invidious/routes/routes.cr b/src/invidious/routes/routes.cr new file mode 100644 index 00000000..68b1ff82 --- /dev/null +++ b/src/invidious/routes/routes.cr @@ -0,0 +1,20 @@ +module Invidious::Routes + private REQUEST_HEADERS_WHITELIST = { + "accept", + "accept-encoding", + "cache-control", + "content-length", + "if-none-match", + "range", + } + private RESPONSE_HEADERS_BLACKLIST = { + "access-control-allow-origin", + "alt-svc", + "server", + "cross-origin-opener-policy-report-only", + "report-to", + "cross-origin", + "timing-allow-origin", + "cross-origin-resource-policy", + } +end diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index 083087a9..7c01aa36 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -1,4 +1,6 @@ module Invidious::Routes::VideoPlayback + private HTTP_CHUNK_SIZE = 10485760 # ~10MB + # /videoplayback def self.get_video_playback(env) locale = env.get("preferences").as(Preferences).locale diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index 4c181503..b829b0f5 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -193,17 +193,6 @@ module Invidious::Routes::Watch if CONFIG.invidious_companion.present? invidious_companion = CONFIG.invidious_companion.sample - invidious_companion_urls = CONFIG.invidious_companion.reject(&.builtin_proxy).map do |companion| - uri = - "#{companion.public_url.scheme}://#{companion.public_url.host}#{companion.public_url.port ? ":#{companion.public_url.port}" : ""}" - end.join(" ") - - if !invidious_companion_urls.empty? - env.response.headers["Content-Security-Policy"] = - env.response.headers["Content-Security-Policy"] - .gsub("media-src", "media-src #{invidious_companion_urls}") - .gsub("connect-src", "connect-src #{invidious_companion_urls}") - end end templated "watch" diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr index 007eb666..7c4101cc 100644 --- a/src/invidious/user/imports.cr +++ b/src/invidious/user/imports.cr @@ -30,28 +30,24 @@ struct Invidious::User return subscriptions end - def parse_playlist_export_csv(user : User, raw_input : String) + # Parse a CSV Google Takeout - Youtube Playlist file + def parse_playlist_export_csv(user : User, playlist_name : String, raw_input : String) # Split the input into head and body content - raw_head, raw_body = raw_input.strip('\n').split("\n\n", limit: 2, remove_empty: true) + raw_head, raw_body = raw_input.split("\n", limit: 2, remove_empty: true) # Create the playlist from the head content csv_head = CSV.new(raw_head.strip('\n'), headers: true) csv_head.next - title = csv_head[4] - description = csv_head[5] - visibility = csv_head[6] + title = playlist_name - if visibility.compare("Public", case_insensitive: true) == 0 - privacy = PlaylistPrivacy::Public - else - privacy = PlaylistPrivacy::Private - end + description = "This is the default description of an imported playlist. Feel Free to change it as you see fit." + privacy = PlaylistPrivacy::Private playlist = create_playlist(title, privacy, user) Invidious::Database::Playlists.update_description(playlist.id, description) # Add each video to the playlist from the body content - csv_body = CSV.new(raw_body.strip('\n'), headers: true) + csv_body = CSV.new(raw_body.strip('\n'), headers: false) csv_body.each do |row| video_id = row[0] if playlist @@ -204,10 +200,12 @@ struct Invidious::User end def from_youtube_pl(user : User, body : String, filename : String, type : String) : Bool - extension = filename.split(".").last + filename_array = filename.split(".") + playlist_name = filename_array.first + extension = filename_array.last if extension == "csv" || type == "text/csv" - playlist = parse_playlist_export_csv(user, body) + playlist = parse_playlist_export_csv(user, playlist_name, body) if playlist return true else diff --git a/src/invidious/views/components/channel_info.ecr b/src/invidious/views/components/channel_info.ecr index 2c177b59..97a2d7da 100644 --- a/src/invidious/views/components/channel_info.ecr +++ b/src/invidious/views/components/channel_info.ecr @@ -12,7 +12,10 @@
- <%= author %><% if !channel.verified.nil? && channel.verified %> <% end %> +
+ <%= author %><% if !channel.verified.nil? && channel.verified %> <% end %> + <% if !channel.pronouns.nil? %>
<%= channel.pronouns %><% end %> +