From fd2404b85c09c9ed2d9fe5d749bcc7ef0ab2ed1e Mon Sep 17 00:00:00 2001 From: Emilien <4016501+unixfox@users.noreply.github.com> Date: Sat, 3 May 2025 01:10:12 +0200 Subject: [PATCH 1/4] initial support for base_url with invidious companion + proxy invidious_companion --- config/config.example.yml | 6 ++-- src/invidious/routes/companion.cr | 37 +++++++++++++++++++++ src/invidious/routes/embed.cr | 8 +++-- src/invidious/routes/watch.cr | 8 +++-- src/invidious/routing.cr | 9 ++++- src/invidious/yt_backend/connection_pool.cr | 34 ++++++++++++++----- src/invidious/yt_backend/youtube_api.cr | 30 ++++++++++------- 7 files changed, 103 insertions(+), 29 deletions(-) create mode 100644 src/invidious/routes/companion.cr diff --git a/config/config.example.yml b/config/config.example.yml index 8d3e6212..d46fec89 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -75,7 +75,7 @@ db: ## If you are using a reverse proxy then you will probably need to ## configure the public_url to be the same as the domain used for Invidious. ## Also apply when used from an external IP address (without a domain). -## Examples: https://MYINVIDIOUSDOMAIN or http://192.168.1.100:8282 +## Examples: https://MYINVIDIOUSDOMAIN/companion or http://192.168.1.100:8282/companion ## ## Both parameter can have identical URL when Invidious is hosted in ## an internal network or at home or locally (localhost). @@ -84,8 +84,8 @@ db: ## Default: ## #invidious_companion: -# - private_url: "http://localhost:8282" -# public_url: "http://localhost:8282" +# - private_url: "http://localhost:8282/companion" +# public_url: "http://localhost:8282/companion" ## ## API key for Invidious companion, used for securing the communication diff --git a/src/invidious/routes/companion.cr b/src/invidious/routes/companion.cr new file mode 100644 index 00000000..23b62e9d --- /dev/null +++ b/src/invidious/routes/companion.cr @@ -0,0 +1,37 @@ +module Invidious::Routes::Companion + # /companion + def self.get_companion(env) + url = env.request.path.lchop("/companion") + + begin + COMPANION_POOL.client &.get(url, env.request.header) do |resp| + return self.proxy_companion(env, resp) + end + rescue ex + end + end + + def self.options_companion(env) + url = env.request.path.lchop("/companion") + + begin + COMPANION_POOL.client &.options(url, env.request.header) do |resp| + return self.proxy_companion(env, resp) + end + rescue ex + end + end + + private def self.proxy_companion(env, response) + env.response.status_code = response.status_code + response.headers.each do |key, value| + env.response.headers[key] = value + end + + if response.status_code >= 300 + return env.response.headers.delete("Transfer-Encoding") + end + + return proxy_file(response, env) + end +end diff --git a/src/invidious/routes/embed.cr b/src/invidious/routes/embed.cr index 930e4915..712295a3 100644 --- a/src/invidious/routes/embed.cr +++ b/src/invidious/routes/embed.cr @@ -209,10 +209,14 @@ module Invidious::Routes::Embed if CONFIG.invidious_companion.present? invidious_companion = CONFIG.invidious_companion.sample + invidious_companion_urls = CONFIG.invidious_companion.map do |companion| + uri = + "#{companion.public_url.scheme}://#{companion.public_url.host}#{companion.public_url.port ? ":#{companion.public_url.port}" : ""}" + end.join(" ") env.response.headers["Content-Security-Policy"] = env.response.headers["Content-Security-Policy"] - .gsub("media-src", "media-src #{invidious_companion.public_url}") - .gsub("connect-src", "connect-src #{invidious_companion.public_url}") + .gsub("media-src", "media-src #{invidious_companion_urls}") + .gsub("connect-src", "connect-src #{invidious_companion_urls}") end rendered "embed" diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index e777b3f1..a50a146d 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -194,10 +194,14 @@ module Invidious::Routes::Watch if CONFIG.invidious_companion.present? invidious_companion = CONFIG.invidious_companion.sample + invidious_companion_urls = CONFIG.invidious_companion.map do |companion| + uri = + "#{companion.public_url.scheme}://#{companion.public_url.host}#{companion.public_url.port ? ":#{companion.public_url.port}" : ""}" + end.join(" ") env.response.headers["Content-Security-Policy"] = env.response.headers["Content-Security-Policy"] - .gsub("media-src", "media-src #{invidious_companion.public_url}") - .gsub("connect-src", "connect-src #{invidious_companion.public_url}") + .gsub("media-src", "media-src #{invidious_companion_urls}") + .gsub("connect-src", "connect-src #{invidious_companion_urls}") end templated "watch" diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index 46b71f1f..b95ac706 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -188,7 +188,7 @@ module Invidious::Routing end # ------------------- - # Media proxy routes + # Proxy routes # ------------------- def register_api_manifest_routes @@ -223,6 +223,13 @@ module Invidious::Routing get "/vi/:id/:name", Routes::Images, :thumbnails end + def register_companion_routes + if CONFIG.invidious_companion.present? + get "/companion/*", Routes::Companion, :get_companion + options "/companion/*", Routes::Companion, :options_companion + end + end + # ------------------- # API routes # ------------------- diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index 0daed46c..97ce7c40 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -46,8 +46,22 @@ struct YoutubeConnectionPool end end +class CompanionWrapper + property client : HTTP::Client + property companion : Config::CompanionConfig + + def initialize(companion : Config::CompanionConfig) + @companion = companion + @client = HTTP::Client.new(companion.private_url) + end + + def close + @client.close + end +end + struct CompanionConnectionPool - property pool : DB::Pool(HTTP::Client) + property pool : DB::Pool(CompanionWrapper) def initialize(capacity = 5, timeout = 5.0) options = DB::Pool::Options.new( @@ -57,26 +71,28 @@ struct CompanionConnectionPool checkout_timeout: timeout ) - @pool = DB::Pool(HTTP::Client).new(options) do + @pool = DB::Pool(CompanionWrapper).new(options) do companion = CONFIG.invidious_companion.sample - next make_client(companion.private_url, use_http_proxy: false) + client = make_client(companion.private_url, use_http_proxy: false) + CompanionWrapper.new(companion: companion) end end def client(&) - conn = pool.checkout + wrapper = pool.checkout begin - response = yield conn + response = yield wrapper rescue ex - conn.close + wrapper.client.close companion = CONFIG.invidious_companion.sample - conn = make_client(companion.private_url, use_http_proxy: false) + client = make_client(companion.private_url, use_http_proxy: false) + wrapper = CompanionWrapper.new(companion: companion) - response = yield conn + response = yield wrapper ensure - pool.release(conn) + pool.release(wrapper) end response diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index b40092a1..455aa836 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -695,22 +695,28 @@ module YoutubeAPI # Send the POST request begin - response = COMPANION_POOL.client &.post(endpoint, headers: headers, body: data.to_json) - body = response.body - if (response.status_code != 200) - raise Exception.new( - "Error while communicating with Invidious companion: \ - status code: #{response.status_code} and body: #{body.dump}" - ) + response_body = "" + + COMPANION_POOL.client do |wrapper| + companion_base_url = wrapper.companion.private_url.path + puts "Using companion: #{wrapper.companion.private_url}" + + response = wrapper.client.post(companion_base_url + endpoint, headers: headers, body: data.to_json) + response_body = response.body + + if response.status_code != 200 + raise Exception.new( + "Error while communicating with Invidious companion: " \ + "status code: #{response.status_code} and body: #{response_body.dump}" + ) + end end + + # Convert result to Hash + return JSON.parse(response_body).as_h rescue ex raise InfoException.new("Error while communicating with Invidious companion: " + (ex.message || "no extra info found")) end - - # Convert result to Hash - initial_data = JSON.parse(body).as_h - - return initial_data end #################################################################### From a1d61e05ce60d51f6ed29bea9289e348cc9d5a73 Mon Sep 17 00:00:00 2001 From: Emilien <4016501+unixfox@users.noreply.github.com> Date: Sat, 14 Jun 2025 18:01:44 +0200 Subject: [PATCH 2/4] chore: add the suggestions --- src/invidious/routes/before_all.cr | 1 + src/invidious/routes/companion.cr | 6 +----- src/invidious/yt_backend/connection_pool.cr | 15 ++++++++++----- src/invidious/yt_backend/youtube_api.cr | 16 ++++------------ 4 files changed, 16 insertions(+), 22 deletions(-) diff --git a/src/invidious/routes/before_all.cr b/src/invidious/routes/before_all.cr index b5269668..63b935ec 100644 --- a/src/invidious/routes/before_all.cr +++ b/src/invidious/routes/before_all.cr @@ -63,6 +63,7 @@ module Invidious::Routes::BeforeAll "/videoplayback", "/latest_version", "/download", + "/companion/", }.any? { |r| env.request.resource.starts_with? r } if env.request.cookies.has_key? "SID" diff --git a/src/invidious/routes/companion.cr b/src/invidious/routes/companion.cr index 23b62e9d..cd7ed422 100644 --- a/src/invidious/routes/companion.cr +++ b/src/invidious/routes/companion.cr @@ -28,10 +28,6 @@ module Invidious::Routes::Companion env.response.headers[key] = value end - if response.status_code >= 300 - return env.response.headers.delete("Transfer-Encoding") - end - - return proxy_file(response, env) + return IO.copy response.body_io, env.response end end diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index 97ce7c40..45455a8a 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -46,13 +46,18 @@ struct YoutubeConnectionPool end end -class CompanionWrapper +# Packages a `HTTP::Client` to an Invidious companion instance alongside the configuration for that instance. +# +# This is used as the resource for the `CompanionPool` as to allow the ability to +# proxy the requests to Invidious companion from Invidious directly. +# Instead of setting up routes in a reverse proxy. +struct CompanionWrapper property client : HTTP::Client property companion : Config::CompanionConfig def initialize(companion : Config::CompanionConfig) @companion = companion - @client = HTTP::Client.new(companion.private_url) + @client = make_client(companion.private_url, use_http_proxy: false) end def close @@ -73,7 +78,7 @@ struct CompanionConnectionPool @pool = DB::Pool(CompanionWrapper).new(options) do companion = CONFIG.invidious_companion.sample - client = make_client(companion.private_url, use_http_proxy: false) + make_client(companion.private_url, use_http_proxy: false) CompanionWrapper.new(companion: companion) end end @@ -84,10 +89,10 @@ struct CompanionConnectionPool begin response = yield wrapper rescue ex - wrapper.client.close + wrapper.close companion = CONFIG.invidious_companion.sample - client = make_client(companion.private_url, use_http_proxy: false) + make_client(companion.private_url, use_http_proxy: false) wrapper = CompanionWrapper.new(companion: companion) response = yield wrapper diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 455aa836..b1775070 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -695,25 +695,17 @@ module YoutubeAPI # Send the POST request begin - response_body = "" + response_body = Hash(String, JSON::Any).new COMPANION_POOL.client do |wrapper| companion_base_url = wrapper.companion.private_url.path - puts "Using companion: #{wrapper.companion.private_url}" - response = wrapper.client.post(companion_base_url + endpoint, headers: headers, body: data.to_json) - response_body = response.body - - if response.status_code != 200 - raise Exception.new( - "Error while communicating with Invidious companion: " \ - "status code: #{response.status_code} and body: #{response_body.dump}" - ) + wrapper.client.post("#{companion_base_url}#{endpoint}", headers: headers, body: data.to_json) do | response | + response_body = JSON.parse(response.body_io).as_h end end - # Convert result to Hash - return JSON.parse(response_body).as_h + return response_body rescue ex raise InfoException.new("Error while communicating with Invidious companion: " + (ex.message || "no extra info found")) end From a47336365fc1bd8c671b2c490dc489ced06e3269 Mon Sep 17 00:00:00 2001 From: Emilien <4016501+unixfox@users.noreply.github.com> Date: Sat, 14 Jun 2025 19:10:52 +0200 Subject: [PATCH 3/4] fix csp + progress proxy + allow omit public_url --- config/config.example.yml | 8 ++++++++ src/invidious/config.cr | 11 +++++++++++ src/invidious/routes/companion.cr | 23 +++++++++++++++++------ src/invidious/routes/embed.cr | 13 ++++++++----- src/invidious/routes/watch.cr | 13 ++++++++----- src/invidious/routing.cr | 1 + 6 files changed, 53 insertions(+), 16 deletions(-) diff --git a/config/config.example.yml b/config/config.example.yml index d46fec89..6a2a71f8 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -80,12 +80,20 @@ db: ## Both parameter can have identical URL when Invidious is hosted in ## an internal network or at home or locally (localhost). ## +## NOTE: If public_url is omitted, Invidious will use its built-in proxy +## to route companion requests through /companion, which is useful for +## simple setups where companion runs on the same network. When using +## the built-in proxy, CSP headers are not modified since requests +## stay within the same domain. +## ## Accepted values: "http(s)://:" ## Default: ## #invidious_companion: # - private_url: "http://localhost:8282/companion" # public_url: "http://localhost:8282/companion" +# # Example with built-in proxy (omit public_url): +# # - private_url: "http://localhost:8282/companion" ## ## API key for Invidious companion, used for securing the communication diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 4d69854c..e47e405c 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -82,6 +82,9 @@ class Config @[YAML::Field(converter: Preferences::URIConverter)] property public_url : URI = URI.parse("") + + # Indicates if this companion instance uses the built-in proxy + property builtin_proxy : Bool = false end # Number of threads to use for crawling videos from channels (for updating subscriptions) @@ -271,6 +274,14 @@ class Config puts "Config: The value of 'invidious_companion_key' needs to be a size of 16 characters." exit(1) end + + # Set public_url to built-in proxy path when omitted + config.invidious_companion.each do |companion| + if companion.public_url.to_s.empty? + companion.public_url = URI.parse("/companion") + companion.builtin_proxy = true + end + end elsif config.signature_server puts("WARNING: inv-sig-helper is deprecated. Please switch to Invidious companion: https://docs.invidious.io/companion-installation/") else diff --git a/src/invidious/routes/companion.cr b/src/invidious/routes/companion.cr index cd7ed422..bcfbad6b 100644 --- a/src/invidious/routes/companion.cr +++ b/src/invidious/routes/companion.cr @@ -1,22 +1,33 @@ module Invidious::Routes::Companion # /companion def self.get_companion(env) - url = env.request.path.lchop("/companion") + url = env.request.path + if env.request.query + url += "?#{env.request.query}" + end begin - COMPANION_POOL.client &.get(url, env.request.header) do |resp| - return self.proxy_companion(env, resp) + COMPANION_POOL.client do |wrapper| + puts env.request.headers + wrapper.client.get(url, env.request.headers) do |resp| + return self.proxy_companion(env, resp) + end end rescue ex end end def self.options_companion(env) - url = env.request.path.lchop("/companion") + url = env.request.path + if env.request.query + url += "?#{env.request.query}" + end begin - COMPANION_POOL.client &.options(url, env.request.header) do |resp| - return self.proxy_companion(env, resp) + COMPANION_POOL.client do |wrapper| + wrapper.client.options(url, env.request.headers) do |resp| + return self.proxy_companion(env, resp) + end end rescue ex end diff --git a/src/invidious/routes/embed.cr b/src/invidious/routes/embed.cr index 712295a3..53da7068 100644 --- a/src/invidious/routes/embed.cr +++ b/src/invidious/routes/embed.cr @@ -209,14 +209,17 @@ module Invidious::Routes::Embed if CONFIG.invidious_companion.present? invidious_companion = CONFIG.invidious_companion.sample - invidious_companion_urls = CONFIG.invidious_companion.map do |companion| + invidious_companion_urls = CONFIG.invidious_companion.reject(&.builtin_proxy).map do |companion| uri = "#{companion.public_url.scheme}://#{companion.public_url.host}#{companion.public_url.port ? ":#{companion.public_url.port}" : ""}" end.join(" ") - env.response.headers["Content-Security-Policy"] = - env.response.headers["Content-Security-Policy"] - .gsub("media-src", "media-src #{invidious_companion_urls}") - .gsub("connect-src", "connect-src #{invidious_companion_urls}") + + if !invidious_companion_urls.empty? + env.response.headers["Content-Security-Policy"] = + env.response.headers["Content-Security-Policy"] + .gsub("media-src", "media-src #{invidious_companion_urls}") + .gsub("connect-src", "connect-src #{invidious_companion_urls}") + end end rendered "embed" diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index a50a146d..83893457 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -194,14 +194,17 @@ module Invidious::Routes::Watch if CONFIG.invidious_companion.present? invidious_companion = CONFIG.invidious_companion.sample - invidious_companion_urls = CONFIG.invidious_companion.map do |companion| + invidious_companion_urls = CONFIG.invidious_companion.reject(&.builtin_proxy).map do |companion| uri = "#{companion.public_url.scheme}://#{companion.public_url.host}#{companion.public_url.port ? ":#{companion.public_url.port}" : ""}" end.join(" ") - env.response.headers["Content-Security-Policy"] = - env.response.headers["Content-Security-Policy"] - .gsub("media-src", "media-src #{invidious_companion_urls}") - .gsub("connect-src", "connect-src #{invidious_companion_urls}") + + if !invidious_companion_urls.empty? + env.response.headers["Content-Security-Policy"] = + env.response.headers["Content-Security-Policy"] + .gsub("media-src", "media-src #{invidious_companion_urls}") + .gsub("connect-src", "connect-src #{invidious_companion_urls}") + end end templated "watch" diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index b95ac706..a51bb4b6 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -46,6 +46,7 @@ module Invidious::Routing self.register_api_v1_routes self.register_api_manifest_routes self.register_video_playback_routes + self.register_companion_routes end # ------------------- From 65fee592344be7c2df4f2ddd82e2faa7bd05e97e Mon Sep 17 00:00:00 2001 From: Emilien <4016501+unixfox@users.noreply.github.com> Date: Sat, 14 Jun 2025 22:35:11 +0200 Subject: [PATCH 4/4] fix formatting --- src/invidious/routes/embed.cr | 2 +- src/invidious/routes/watch.cr | 2 +- src/invidious/yt_backend/connection_pool.cr | 2 +- src/invidious/yt_backend/youtube_api.cr | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/invidious/routes/embed.cr b/src/invidious/routes/embed.cr index 53da7068..f04302a2 100644 --- a/src/invidious/routes/embed.cr +++ b/src/invidious/routes/embed.cr @@ -213,7 +213,7 @@ module Invidious::Routes::Embed uri = "#{companion.public_url.scheme}://#{companion.public_url.host}#{companion.public_url.port ? ":#{companion.public_url.port}" : ""}" end.join(" ") - + if !invidious_companion_urls.empty? env.response.headers["Content-Security-Policy"] = env.response.headers["Content-Security-Policy"] diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index 83893457..8a4fa246 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -198,7 +198,7 @@ module Invidious::Routes::Watch uri = "#{companion.public_url.scheme}://#{companion.public_url.host}#{companion.public_url.port ? ":#{companion.public_url.port}" : ""}" end.join(" ") - + if !invidious_companion_urls.empty? env.response.headers["Content-Security-Policy"] = env.response.headers["Content-Security-Policy"] diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index 45455a8a..42241d15 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -47,7 +47,7 @@ struct YoutubeConnectionPool end # Packages a `HTTP::Client` to an Invidious companion instance alongside the configuration for that instance. -# +# # This is used as the resource for the `CompanionPool` as to allow the ability to # proxy the requests to Invidious companion from Invidious directly. # Instead of setting up routes in a reverse proxy. diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index b1775070..44861c37 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -700,7 +700,7 @@ module YoutubeAPI COMPANION_POOL.client do |wrapper| companion_base_url = wrapper.companion.private_url.path - wrapper.client.post("#{companion_base_url}#{endpoint}", headers: headers, body: data.to_json) do | response | + wrapper.client.post("#{companion_base_url}#{endpoint}", headers: headers, body: data.to_json) do |response| response_body = JSON.parse(response.body_io).as_h end end