From c0a6587d5255de332a07b72a6240b93bf97fe60f Mon Sep 17 00:00:00 2001 From: NeskireDK <10115530+NeskireDK@users.noreply.github.com> Date: Thu, 13 Aug 2026 06:14:01 +0200 Subject: [PATCH] Store the fork's settings in Postgres, overriding the config Production feeds the config through INVIDIOUS_CONFIG, so a config.yml written at runtime is thrown away on the next restart. The fork's own settings therefore need a home the admin can write to: a key/value table arik_settings, one row per setting. - The environment config is parsed and validated first and seeds every value; a stored row then overrides the value of its own key. A key without a row keeps the environment value. - Applied at boot right after the table integrity check, so Config.check still fails closed on a bad environment config while a bad database row only loses its own override: decoding reports the reason and the environment value stands. - Validation lives in ArikSettings and is pure, so the admin UI can refuse an entry before it is stored. It refuses what hurts later: a CIDR range in trusted_proxies (Config.check exits on it at the next boot), header auth enabled with no trusted proxy at all, and callback origins that are not a bare scheme+host+port. - Two new trusted_header_auth fields are declared here and wired up in the commits that follow: password_self_service and auto_approve_token_callbacks. The table is created both by a migration and by check_integrity, so instances on either path get it. Co-Authored-By: Claude Fable 5 --- config/config.example.yml | 22 + config/sql/arik_settings.sql | 13 + src/invidious.cr | 6 + src/invidious/arik_settings.cr | 385 ++++++++++++++++++ src/invidious/config.cr | 17 + src/invidious/database/base.cr | 4 + .../0011_create_arik_settings_table.cr | 17 + 7 files changed, 464 insertions(+) create mode 100644 config/sql/arik_settings.sql create mode 100644 src/invidious/arik_settings.cr create mode 100644 src/invidious/database/migrations/0011_create_arik_settings_table.cr diff --git a/config/config.example.yml b/config/config.example.yml index 5b4a14662..8ae6bc5ce 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -382,6 +382,17 @@ https_only: false ## WARNING: the reverse proxy MUST strip this header from client ## requests on every route that bypasses its authentication. ## +## password_self_service lets such a session set a password without +## typing the current one — those accounts were given a random password +## nobody ever saw. Set it to false to require the current password. +## +## auto_approve_token_callbacks lists exact origins (scheme, host and +## optional port, nothing else) whose token authorization requests skip +## the consent page, so a client behind the same proxy signs in without +## a click. Only sessions established by the trusted header qualify, and +## the callback origin must match an entry whole. Empty = every client +## sees the consent page. +## ## Default: disabled ## #trusted_header_auth: @@ -390,6 +401,9 @@ https_only: false # trusted_proxies: # - 192.168.1.101 # logout_url: "https://auth.example.com/logout" +# password_self_service: true +# auto_approve_token_callbacks: +# - "https://yt.example.com" ## ## Playlist-backed feeds (ArikTube extension). @@ -409,6 +423,14 @@ https_only: false #popular_playlists: # - IVPLxxxxxxxxxxxxxxxxxxxx +## +## NOTE: the two settings above and trusted_header_auth can also be +## edited at runtime, on the ArikTube settings page an administrator +## reaches from /preferences. Those edits are stored in the database +## (table arik_settings) and override what is written here, because a +## config passed through INVIDIOUS_CONFIG can't be written back. +## + ## ## Enable/Disable the captcha challenge on the login page. ## diff --git a/config/sql/arik_settings.sql b/config/sql/arik_settings.sql new file mode 100644 index 000000000..d6c997d3e --- /dev/null +++ b/config/sql/arik_settings.sql @@ -0,0 +1,13 @@ +-- Table: public.arik_settings + +-- DROP TABLE public.arik_settings; + +CREATE TABLE IF NOT EXISTS public.arik_settings +( + key text NOT NULL, + value jsonb NOT NULL, + updated timestamp with time zone, + CONSTRAINT arik_settings_key_key PRIMARY KEY (key) +); + +GRANT ALL ON TABLE public.arik_settings TO current_user; diff --git a/src/invidious.cr b/src/invidious.cr index 60e7ab6c1..7505696a2 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -145,6 +145,12 @@ LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level, CONFIG.colorize_log # Check table integrity Invidious::Database.check_integrity(CONFIG) +# ArikTube: the environment config has seeded every value, now let the +# database override the keys the admin edited in the web UI. Deliberately +# after Config.load, so its fail-closed checks still run on the environment +# config, and after the integrity check, so the table is there to read. +Invidious::ArikSettings.apply_overrides!(CONFIG) + {% if !flag?(:skip_videojs_download) %} # Resolve player dependencies. This is done at compile time. # diff --git a/src/invidious/arik_settings.cr b/src/invidious/arik_settings.cr new file mode 100644 index 000000000..69e6cd69a --- /dev/null +++ b/src/invidious/arik_settings.cr @@ -0,0 +1,385 @@ +require "json" +require "socket" +require "uri" + +# Runtime-configurable settings (ArikTube extension). +# +# The fork's own knobs — playlist-backed feeds and trusted-header auth — are +# read from the YAML config. Production passes that config through the +# INVIDIOUS_CONFIG environment variable, so a `config.yml` written at runtime +# is thrown away on the next restart. These overrides live in the +# `arik_settings` table instead: +# +# 1. the environment config is parsed first and seeds every value, +# 2. a row in `arik_settings` overrides the value of its own key, +# 3. the admin settings page writes the row and applies it to the running +# CONFIG, so a save takes effect without a restart. +# +# A row that is malformed or fails validation is reported and ignored, and the +# environment config stays authoritative for that key. A bad write can +# therefore never keep the instance from booting. +module Invidious::ArikSettings + extend self + + # One row per key. A key without a row keeps the environment config value. + KEY_POPULAR_PLAYLISTS = "popular_playlists" + KEY_TRENDING_PLAYLISTS = "trending_playlists" + KEY_TRUSTED_HEADER_AUTH = "trusted_header_auth" + + # Playlist IDs are opaque strings; this accepts both the local "IVPL…" and + # the YouTube "PL…" shapes while rejecting anything with separators in it. + PLAYLIST_ID_REGEX = /\A[A-Za-z0-9_-]{2,64}\z/ + + # ------------------------------------------------------------------ + # Validation (pure: no database, no configuration, no logger) + # ------------------------------------------------------------------ + + # Returns the reason `address` is not usable as a trusted proxy entry. + # + # A CIDR range is called out separately because it is the mistake that + # hurts: `Config.check` rejects it and the process exits on the next boot, + # long after the admin saved it here. + def trusted_proxy_error(address : String) : String? + value = address.strip + + return "A trusted proxy entry can't be empty" if value.empty? + + if value.includes?('/') + return "Trusted proxy '#{value}' looks like a CIDR range. \ + Literal IP addresses only — a range stops the instance from starting." + end + + return "Trusted proxy '#{value}' is not a valid IP address" if !Socket::IPAddress.valid?(value) + + nil + end + + # Returns the reason `origin` is not usable as an allowed callback origin. + # + # An entry is an origin and nothing else, so that the comparison against a + # callback URL stays a whole-value match and can't be widened by a path. + def origin_error(origin : String) : String? + value = origin.strip + + return "An allowed callback origin can't be empty" if value.empty? + + uri = begin + URI.parse(value) + rescue + return "'#{value}' is not a valid URL" + end + + scheme = uri.scheme.try &.downcase + if scheme != "http" && scheme != "https" + return "'#{value}' needs an http:// or https:// scheme" + end + + host = uri.host + return "'#{value}' needs a host name" if host.nil? || host.empty? + + if uri.user || uri.password + return "'#{value}' must not carry credentials" + end + + if !uri.path.empty? && uri.path != "/" + return "'#{value}' must be an origin only (scheme, host and optional port), without a path" + end + + return "'#{value}' must not carry a query string" if uri.query + return "'#{value}' must not carry a fragment" if uri.fragment + + nil + end + + # "scheme://host[:port]" of `url`, lowercased, with the scheme's default + # port dropped. Returns nil when `url` has no usable origin. + # + # A URL that carries credentials is refused outright: "https://good@evil.tld" + # is the classic way to make an origin look like something it is not. + def normalize_origin(url : String) : String? + uri = begin + URI.parse(url.strip) + rescue + return nil + end + + scheme = uri.scheme.try &.downcase + return nil if scheme != "http" && scheme != "https" + + host = uri.host.try &.downcase + return nil if host.nil? || host.empty? + + return nil if uri.user || uri.password + + port = uri.port + port = nil if port == (scheme == "https" ? 443 : 80) + + port ? "#{scheme}://#{host}:#{port}" : "#{scheme}://#{host}" + end + + # True when the origin of `callback_url` is literally one of `allowed`. + # + # Both sides are normalized and then compared whole, so neither a prefix + # ("https://example.com.evil.tld") nor a path ("https://evil.tld/example.com") + # can ever match an allowed entry. + def origin_allowed?(callback_url : String?, allowed : Array(String)) : Bool + return false if allowed.empty? + return false if callback_url.nil? + + origin = normalize_origin(callback_url) + return false if origin.nil? + + allowed.any? { |entry| normalize_origin(entry) == origin } + end + + # Cleans a list of playlist IDs: blanks dropped, order kept, duplicates + # dropped. Returns the cleaned list and a message for every entry refused. + def clean_playlist_ids(entries : Array(String)) : {Array(String), Array(String)} + cleaned = [] of String + errors = [] of String + + entries.each do |entry| + plid = entry.strip + next if plid.empty? + + if !PLAYLIST_ID_REGEX.matches?(plid) + errors << "'#{plid}' is not a valid playlist ID" + next + end + + cleaned << plid if !cleaned.includes?(plid) + end + + {cleaned, errors} + end + + # The current-password waiver for a `/change_password` request. + # + # The trusted header must assert the very identity the session belongs to, + # and the admin must not have turned self service off. + def password_self_service?(flag : Bool, asserted_email : String?, user_email : String) : Bool + return false if !flag + return false if asserted_email.nil? + + asserted_email == user_email + end + + # Whether a token authorization request may skip the consent page. + # + # Every condition has to hold: header auth is on, the admin listed at least + # one origin, this session was established by the trusted header for this + # very user, and the callback lands on one of the listed origins. + def auto_approve_token?( + enabled : Bool, + allowed : Array(String), + asserted_email : String?, + user_email : String, + callback_url : String?, + ) : Bool + return false if !enabled + return false if allowed.empty? + return false if asserted_email.nil? || asserted_email != user_email + + origin_allowed?(callback_url, allowed) + end + + # ------------------------------------------------------------------ + # Stored shape + # ------------------------------------------------------------------ + + # The trusted-header block as it is stored and edited. + # + # Deliberately not `TrustedHeaderAuthConfig` itself: a stored block is + # untrusted input and has to survive validation before it is allowed + # anywhere near the running configuration. + struct TrustedHeaderAuthSettings + include JSON::Serializable + + property enabled : Bool = false + property header : String = "Remote-User" + property trusted_proxies : Array(String) = [] of String + # "" means "no proxy logout URL", so the local sign-out form is used + property logout_url : String = "" + property password_self_service : Bool = true + property auto_approve_token_callbacks : Array(String) = [] of String + + def initialize( + @enabled : Bool = false, + @header : String = "Remote-User", + @trusted_proxies : Array(String) = [] of String, + @logout_url : String = "", + @password_self_service : Bool = true, + @auto_approve_token_callbacks : Array(String) = [] of String, + ) + end + + # Snapshot of a running `TrustedHeaderAuthConfig` + def self.from_config(config) : self + new( + enabled: config.enabled, + header: config.header, + trusted_proxies: config.trusted_proxies.dup, + logout_url: config.logout_url || "", + password_self_service: config.password_self_service, + auto_approve_token_callbacks: config.auto_approve_token_callbacks.dup, + ) + end + + # Everything wrong with this block, empty when it is safe to store. + def errors : Array(String) + messages = [] of String + + if @enabled + if @header.strip.empty? + messages << "The header name can't be empty while trusted-header authentication is on" + end + + if @trusted_proxies.none? { |address| !address.strip.empty? } + messages << "Trusted-header authentication needs at least one trusted proxy IP. \ + Without one the header would be honored from anywhere." + end + end + + # Checked even while disabled: a bad entry left behind here would stop + # the instance from booting the moment somebody turns the feature on. + @trusted_proxies.each do |address| + if error = ArikSettings.trusted_proxy_error(address) + messages << error + end + end + + logout_url = @logout_url.strip + if !logout_url.empty? + uri = URI.parse(logout_url) rescue nil + scheme = uri.try &.scheme.try &.downcase + if uri.nil? || (scheme != "http" && scheme != "https") || uri.host.to_s.empty? + messages << "The logout URL must be an absolute http:// or https:// URL" + end + end + + @auto_approve_token_callbacks.each do |origin| + if error = ArikSettings.origin_error(origin) + messages << error + end + end + + messages + end + + # Normalized copy: blanks dropped, values trimmed. Call before storing. + def cleaned : self + self.class.new( + enabled: @enabled, + header: @header.strip.presence || "Remote-User", + trusted_proxies: @trusted_proxies.map(&.strip).reject(&.empty?).uniq, + logout_url: @logout_url.strip, + password_self_service: @password_self_service, + auto_approve_token_callbacks: @auto_approve_token_callbacks.map(&.strip).reject(&.empty?).uniq, + ) + end + + # Copies this block onto a running `TrustedHeaderAuthConfig`. + def apply_to(config) : Nil + config.enabled = @enabled + config.header = @header + config.trusted_proxies = @trusted_proxies + config.logout_url = @logout_url.empty? ? nil : @logout_url + config.password_self_service = @password_self_service + config.auto_approve_token_callbacks = @auto_approve_token_callbacks + end + end + + # ------------------------------------------------------------------ + # Decoding (pure: takes the stored text, never touches the database) + # ------------------------------------------------------------------ + + # Decodes a stored playlist list. Returns the list and nil, or nil and the + # reason the row was ignored — an exception never leaves here, because a + # single bad row must not stop the instance from booting. + def decode_playlists(raw : String?) : {Array(String)?, String?} + return {nil, nil} if raw.nil? + + plids = Array(String).from_json(raw) + cleaned, errors = clean_playlist_ids(plids) + return {nil, errors.join("; ")} if !errors.empty? + + {cleaned, nil} + rescue ex + {nil, "not a JSON list of playlist IDs (#{ex.message})"} + end + + # Decodes a stored trusted-header block. An invalid block is refused here, + # not at boot: `Config.check` already exited for a bad environment config, + # and a bad database row must never do the same. + def decode_trusted_header_auth(raw : String?) : {TrustedHeaderAuthSettings?, String?} + return {nil, nil} if raw.nil? + + settings = TrustedHeaderAuthSettings.from_json(raw) + errors = settings.errors + return {nil, errors.join("; ")} if !errors.empty? + + {settings, nil} + rescue ex + {nil, "not a valid trusted_header_auth block (#{ex.message})"} + end + + # Applies decoded overrides over `config`. A nil override leaves the key + # alone, which is what makes the environment config the default and the + # database the override. + def merge_overrides!( + config, + popular_playlists : Array(String)?, + trending_playlists : Array(String)?, + trusted_header_auth : TrustedHeaderAuthSettings?, + ) : Nil + config.popular_playlists = popular_playlists if popular_playlists + config.trending_playlists = trending_playlists if trending_playlists + trusted_header_auth.try &.apply_to(config.trusted_header_auth) + end + + # ------------------------------------------------------------------ + # Persistence + # ------------------------------------------------------------------ + + # The stored JSON text of `key`, or nil when there is no row. A database + # that can't answer is reported and treated as "no override". + def fetch(key : String) : String? + PG_DB.query_one?("SELECT value::text FROM arik_settings WHERE key = $1", key, as: String) + rescue ex + LOGGER.error("ArikSettings: cannot read setting '#{key}' (#{ex.message})") + nil + end + + # Writes `json` as the value of `key`. Raises on failure so the caller can + # tell the admin the save did not happen. + def store(key : String, json : String) : Nil + request = <<-SQL + INSERT INTO arik_settings (key, value, updated) + VALUES ($1, $2::jsonb, now()) + ON CONFLICT (key) DO UPDATE + SET value = EXCLUDED.value, updated = EXCLUDED.updated + SQL + + PG_DB.exec(request, key, json) + end + + # Loads every override and applies it over `config`. Called once at boot, + # after the environment config is parsed and validated. + def apply_overrides!(config = CONFIG) : Nil + popular, popular_error = decode_playlists(fetch(KEY_POPULAR_PLAYLISTS)) + trending, trending_error = decode_playlists(fetch(KEY_TRENDING_PLAYLISTS)) + trusted_header_auth, tha_error = decode_trusted_header_auth(fetch(KEY_TRUSTED_HEADER_AUTH)) + + if error = popular_error + LOGGER.error("ArikSettings: ignoring stored '#{KEY_POPULAR_PLAYLISTS}' — #{error}") + end + if error = trending_error + LOGGER.error("ArikSettings: ignoring stored '#{KEY_TRENDING_PLAYLISTS}' — #{error}") + end + if error = tha_error + LOGGER.error("ArikSettings: ignoring stored '#{KEY_TRUSTED_HEADER_AUTH}' — #{error}") + end + + merge_overrides!(config, popular, trending, trusted_header_auth) + end +end diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 7f02bb792..2966c2345 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -93,6 +93,14 @@ struct TrustedHeaderAuthConfig property trusted_proxies : Array(String) = [] of String # Optional target for the sign-out link while header auth is active property logout_url : String? = nil + # Let a session the proxy vouches for set a password without typing the + # current one. Accounts provisioned here were given a random password + # nobody ever saw, so this is what makes them usable from native clients. + property password_self_service : Bool = true + # Exact origins (scheme, host, optional port) whose token authorization + # requests skip the consent page. Only for sessions established by the + # trusted header; empty means every client sees the consent page. + property auto_approve_token_callbacks : Array(String) = [] of String end class Config @@ -365,6 +373,15 @@ class Config end end + # The auto-approval list waives a consent screen, so a typo in it is + # refused loudly instead of silently never matching. + config.trusted_header_auth.auto_approve_token_callbacks.each do |origin| + if error = Invidious::ArikSettings.origin_error(origin) + puts "Config: trusted_header_auth.auto_approve_token_callbacks: #{error}" + exit(1) + end + end + # Check if the socket configuration is valid if sb = config.socket_binding if sb.path.ends_with?("/") || File.directory?(sb.path) diff --git a/src/invidious/database/base.cr b/src/invidious/database/base.cr index 0fb1b6af0..6209dbe9a 100644 --- a/src/invidious/database/base.cr +++ b/src/invidious/database/base.cr @@ -20,6 +20,10 @@ module Invidious::Database Invidious::Database.check_table("users", User) Invidious::Database.check_table("videos", Video) + # ArikTube: settings the admin edits at runtime. No struct is passed on + # purpose — the table is key/value and must not be reshaped from a struct. + Invidious::Database.check_table("arik_settings") + if cfg.cache_annotations Invidious::Database.check_table("annotations", Annotation) end diff --git a/src/invidious/database/migrations/0011_create_arik_settings_table.cr b/src/invidious/database/migrations/0011_create_arik_settings_table.cr new file mode 100644 index 000000000..f7f092c35 --- /dev/null +++ b/src/invidious/database/migrations/0011_create_arik_settings_table.cr @@ -0,0 +1,17 @@ +module Invidious::Database::Migrations + class CreateArikSettingsTable < Migration + version 11 + + def up(conn : DB::Connection) + conn.exec <<-SQL + CREATE TABLE IF NOT EXISTS public.arik_settings + ( + key text NOT NULL, + value jsonb NOT NULL, + updated timestamp with time zone, + CONSTRAINT arik_settings_key_key PRIMARY KEY (key) + ); + SQL + end + end +end