diff --git a/.github/workflows/build-nightly-container.yml b/.github/workflows/build-nightly-container.yml index ba005d9a..44be0bae 100644 --- a/.github/workflows/build-nightly-container.yml +++ b/.github/workflows/build-nightly-container.yml @@ -36,7 +36,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 diff --git a/.github/workflows/build-stable-container.yml b/.github/workflows/build-stable-container.yml index 1423bb69..e119880d 100644 --- a/.github/workflows/build-stable-container.yml +++ b/.github/workflows/build-stable-container.yml @@ -27,7 +27,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@v6 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 94bcbcfb..847342f7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,17 +38,17 @@ jobs: matrix: stable: [true] crystal: - - 1.12.2 - - 1.13.3 - 1.14.1 - 1.15.1 - 1.16.3 + - 1.17.1 + - 1.18.2 include: - crystal: nightly stable: false steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: submodules: true @@ -63,7 +63,7 @@ jobs: crystal: ${{ matrix.crystal }} - name: Cache Shards - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: | ./lib @@ -96,7 +96,7 @@ jobs: runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: Use ARM64 Dockerfile if ARM64 if: ${{ matrix.name == 'ARM64' }} @@ -128,7 +128,7 @@ jobs: continue-on-error: true steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 with: submodules: true @@ -139,7 +139,7 @@ jobs: crystal: latest - name: Cache Shards - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: | ./lib diff --git a/CHANGELOG.md b/CHANGELOG.md index fe0c7a1a..86e1511c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,96 @@ ## vX.Y.0 (future) +## v2.20260207.0 + +### Wrap-up + +This release hardens the Invidious companion pipeline and cleans up a long list of UI papercuts. Companion downloads now work end-to-end, CSP headers and check identifiers are generated once and reused, proxy responses strip stray headers, and the final traces of the legacy signature helper are gone so the helper can be rolled out safely. + +Livestream navigation, playlists, and channel metadata also see overdue fixes: Trending once again lists livestreams, "Watch on YouTube" buttons stop jumping to arbitrary timestamps, playlist imports/API calls handle missing data, and channel pages now display creator pronouns and playlist thumbnails. Deployments benefit from compiling OpenSSL into docker images to mitigate a long-standing memory leak observed with Alpine-provided OpenSSL, Crystal pinned back to 1.16.3 for docker and OCI builds, a rewritten static file handler, clarified README/HTTP proxy/unix socket docs, and dozens of smaller cleanups. + +### New features & important changes +#### For Users + - Livestream experiences are restored: Trending shows livestreams again, the gaming feed remains accessible, and "Watch on YouTube" links stop carrying stale timestamps (#5480, #5555, #5481) + - Channel and playlist metadata is richer thanks to pronoun support, topic playlist thumbnails, and accurate related video counts (#5617, #5616, #5446) + - Downloads get smoother because download actions are URL-safe and downloads can flow through Invidious companion when available (#5367, #5561) + - Users see clearer feedback with Erroneous CAPTCHA messages, DMCA controls restored, and a footer link pointing at the current release (#5508, #5228, #4702) + +#### For instance owners + - Companion integration is sturdier: CSP is generated once, check identifiers persist, and the helper hyperlink is fixed (#5497, #5575, #5491) + - Proxied images and videoplayback strip unwanted response headers (shared header-strip list) (#5595) + - Runtime and packaging updates pin docker/OCI builds to Crystal 1.16.3, bring an optional Crystal 1.18.2 + Alpine 3.23 image, and compile OpenSSL from source to mitigate the memory leak seen with Alpine-provided OpenSSL (#5604, #5577, #5574, #5441) + - Configuration docs saw polish with unix socket instructions, refreshed HTTP proxy comments, and corrected README commands (#5347, #5586, #5607) + - Server stability improves via a larger `max_request_line_size` that is required to be able to access some next pages of Youtube channels videos and a rewritten static file handler (#5566, #5338) + +#### For developers + - Top-level constants moved into dedicated modules, preferences handling was cleaned up, and the legacy signature helper is finally removed (#5596, #5450, #5550) + - Crystal API updates replaced the deprecated `Socket#blocking` property and restored the shard target plus SPDX license metadata (#5538, #5608, #5552) + - CI/tooling stayed current with newer GitHub Actions, install-crystal releases, and cache/checkout bumps (#5569, #5544, #5530, #5499) + +### Bugs fixed +#### User-side + - Playlist importer edge cases, playlist API author URLs, and channel continuation tokens now handle empty values without crashing (#4787, #5618, #5614) + - Thin mode community posts, posts that reference unavailable videos, and DMCA content toggles work again (#5567, #5549, #5228) + - UI cleanups prevent channel name/button overflow, show explicit Erroneous CAPTCHA errors, and keep livestream timestamps clean (#5553, #5452, #5508, #5481) + - Trending feeds and related video counts regained accuracy alongside livestream/gaming categories (#5555, #5480, #5446) + +#### For instance owners + - Companion downloads, CSP reuse, and check id generation behave predictably even under load (#5561, #5497, #5575) + - Proxy responses drop stray headers and HTTP proxy examples in the config were clarified (#5595, #5586) + - Docker/OCI builds were pinned to stable Crystal releases with OpenSSL bundled to avoid memory leaks (#5604, #5577, #5441) + +#### For developers + - README commit instructions, shard targets, and unix socket docs were corrected (#5607, #5608, #5347) + - Thin mode preference comparisons no longer convert unnecessary strings (#5568) + - URL encoding fixes in the download widget and socket API updates prevent regressions when upgrading Crystal (#5367, #5538) + +### Full list of pull requests merged since the last release (newest first) + +* refactor: Move top level constants to it's own modules (https://github.com/iv-org/invidious/pull/5596, by @Fijxu) +* pages/watch: URL encode 'action' in download widget (https://github.com/iv-org/invidious/pull/5367, by @SamantazFox) +* Document use of unix sockets for `db` (https://github.com/iv-org/invidious/pull/5347, by @Fijxu) +* Generate companion CSP only once to reuse it (https://github.com/iv-org/invidious/pull/5497, by @Fijxu) +* Fix youtube CSV playlist importer (https://github.com/iv-org/invidious/pull/4787, by @ThatMatrix) +* Playlist API: return empty author url if ucid is empty (https://github.com/iv-org/invidious/pull/5618, by @radmorecameron) +* Channels: parse pronouns and display them on channel page (https://github.com/iv-org/invidious/pull/5617, by @radmorecameron) +* playlist: parse playlist thumbnails for topic autogenerated playlists (https://github.com/iv-org/invidious/pull/5616, by @radmorecameron) +* fix: add missing embedded protobuf message in continuation token for channel videos (https://github.com/iv-org/invidious/pull/5614, by @Fijxu) +* Update shard.yml to include target that was removed in commit 9d54cf9 (https://github.com/iv-org/invidious/pull/5608, by @Harm133) +* chore: Do not convert thin_mode preference to string to compare it in before_all (https://github.com/iv-org/invidious/pull/5568, by @Fijxu) +* Fix thin_mode preference for channel community page (https://github.com/iv-org/invidious/pull/5567, by @Fijxu) +* Fix commit command in README instructions, as per #5606 (https://github.com/iv-org/invidious/pull/5607, by @kirisakow) +* Revert "Bump crystallang/crystal from 1.16.3-alpine to 1.19.0-alpine in /docker" (https://github.com/iv-org/invidious/pull/5604, by @unixfox) +* Bump crystallang/crystal from 1.16.3-alpine to 1.19.0-alpine in /docker (https://github.com/iv-org/invidious/pull/5603, by @dependabot[bot]) +* doc: Update HTTP proxy configuration comments (https://github.com/iv-org/invidious/pull/5586, by @unixfox) +* Strip unwanted headers from response headers in images and videoplayback (https://github.com/iv-org/invidious/pull/5595, by @Fijxu) +* Generate companion check id one time and add missing companion check id on captions (https://github.com/iv-org/invidious/pull/5575, by @Fijxu) +* Downgrade Crystal to 1.16.3 in OCI (https://github.com/iv-org/invidious/pull/5577, by @Fijxu) +* Allow downloading via companion (https://github.com/iv-org/invidious/pull/5561, by @JeroenBoersma) +* chore: crystal 1.8.2 + alpine 3.23 (https://github.com/iv-org/invidious/pull/5574, by @unixfox) +* Replace deprecated `blocking` property of `Socket` (https://github.com/iv-org/invidious/pull/5538, by @Fijxu) +* Replace `Kemal::StaticFileHandler` with direct subclass of stdlib `HTTP::StaticFileHandler` on Crystal >= 1.17.0 (https://github.com/iv-org/invidious/pull/5338, by @syeopite) +* dockerfile: compile openssl instead of using the one bundled on the crystal alpine image. (https://github.com/iv-org/invidious/pull/5441, by @Fijxu) +* Bump actions/cache from 4 to 5 (https://github.com/iv-org/invidious/pull/5569, by @dependabot[bot]) +* Set Kemal `max_request_line_size` to 16384 for large channel continuation query parameters. (https://github.com/iv-org/invidious/pull/5566, by @Fijxu) +* Add link to GitHub release/tag/commit in footer (https://github.com/iv-org/invidious/pull/4702, by @shaedrich) +* Display "Erroneous CAPTCHA" for invalid captchas (https://github.com/iv-org/invidious/pull/5508, by @Fijxu) +* Fix channel name overflow (https://github.com/iv-org/invidious/pull/5553, by @Fijxu) +* Fix trending page by leaving livestream and gaming trending pages (https://github.com/iv-org/invidious/pull/5555, by @Fijxu) +* fix: restore dmca_content functionality (https://github.com/iv-org/invidious/pull/5228, by @Fijxu) +* Remove signature helper completely from Invidious (https://github.com/iv-org/invidious/pull/5550, by @Fijxu) +* Fix community posts when there is a unavailable video in a post (https://github.com/iv-org/invidious/pull/5549, by @Fijxu) +* chore: Update shard.yml to use SPDX license identifier (https://github.com/iv-org/invidious/pull/5552, by @Fijxu) +* Store `preferences` in a variable when reused and rename `prefs` to `preferences` (https://github.com/iv-org/invidious/pull/5450, by @Fijxu) +* Bump actions/checkout from 5 to 6 (https://github.com/iv-org/invidious/pull/5544, by @dependabot[bot]) +* Bump crystal-lang/install-crystal from 1.8.3 to 1.9.1 (https://github.com/iv-org/invidious/pull/5530, by @dependabot[bot]) +* Fix 0 view count on related videos section (https://github.com/iv-org/invidious/pull/5446, by @shiny-comic) +* Prevent timestamp from being set for Livestreams on "Watch on Youtube" links (https://github.com/iv-org/invidious/pull/5481, by @Fijxu) +* Add Livestreams to trending page (https://github.com/iv-org/invidious/pull/5480, by @Fijxu) +* Fix button overflow (https://github.com/iv-org/invidious/pull/5452, by @Fijxu) +* Bump crystal-lang/install-crystal from 1.8.2 to 1.8.3 (https://github.com/iv-org/invidious/pull/5499, by @dependabot[bot]) +* Fixed broken companion hyperlink (https://github.com/iv-org/invidious/pull/5491, by @ndsvw) + ## v2.20250913.0 ### Wrap-up diff --git a/README.md b/README.md index 97d2109b..5b789a50 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,7 @@ You can read more here: https://docs.invidious.io/applications/ 1. Fork it ( https://github.com/iv-org/invidious/fork ). 1. Create your feature branch (`git checkout -b my-new-feature`). 1. Stage your files (`git add .`). -1. Commit your changes (`git commit -am 'Add some feature'`). +1. Commit your changes (`git commit -m 'Add some feature'`). 1. Push to the branch (`git push origin my-new-feature`). 1. Create a new pull request ( https://github.com/iv-org/invidious/compare ). diff --git a/assets/css/default.css b/assets/css/default.css index 644d91c2..ff07bdb4 100644 --- a/assets/css/default.css +++ b/assets/css/default.css @@ -75,6 +75,16 @@ body { height: auto; } +.channel-profile > .channel-name-pronouns { + display: inline-block; +} + +.channel-profile > .channel-name-pronouns > .channel-pronouns { + font-style: italic; + font-size: .8em; + font-weight: lighter; +} + body a.channel-owner { background-color: #008bec; color: #fff; @@ -404,9 +414,15 @@ input[type="search"]::-webkit-search-cancel-button { .video-card-row { margin: 15px 0; } -p.channel-name { margin: 0; } +p.channel-name { margin: 0; overflow-wrap: anywhere;} p.video-data { margin: 0; font-weight: bold; font-size: 80%; } +.channel-profile > .channel-name, +.channel-profile > .channel-name-pronouns > .channel-name +{ + overflow-wrap: anywhere; +} + /* * Comments & community posts diff --git a/assets/js/_helpers.js b/assets/js/_helpers.js index 8e18169e..ae3b157c 100644 --- a/assets/js/_helpers.js +++ b/assets/js/_helpers.js @@ -211,9 +211,9 @@ window.helpers = window.helpers || { helpers.storage.remove(key); } }, - set: function (key, value) { + set: function (key, value) { let encoded_value = encodeURIComponent(JSON.stringify(value)) - localStorage.setItem(key, encoded_value); + localStorage.setItem(key, encoded_value); }, remove: function (key) { localStorage.removeItem(key); } }; diff --git a/assets/js/player.js b/assets/js/player.js index ecdc0448..e9e9038d 100644 --- a/assets/js/player.js +++ b/assets/js/player.js @@ -143,7 +143,7 @@ player.on('timeupdate', function () { let base_url_yt_watch = elem_yt_watch.getAttribute('data-base-url'); elem_yt_watch.href = addCurrentTimeToURL(base_url_yt_watch); } - + let elem_yt_embed = document.getElementById('link-yt-embed'); if (elem_yt_embed) { let base_url_yt_embed = elem_yt_embed.getAttribute('data-base-url'); @@ -160,12 +160,18 @@ player.on('timeupdate', function () { let base_url_iv_embed = elem_iv_embed.getAttribute('data-base-url'); elem_iv_embed.href = addCurrentTimeToURL(base_url_iv_embed, domain); } - + let elem_iv_other = document.getElementById('link-iv-other'); if (elem_iv_other) { let base_url_iv_other = elem_iv_other.getAttribute('data-base-url'); elem_iv_other.href = addCurrentTimeToURL(base_url_iv_other, domain); } + + let elem_iv_listen = document.getElementById('link-iv-listen'); + if (elem_iv_listen) { + let base_url_iv_listen = elem_iv_listen.getAttribute('data-base-url'); + elem_iv_listen.href = addCurrentTimeToURL(base_url_iv_listen, domain); + } }); @@ -628,7 +634,7 @@ function toggle_caption_window() { player.textTrackSettings.setValues({ windowOpacity: options.windowOpacity[newIndex] }); update_captions(); } - + function toggle_caption_opacity() { const numOptions = options.textOpacity.length; const textOpacity = player.textTrackSettings.getValues().textOpacity || '1'; @@ -733,7 +739,7 @@ addEventListener('keydown', function (e) { case '>': action = increase_playback_rate.bind(this, 1); break; case '<': action = increase_playback_rate.bind(this, -1); break; - + case '=': action = increase_caption_size.bind(this, 1); break; case '-': action = increase_caption_size.bind(this, -1); break; diff --git a/config/config.example.yml b/config/config.example.yml index 1715188b..63135b52 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -8,6 +8,13 @@ ## Database configuration with separate parameters. ## This setting is MANDATORY, unless 'database_url' is used. ## +## Note: The 'db' setting allows the use of UNIX +## sockets. To do so, set 'host' to "" +## E.g: +## password: kemal +## host: "" +## port: 5432 +## db: user: kemal password: kemal @@ -40,27 +47,13 @@ db: ## #check_tables: false - -## -## Path to an external signature resolver, used to emulate -## the Youtube client's Javascript. If no such server is -## available, some videos will not be playable. -## -## When this setting is commented out, no external -## resolver will be used. -## -## Accepted values: a path to a UNIX socket or ":" -## Default: -## -#signature_server: - ## ## Invidious companion is an external program ## for loading the video streams from YouTube servers. ## ## When this setting is commented out, Invidious companion is not used. ## Otherwise, Invidious will proxy the requests to Invidious companion. -## +## ## Note: multiple URL can be configured. In this case, Invidious will ## randomly pick one every time video data needs to be retrieved. This ## URL is then kept in the video metadata cache to allow video playback @@ -70,7 +63,7 @@ db: ## The parameter private_url is required for the internal communication ## between Invidious companion and Invidious. ## -## The optional parameter public_url is the public URL from which +## The optional parameter public_url is the public URL from which ## Invidious companion is listening to the requests from the user(s). ## When this setting is commented out, Invidious proxy all requests to ## Invidious companion. Useful for simple setups. @@ -237,9 +230,13 @@ https_only: false ## ## Configuration for using a HTTP proxy -## ## If unset, then no HTTP proxy will be used. -## +## Proxy type supported: HTTP, HTTPS +## +## This is not used for loading the video streams from YouTube servers (circumvent YouTube restrictions) +## Please instead configure the proxy in Invidious companion: +## https://github.com/iv-org/invidious-companion/blob/master/config/config.example.toml +## #http_proxy: # user: # password: @@ -259,19 +256,6 @@ https_only: false ## # use_innertube_for_captions: false -## -## Send Google session informations. This is useful when Invidious is blocked -## by the message "This helps protect our community." -## See https://github.com/iv-org/invidious/issues/4734. -## -## Warning: These strings gives much more identifiable information to Google! -## -## Accepted values: String -## Default: -## -# po_token: "" -# visitor_data: "" - # ----------------------------- # Logging # ----------------------------- @@ -902,7 +886,7 @@ default_user_preferences: ## Default: true ## #vr_mode: true - + ## ## Save the playback position ## Allow to continue watching at the previous position when diff --git a/docker-compose.yml b/docker-compose.yml index afda8726..dedd2e20 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,7 +32,7 @@ services: # statistics_enabled: false hmac_key: "CHANGE_ME!!" healthcheck: - test: wget -nv --tries=1 --spider http://127.0.0.1:3000/api/v1/trending || exit 1 + test: wget -nv --tries=1 --spider http://127.0.0.1:3000/api/v1/stats || exit 1 interval: 30s timeout: 5s retries: 2 diff --git a/docker/Dockerfile b/docker/Dockerfile index 4cfc3c72..e2d30364 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,29 @@ -FROM crystallang/crystal:1.16.3-alpine AS builder +# https://github.com/openssl/openssl/releases/tag/openssl-3.5.2 +ARG OPENSSL_VERSION='3.5.2' +ARG OPENSSL_SHA256='c53a47e5e441c930c3928cf7bf6fb00e5d129b630e0aa873b08258656e7345ec' + +FROM crystallang/crystal:1.16.3-alpine AS dependabot-crystal + +# We compile openssl ourselves due to a memory leak in how crystal interacts +# with openssl +# Reference: https://github.com/iv-org/invidious/issues/1438#issuecomment-3087636228 +FROM dependabot-crystal AS openssl-builder +RUN apk add --no-cache curl perl linux-headers + +WORKDIR / + +ARG OPENSSL_VERSION +ARG OPENSSL_SHA256 +RUN curl -Ls "https://github.com/openssl/openssl/releases/download/openssl-${OPENSSL_VERSION}/openssl-${OPENSSL_VERSION}.tar.gz" --output openssl-${OPENSSL_VERSION}.tar.gz +RUN echo "${OPENSSL_SHA256} openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c +RUN tar -xzvf openssl-${OPENSSL_VERSION}.tar.gz + +RUN cd openssl-${OPENSSL_VERSION} && ./Configure --openssldir=/etc/ssl && make -j$(nproc) + +FROM dependabot-crystal AS builder RUN apk add --no-cache sqlite-static yaml-static +RUN apk del openssl-dev openssl-libs-static ARG release @@ -21,18 +44,24 @@ COPY ./videojs-dependencies.yml ./videojs-dependencies.yml RUN crystal spec --warnings all \ --link-flags "-lxml2 -llzma" + +ARG OPENSSL_VERSION +COPY --from=openssl-builder /openssl-${OPENSSL_VERSION} /openssl-${OPENSSL_VERSION} + RUN --mount=type=cache,target=/root/.cache/crystal if [[ "${release}" == 1 ]] ; then \ + PKG_CONFIG_PATH=/openssl-${OPENSSL_VERSION} \ crystal build ./src/invidious.cr \ --release \ --static --warnings all \ --link-flags "-lxml2 -llzma"; \ else \ + PKG_CONFIG_PATH=/openssl-${OPENSSL_VERSION} \ crystal build ./src/invidious.cr \ --static --warnings all \ --link-flags "-lxml2 -llzma"; \ fi -FROM alpine:3.21 +FROM alpine:3.23 RUN apk add --no-cache rsvg-convert ttf-opensans tini tzdata WORKDIR /invidious RUN addgroup -g 1000 -S invidious && \ diff --git a/docker/Dockerfile.arm64 b/docker/Dockerfile.arm64 index 758e7950..ce691c91 100644 --- a/docker/Dockerfile.arm64 +++ b/docker/Dockerfile.arm64 @@ -1,6 +1,31 @@ -FROM alpine:3.21 AS builder -RUN apk add --no-cache 'crystal=1.14.0-r0' shards sqlite-static yaml-static yaml-dev libxml2-static \ - zlib-static openssl-libs-static openssl-dev musl-dev xz-static +# https://github.com/openssl/openssl/releases/tag/openssl-3.5.2 +ARG OPENSSL_VERSION='3.5.2' +ARG OPENSSL_SHA256='c53a47e5e441c930c3928cf7bf6fb00e5d129b630e0aa873b08258656e7345ec' + +FROM alpine:3.22 AS dependabot-alpine + +# We compile openssl ourselves due to a memory leak in how crystal interacts +# with openssl +# Reference: https://github.com/iv-org/invidious/issues/1438#issuecomment-3087636228 +FROM dependabot-alpine AS openssl-builder +RUN apk add --no-cache curl perl linux-headers build-base + +WORKDIR / + +ARG OPENSSL_VERSION +ARG OPENSSL_SHA256 +RUN curl -Ls "https://github.com/openssl/openssl/releases/download/openssl-${OPENSSL_VERSION}/openssl-${OPENSSL_VERSION}.tar.gz" --output openssl-${OPENSSL_VERSION}.tar.gz +RUN echo "${OPENSSL_SHA256} openssl-${OPENSSL_VERSION}.tar.gz" | sha256sum -c +RUN tar -xzvf openssl-${OPENSSL_VERSION}.tar.gz + +RUN cd openssl-${OPENSSL_VERSION} && ./Configure --openssldir=/etc/ssl && make -j$(nproc) + +FROM dependabot-alpine AS builder +RUN apk add --no-cache 'crystal=1.16.3-r0' shards \ + sqlite-static yaml-static yaml-dev \ + pcre2-static gc-static \ + libxml2-static zlib-static \ + openssl-libs-static openssl-dev musl-dev xz-static ARG release @@ -22,18 +47,23 @@ COPY ./videojs-dependencies.yml ./videojs-dependencies.yml RUN crystal spec --warnings all \ --link-flags "-lxml2 -llzma" +ARG OPENSSL_VERSION +COPY --from=openssl-builder /openssl-${OPENSSL_VERSION} /openssl-${OPENSSL_VERSION} + RUN --mount=type=cache,target=/root/.cache/crystal if [[ "${release}" == 1 ]] ; then \ + PKG_CONFIG_PATH=/openssl-${OPENSSL_VERSION} \ crystal build ./src/invidious.cr \ --release \ --static --warnings all \ --link-flags "-lxml2 -llzma"; \ else \ + PKG_CONFIG_PATH=/openssl-${OPENSSL_VERSION} \ crystal build ./src/invidious.cr \ --static --warnings all \ --link-flags "-lxml2 -llzma"; \ fi -FROM alpine:3.21 +FROM alpine:3.22 RUN apk add --no-cache rsvg-convert ttf-opensans tini tzdata WORKDIR /invidious RUN addgroup -g 1000 -S invidious && \ diff --git a/locales/en-US.json b/locales/en-US.json index 3bfd9a84..1530e1ba 100644 --- a/locales/en-US.json +++ b/locales/en-US.json @@ -509,5 +509,5 @@ "timeline_parse_error_placeholder_heading": "Unable to parse item", "timeline_parse_error_placeholder_message": "Invidious encountered an error while trying to parse this item. For more information see below:", "timeline_parse_error_show_technical_details": "Show technical details", - "search_page_disabled": "Search has been disabled by the administrator." -} \ No newline at end of file + "dmca_content": "This video cannot be downloaded on this instance due to a DMCA/copyright infringement letter sent to the instance administrator." +} diff --git a/scripts/git/pre-commit b/scripts/git/pre-commit index 4460b670..0b19802d 100644 --- a/scripts/git/pre-commit +++ b/scripts/git/pre-commit @@ -3,7 +3,7 @@ # Crystal linter # This is a modified version of the pre-commit hook from the crystal repo. https://github.com/crystal-lang/crystal/blob/master/scripts/git/pre-commit -# Please refer to that if you'd like an version that doesn't automatically format staged files. +# Please refer to that if you'd like an version that doesn't automatically format staged files. changed_cr_files=$(git diff --cached --name-only --diff-filter=ACM | grep '\.cr$') if [ ! -z "$changed_cr_files" ]; then if [ -x bin/crystal ]; then diff --git a/shard.yml b/shard.yml index 4dc8aa02..95397dfd 100644 --- a/shard.yml +++ b/shard.yml @@ -1,10 +1,14 @@ name: invidious -version: 2.20250913.0-dev +version: 2.20260207.0-dev authors: - Invidious team - Contributors! +targets: + invidious: + main: src/invidious.cr + description: | Invidious is an alternative front-end to YouTube @@ -38,7 +42,7 @@ development_dependencies: crystal: ">= 1.10.0, < 2.0.0" -license: AGPLv3 +license: AGPL-3.0-only repository: https://github.com/iv-org/invidious homepage: https://invidious.io diff --git a/spec/http_server/handlers/static_assets_handler/test.txt b/spec/http_server/handlers/static_assets_handler/test.txt new file mode 100644 index 00000000..70c379b6 --- /dev/null +++ b/spec/http_server/handlers/static_assets_handler/test.txt @@ -0,0 +1 @@ +Hello world \ No newline at end of file diff --git a/spec/http_server/handlers/static_assets_handler_spec.cr b/spec/http_server/handlers/static_assets_handler_spec.cr new file mode 100644 index 00000000..76dc7be7 --- /dev/null +++ b/spec/http_server/handlers/static_assets_handler_spec.cr @@ -0,0 +1,233 @@ +# Due to the way that specs are handled this file cannot be run together with +# everything else without causing a compile time error that'll be incredibly +# annoying to resolve. +# +# TODO: Create different spec categories that can then be ran through make. +# An implementation of this can be seen with the tests for the Crystal compiler itself. +# +# For now run this with `crystal spec spec/http_server/handlers/static_assets_handler_spec.cr -Drunning_by_self` + +{% skip_file if compare_versions(Crystal::VERSION, "1.17.0-dev") < 0 || !flag?(:running_by_self) %} + +require "http" +require "spectator" +require "../../../src/invidious/http_server/static_assets_handler.cr" + +private def get_static_assets_handler + return Invidious::HttpServer::StaticAssetsHandler.new "spec/http_server/handlers/static_assets_handler", directory_listing: false +end + +# Slightly modified version of `handle` function from +# +# https://github.com/crystal-lang/crystal/blob/3f369d2c721e9462d9f6126cb0bcd4c6992f0225/spec/std/http/server/handlers/static_file_handler_spec.cr#L5 + +private def handle(request, handler : HTTP::Handler? = nil, decompress : Bool = false) + io = IO::Memory.new + response = HTTP::Server::Response.new(io) + context = HTTP::Server::Context.new(request, response) + + if !handler + handler = get_static_assets_handler + get_static_assets_handler.call context + else + handler.call(context) + end + + response.close + io.rewind + + HTTP::Client::Response.from_io(io, decompress: decompress) +end + +# Makes and yields a temporary file with the given prefix +private def make_temporary_file(prefix, contents = nil, &) + tempfile = File.tempfile(prefix, "static_assets_handler_spec", dir: "spec/http_server/handlers/static_assets_handler") + file_link = "/#{File.basename(tempfile.path)}" + yield tempfile, file_link +ensure + tempfile.try &.delete +end + +# Changes the contents of the temporary file after yield +private def cycle_temporary_file_contents(temporary_file, initial, &) + temporary_file.rewind << initial + temporary_file.rewind.flush + yield + temporary_file.rewind << "something else" + temporary_file.rewind.flush +end + +# Get relative file path to a file within the static_assets_handler folder +macro get_file_path(basename) + "spec/http_server/handlers/static_assets_handler/#{ {{basename}} }" +end + +Spectator.describe StaticAssetsHandler do + it "Can serve a file" do + response = handle HTTP::Request.new("GET", "/test.txt") + expect(response.status_code).to eq(200) + expect(response.body).to eq(File.read(get_file_path("test.txt"))) + end + + it "Can serve cached file" do + make_temporary_file("cache_test") do |temporary_file, file_link| + cycle_temporary_file_contents(temporary_file, "foo") do + expect(temporary_file.rewind.gets_to_end).to eq("foo") + + # Should get cached by the first run + response = handle HTTP::Request.new("GET", file_link) + expect(response.status_code).to eq(200) + expect(response.body).to eq("foo") + end + + # Temporary file is updated after `cycle_temporary_file_contents` is called + # but if the file is successfully cached then we'll only get the original + # contents. + response = handle HTTP::Request.new("GET", file_link) + expect(response.status_code).to eq(200) + expect(response.body).to eq("foo") + end + end + + it "Adds cache headers" do + response = handle HTTP::Request.new("GET", "/test.txt") + expect(response.headers["cache_control"]).to eq("max-age=2629800") + end + + context "Can handle range requests" do + it "Can serve range request" do + headers = HTTP::Headers{"Range" => "bytes=0-2"} + response = handle HTTP::Request.new("GET", "/test.txt", headers) + + expect(response.status_code).to eq(206) + expect(response.headers["Content-Range"]?).to eq "bytes 0-2/11" + expect(response.body).to eq "Hel" + end + + it "Will cache entire file even if doing partial requests" do + make_temporary_file("range_cache") do |temporary_file, file_link| + cycle_temporary_file_contents(temporary_file, "Hello world") do + handle HTTP::Request.new("GET", file_link, HTTP::Headers{"Range" => "bytes=0-2"}) + end + + # Second request shouldn't have changed + headers = HTTP::Headers{"Range" => "bytes=3-8"} + response = handle HTTP::Request.new("GET", file_link, headers) + expect(response.status_code).to eq(206) + expect(response.body).to eq "lo wor" + end + end + end + + context "Is able to support compression" do + def decompressed(string : String) + decompressed = Compress::Gzip::Reader.open(IO::Memory.new(string)) do |gzip| + gzip.gets_to_end + end + + return expect(decompressed) + end + + it "For full file requests" do + handler = HTTP::CompressHandler.new + handler.next = get_static_assets_handler() + + make_temporary_file("check decompression handler") do |temporary_file, file_link| + cycle_temporary_file_contents(temporary_file, "Hello world") do + response = handle HTTP::Request.new("GET", file_link, headers: HTTP::Headers{"Accept-Encoding" => "gzip"}), handler: handler + expect(response.headers["Content-Encoding"]).to eq("gzip") + decompressed(response.body).to eq("Hello world") + end + + # Are cached requests working? + response = handle HTTP::Request.new("GET", file_link, headers: HTTP::Headers{"Accept-Encoding" => "gzip"}), handler: handler + expect(response.headers["Content-Encoding"]).to eq("gzip") + decompressed(response.body).to eq("Hello world") + + # Able to retrieve non gzipped file? + response = handle HTTP::Request.new("GET", file_link), handler: handler + expect(response.body).to eq("Hello world") + expect(response.headers).to_not have_key("Content-Encoding") + end + end + + # Inspired by the equivalent tests from upstream + it "For partial file requests" do + handler = HTTP::CompressHandler.new + handler.next = get_static_assets_handler() + + make_temporary_file("check_decompression_handler_on_partial_requests") do |temporary_file, file_link| + cycle_temporary_file_contents(temporary_file, "Hello world this is a very long string") do + range_response_results = { + "10-20/38" => "d this is a", + "0-0/38" => "H", + "5-9/38" => " worl", + } + + range_request_header_value = {"10-20", "5-9", "0-0"}.join(',') + range_response_header_value = range_response_results.keys + + response = handle HTTP::Request.new("GET", file_link, headers: HTTP::Headers{"Range" => "bytes=#{range_request_header_value}", "Accept-Encoding" => "gzip"}), handler: handler + expect(response.headers["Content-Encoding"]).to eq("gzip") + + # Decompress response + response = HTTP::Client::Response.new( + status: response.status, + headers: response.headers, + body_io: Compress::Gzip::Reader.new(IO::Memory.new(response.body)), + ) + + count = 0 + MIME::Multipart.parse(response) do |headers, part| + part_range = headers["Content-Range"][6..] + expect(part_range).to be_within(range_response_header_value) + expect(part.gets_to_end).to eq(range_response_results[part_range]) + count += 1 + end + + expect(count).to eq(3) + end + + # Is the file cached? + temporary_file << "Something else" + temporary_file.flush.rewind + + response = handle HTTP::Request.new("GET", file_link, headers: HTTP::Headers{"Accept-Encoding" => "gzip"}), handler: handler + decompressed(response.body).to eq("Hello world this is a very long string") + end + end + end + + it "Will not cache additional files if the cache limit is reached" do + 5.times do |times| + data = "a" * 1_000_000 + + make_temporary_file("test cache size limit #{times}") do |temporary_file, file_link| + cycle_temporary_file_contents(temporary_file, data) do + response = handle HTTP::Request.new("GET", file_link) + expect(response.status_code).to eq(200) + expect(response.body).to eq(data) + end + + response = handle HTTP::Request.new("GET", file_link) + expect(response.status_code).to eq(200) + expect(response.body).to eq(data) + end + end + + # Cache should be 5 mb so no more files will be cached. + make_temporary_file("test cache size limit uncached") do |temporary_file, file_link| + cycle_temporary_file_contents(temporary_file, "a") do + response = handle HTTP::Request.new("GET", file_link) + expect(response.status_code).to eq(200) + expect(response.body).to eq("a") + end + + response = handle HTTP::Request.new("GET", file_link) + expect(response.status_code).to eq(200) + expect(response.body).to_not eq("a") + end + end + + after_each { Invidious::HttpServer::StaticAssetsHandler.clear_cache } +end diff --git a/spec/invidious/search/yt_filters_spec.cr b/spec/invidious/search/yt_filters_spec.cr index 8abed5ce..a724fd25 100644 --- a/spec/invidious/search/yt_filters_spec.cr +++ b/spec/invidious/search/yt_filters_spec.cr @@ -48,9 +48,7 @@ FEATURE_FILTERS = { SORT_FILTERS = { Invidious::Search::Filters::Sort::Relevance => "8AEB", - Invidious::Search::Filters::Sort::Date => "CALwAQE%3D", Invidious::Search::Filters::Sort::Views => "CAPwAQE%3D", - Invidious::Search::Filters::Sort::Rating => "CAHwAQE%3D", } Spectator.describe Invidious::Search::Filters do diff --git a/src/ext/kemal_static_file_handler.cr b/src/ext/kemal_static_file_handler.cr index a5f42261..16cb84fb 100644 --- a/src/ext/kemal_static_file_handler.cr +++ b/src/ext/kemal_static_file_handler.cr @@ -1,3 +1,24 @@ +{% if compare_versions(Crystal::VERSION, "1.17.0-dev") >= 0 %} + # Strip StaticFileHandler from the binary + # + # This allows us to compile on 1.17.0 as the compiler won't try to + # semantically check the outdated upstream code. + class Kemal::Config + private def setup_static_file_handler + end + end + + # Nullify `Kemal::StaticFileHandler` + # + # Needed until the next release of Kemal after 1.7 + class Kemal::StaticFileHandler < HTTP::StaticFileHandler + def call(context : HTTP::Server::Context) + end + end + + {% skip_file %} +{% end %} + # Since systems have a limit on number of open files (`ulimit -a`), # we serve them from memory to avoid 'Too many open files' without needing # to modify ulimit. diff --git a/src/invidious.cr b/src/invidious.cr index b276bf1a..ac15ec1f 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -67,23 +67,13 @@ rescue ex puts "Check your 'config.yml' database settings or PostgreSQL settings." exit(1) end -ARCHIVE_URL = URI.parse("https://archive.org") -PUBSUB_URL = URI.parse("https://pubsubhubbub.appspot.com") -REDDIT_URL = URI.parse("https://www.reddit.com") -YT_URL = URI.parse("https://www.youtube.com") -HOST_URL = make_host_url(Kemal.config) - -CHARS_SAFE = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" -TEST_IDS = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"} +HOST_URL = make_host_url(Kemal.config) MAX_ITEMS_PER_PAGE = 1500 -REQUEST_HEADERS_WHITELIST = {"accept", "accept-encoding", "cache-control", "content-length", "if-none-match", "range"} -RESPONSE_HEADERS_BLACKLIST = {"access-control-allow-origin", "alt-svc", "server"} -HTTP_CHUNK_SIZE = 10485760 # ~10MB - CURRENT_BRANCH = {{ "#{`git branch | sed -n '/* /s///p'`.strip}" }} CURRENT_COMMIT = {{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit`.strip}" }} CURRENT_VERSION = {{ "#{`git log -1 --format=%ci | awk '{print $1}' | sed s/-/./g`.strip}" }} +CURRENT_TAG = {{ "#{`git tag --points-at HEAD`.strip}" }} # This is used to determine the `?v=` on the end of file URLs (for cache busting). We # only need to expire modified assets, so we can use this to find the last commit that changes @@ -96,7 +86,7 @@ SOFTWARE = { "branch" => "#{CURRENT_BRANCH}", } -YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size) +YT_POOL = YoutubeConnectionPool.new(URI.parse("https://www.youtube.com"), capacity: CONFIG.pool_size) # Image request pool @@ -170,15 +160,6 @@ Invidious::Database.check_integrity(CONFIG) {% puts "\nDone checking player dependencies, now compiling Invidious...\n" %} {% end %} -# Misc - -DECRYPT_FUNCTION = - if sig_helper_address = CONFIG.signature_server.presence - IV::DecryptFunction.new(sig_helper_address) - else - nil - end - # Start jobs if CONFIG.channel_threads > 0 @@ -231,19 +212,25 @@ error 500 do |env, exception| error_template(500, exception) end -static_headers do |env| - env.response.headers.add("Cache-Control", "max-age=2629800") -end - # Init Kemal -public_folder "assets" - Kemal.config.powered_by_header = false add_handler FilteredCompressHandler.new add_handler APIHandler.new add_handler AuthHandler.new add_handler DenyFrame.new + +{% if compare_versions(Crystal::VERSION, "1.17.0-dev") >= 0 %} + Kemal.config.serve_static = false + add_handler Invidious::HttpServer::StaticAssetsHandler.new("assets", directory_listing: false) +{% else %} + public_folder "assets" + + static_headers do |env| + env.response.headers.add("Cache-Control", "max-age=2629800") + end +{% end %} + add_context_storage_type(Array(String)) add_context_storage_type(Preferences) add_context_storage_type(Invidious::User) @@ -258,6 +245,8 @@ Kemal.config.app_name = "Invidious" {% end %} Kemal.run do |config| + config.server.not_nil!.max_request_line_size = 16384 + if socket_binding = CONFIG.socket_binding File.delete?(socket_binding.path) # Create a socket and set its desired permissions diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr index 13909527..537aa034 100644 --- a/src/invidious/channels/about.cr +++ b/src/invidious/channels/about.cr @@ -12,6 +12,7 @@ record AboutChannel, sub_count : Int32, joined : Time, is_family_friendly : Bool, + pronouns : String?, allowed_regions : Array(String), tabs : Array(String), tags : Array(String), @@ -160,14 +161,21 @@ def get_about_info(ucid, locale) : AboutChannel end sub_count = 0 + pronouns = nil if (metadata_rows = initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "metadata", "contentMetadataViewModel", "metadataRows").try &.as_a) metadata_rows.each do |row| - metadata_part = row.dig?("metadataParts").try &.as_a.find { |i| i.dig?("text", "content").try &.as_s.includes?("subscribers") } - if !metadata_part.nil? - sub_count = short_text_to_number(metadata_part.dig("text", "content").as_s.split(" ")[0]).to_i32 + subscribe_metadata_part = row.dig?("metadataParts").try &.as_a.find { |i| i.dig?("text", "content").try &.as_s.includes?("subscribers") } + if !subscribe_metadata_part.nil? + sub_count = short_text_to_number(subscribe_metadata_part.dig("text", "content").as_s.split(" ")[0]).to_i32 end - break if sub_count != 0 + + pronoun_metadata_part = row.dig?("metadataParts").try &.as_a.find { |i| i.dig?("tooltip").try &.as_s.includes?("Pronouns") } + if !pronoun_metadata_part.nil? + pronouns = pronoun_metadata_part.dig("text", "content").as_s + end + + break if sub_count != 0 && !pronouns.nil? end end @@ -184,6 +192,7 @@ def get_about_info(ucid, locale) : AboutChannel sub_count: sub_count, joined: joined, is_family_friendly: is_family_friendly, + pronouns: pronouns, allowed_regions: allowed_regions, tabs: tab_names, tags: tags, diff --git a/src/invidious/channels/community.cr b/src/invidious/channels/community.cr index 43843b11..4256230c 100644 --- a/src/invidious/channels/community.cr +++ b/src/invidious/channels/community.cr @@ -143,7 +143,7 @@ def extract_channel_community(items, *, ucid, locale, format, thin_mode, is_sing case attachment.as_h when .has_key?("videoRenderer") parse_item(attachment) - .as(SearchVideo) + .as(SearchVideo | ProblematicTimelineItem) .to_json(locale, json) when .has_key?("backstageImageRenderer") json.object do diff --git a/src/invidious/channels/videos.cr b/src/invidious/channels/videos.cr index 96400f47..e2cc8305 100644 --- a/src/invidious/channels/videos.cr +++ b/src/invidious/channels/videos.cr @@ -114,7 +114,11 @@ module Invidious::Channel::Tabs "2:embedded" => { "1:string" => "00000000-0000-0000-0000-000000000000", }, - "4:varint" => sort_options_videos_short(sort_by), + "4:varint" => sort_options_videos_short(sort_by), + "8:embedded" => { + "1:string" => "00000000-0000-0000-0000-000000000000", + "3:varint" => sort_options_videos_short(sort_by), + }, }, } @@ -130,7 +134,11 @@ module Invidious::Channel::Tabs "2:embedded" => { "1:string" => "00000000-0000-0000-0000-000000000000", }, - "4:varint" => sort_options_videos_short(sort_by), + "4:varint" => sort_options_videos_short(sort_by), + "7:embedded" => { + "1:string" => "00000000-0000-0000-0000-000000000000", + "3:varint" => sort_options_videos_short(sort_by), + }, }, } @@ -154,7 +162,11 @@ module Invidious::Channel::Tabs "2:embedded" => { "1:string" => "00000000-0000-0000-0000-000000000000", }, - "5:varint" => sort_by_numerical, + "5:varint" => sort_by_numerical, + "8:embedded" => { + "1:string" => "00000000-0000-0000-0000-000000000000", + "3:varint" => sort_by_numerical, + }, }, } diff --git a/src/invidious/comments/reddit.cr b/src/invidious/comments/reddit.cr index ba9c19f1..e128350c 100644 --- a/src/invidious/comments/reddit.cr +++ b/src/invidious/comments/reddit.cr @@ -1,5 +1,6 @@ module Invidious::Comments extend self + private REDDIT_URL = URI.parse("https://www.reddit.com") def fetch_reddit(id, sort_by = "confidence") client = make_client(REDDIT_URL) diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 5f06d8dc..933c6654 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -202,9 +202,6 @@ class Config @[YAML::Field(converter: Preferences::FamilyConverter)] property force_resolve : Socket::Family = Socket::Family::UNSPEC - # External signature solver server socket (either a path to a UNIX domain socket or ":") - property signature_server : String? = nil - # Port to listen for connections (overridden by command line argument) property port : Int32 = 3000 # Host to bind (overridden by command line argument) @@ -219,11 +216,6 @@ class Config # Use Innertube's transcripts API instead of timedtext for closed captions property use_innertube_for_captions : Bool = false - # visitor data ID for Google session - property visitor_data : String? = nil - # poToken for passing bot attestation - property po_token : String? = nil - # Invidious companion property invidious_companion : Array(CompanionConfig) = [] of CompanionConfig @@ -318,11 +310,7 @@ class Config {% end %} if config.invidious_companion.present? - # invidious_companion and signature_server can't work together - if config.signature_server - puts "Config: You can not run inv_sig_helper and invidious_companion at the same time." - exit(1) - elsif config.invidious_companion_key.empty? + if config.invidious_companion_key.empty? puts "Config: Please configure a key if you are using invidious companion." exit(1) elsif config.invidious_companion_key == "CHANGE_ME!!" @@ -340,8 +328,6 @@ class Config companion.builtin_proxy = true end end - elsif config.signature_server - puts("WARNING: inv-sig-helper is deprecated. Please switch to Invidious companion: https://docs.invidious.io/installation/") else puts("WARNING: Invidious companion is required to view and playback videos. For more information see https://docs.invidious.io/installation/") end diff --git a/src/invidious/frontend/misc.cr b/src/invidious/frontend/misc.cr index 7a6cf79d..9c30724a 100644 --- a/src/invidious/frontend/misc.cr +++ b/src/invidious/frontend/misc.cr @@ -2,9 +2,9 @@ module Invidious::Frontend::Misc extend self def redirect_url(env : HTTP::Server::Context) - prefs = env.get("preferences").as(Preferences) + preferences = env.get("preferences").as(Preferences) - if prefs.automatic_instance_redirect + if preferences.automatic_instance_redirect current_page = env.get?("current_page").as(String) return "/redirect?referer=#{current_page}" else diff --git a/src/invidious/frontend/watch_page.cr b/src/invidious/frontend/watch_page.cr index c0926164..642ab4cc 100644 --- a/src/invidious/frontend/watch_page.cr +++ b/src/invidious/frontend/watch_page.cr @@ -23,6 +23,10 @@ module Invidious::Frontend::WatchPage return "

#{translate(locale, "Download is disabled")}

" end + if CONFIG.dmca_content.includes?(video.id) + return "

#{translate(locale, "dmca_content")}

" + end + url = "/download" if (CONFIG.invidious_companion.present?) invidious_companion = CONFIG.invidious_companion.sample @@ -32,7 +36,7 @@ module Invidious::Frontend::WatchPage return String.build(4000) do |str| str << "" diff --git a/src/invidious/helpers/crystal_class_overrides.cr b/src/invidious/helpers/crystal_class_overrides.cr index fec3f62c..6fa89395 100644 --- a/src/invidious/helpers/crystal_class_overrides.cr +++ b/src/invidious/helpers/crystal_class_overrides.cr @@ -3,15 +3,28 @@ # IPv6 addresses. # class TCPSocket - def initialize(host, port, dns_timeout = nil, connect_timeout = nil, blocking = false, family = Socket::Family::UNSPEC) - Addrinfo.tcp(host, port, timeout: dns_timeout, family: family) do |addrinfo| - super(addrinfo.family, addrinfo.type, addrinfo.protocol, blocking) - connect(addrinfo, timeout: connect_timeout) do |error| - close - error + {% if compare_versions(Crystal::VERSION, "1.18.0-dev") >= 0 %} + def initialize(host : String, port, dns_timeout = nil, connect_timeout = nil, blocking = false, family = Socket::Family::UNSPEC) + Addrinfo.tcp(host, port, timeout: dns_timeout, family: family) do |addrinfo| + super(family: addrinfo.family, type: addrinfo.type, protocol: addrinfo.protocol) + Socket.set_blocking(self.fd, blocking) + connect(addrinfo, timeout: connect_timeout) do |error| + close + error + end end end - end + {% else %} + def initialize(host : String, port, dns_timeout = nil, connect_timeout = nil, blocking = false, family = Socket::Family::UNSPEC) + Addrinfo.tcp(host, port, timeout: dns_timeout, family: family) do |addrinfo| + super(addrinfo.family, addrinfo.type, addrinfo.protocol, blocking) + connect(addrinfo, timeout: connect_timeout) do |error| + close + error + end + end + end + {% end %} end # :ditto: diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 6add0237..ab694b1f 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -1,5 +1,7 @@ require "./macros" +TEST_IDS = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"} + struct Nonce include DB::Serializable diff --git a/src/invidious/helpers/sig_helper.cr b/src/invidious/helpers/sig_helper.cr deleted file mode 100644 index 6d198a42..00000000 --- a/src/invidious/helpers/sig_helper.cr +++ /dev/null @@ -1,349 +0,0 @@ -require "uri" -require "socket" -require "socket/tcp_socket" -require "socket/unix_socket" - -{% if flag?(:advanced_debug) %} - require "io/hexdump" -{% end %} - -private alias NetworkEndian = IO::ByteFormat::NetworkEndian - -module Invidious::SigHelper - enum UpdateStatus - Updated - UpdateNotRequired - Error - end - - # ------------------- - # Payload types - # ------------------- - - abstract struct Payload - end - - struct StringPayload < Payload - getter string : String - - def initialize(str : String) - raise Exception.new("SigHelper: String can't be empty") if str.empty? - @string = str - end - - def self.from_bytes(slice : Bytes) - size = IO::ByteFormat::NetworkEndian.decode(UInt16, slice) - if size == 0 # Error code - raise Exception.new("SigHelper: Server encountered an error") - end - - if (slice.bytesize - 2) != size - raise Exception.new("SigHelper: String size mismatch") - end - - if str = String.new(slice[2..]) - return self.new(str) - else - raise Exception.new("SigHelper: Can't read string from socket") - end - end - - def to_io(io) - # `.to_u16` raises if there is an overflow during the conversion - io.write_bytes(@string.bytesize.to_u16, NetworkEndian) - io.write(@string.to_slice) - end - end - - private enum Opcode - FORCE_UPDATE = 0 - DECRYPT_N_SIGNATURE = 1 - DECRYPT_SIGNATURE = 2 - GET_SIGNATURE_TIMESTAMP = 3 - GET_PLAYER_STATUS = 4 - PLAYER_UPDATE_TIMESTAMP = 5 - end - - private record Request, - opcode : Opcode, - payload : Payload? - - # ---------------------- - # High-level functions - # ---------------------- - - class Client - @mux : Multiplexor - - def initialize(uri_or_path) - @mux = Multiplexor.new(uri_or_path) - end - - # Forces the server to re-fetch the YouTube player, and extract the necessary - # components from it (nsig function code, sig function code, signature timestamp). - def force_update : UpdateStatus - request = Request.new(Opcode::FORCE_UPDATE, nil) - - value = send_request(request) do |bytes| - IO::ByteFormat::NetworkEndian.decode(UInt16, bytes) - end - - case value - when 0x0000 then return UpdateStatus::Error - when 0xFFFF then return UpdateStatus::UpdateNotRequired - when 0xF44F then return UpdateStatus::Updated - else - code = value.nil? ? "nil" : value.to_s(base: 16) - raise Exception.new("SigHelper: Invalid status code received #{code}") - end - end - - # Decrypt a provided n signature using the server's current nsig function - # code, and return the result (or an error). - def decrypt_n_param(n : String) : String? - request = Request.new(Opcode::DECRYPT_N_SIGNATURE, StringPayload.new(n)) - - n_dec = self.send_request(request) do |bytes| - StringPayload.from_bytes(bytes).string - end - - return n_dec - end - - # Decrypt a provided s signature using the server's current sig function - # code, and return the result (or an error). - def decrypt_sig(sig : String) : String? - request = Request.new(Opcode::DECRYPT_SIGNATURE, StringPayload.new(sig)) - - sig_dec = self.send_request(request) do |bytes| - StringPayload.from_bytes(bytes).string - end - - return sig_dec - end - - # Return the signature timestamp from the server's current player - def get_signature_timestamp : UInt64? - request = Request.new(Opcode::GET_SIGNATURE_TIMESTAMP, nil) - - return self.send_request(request) do |bytes| - IO::ByteFormat::NetworkEndian.decode(UInt64, bytes) - end - end - - # Return the current player's version - def get_player : UInt32? - request = Request.new(Opcode::GET_PLAYER_STATUS, nil) - - return self.send_request(request) do |bytes| - has_player = (bytes[0] == 0xFF) - player_version = IO::ByteFormat::NetworkEndian.decode(UInt32, bytes[1..4]) - has_player ? player_version : nil - end - end - - # Return when the player was last updated - def get_player_timestamp : UInt64? - request = Request.new(Opcode::PLAYER_UPDATE_TIMESTAMP, nil) - - return self.send_request(request) do |bytes| - IO::ByteFormat::NetworkEndian.decode(UInt64, bytes) - end - end - - private def send_request(request : Request, &) - channel = @mux.send(request) - slice = channel.receive - return yield slice - rescue ex - LOGGER.debug("SigHelper: Error when sending a request") - LOGGER.trace(ex.inspect_with_backtrace) - return nil - end - end - - # --------------------- - # Low level functions - # --------------------- - - class Multiplexor - alias TransactionID = UInt32 - record Transaction, channel = ::Channel(Bytes).new - - @prng = Random.new - @mutex = Mutex.new - @queue = {} of TransactionID => Transaction - - @conn : Connection - @uri_or_path : String - - def initialize(@uri_or_path) - @conn = Connection.new(uri_or_path) - listen - end - - def listen : Nil - raise "Socket is closed" if @conn.closed? - - LOGGER.debug("SigHelper: Multiplexor listening") - - spawn do - loop do - begin - receive_data - rescue ex - LOGGER.info("SigHelper: Connection to helper died with '#{ex.message}' trying to reconnect...") - # We close the socket because for some reason is not closed. - @conn.close - loop do - begin - @conn = Connection.new(@uri_or_path) - LOGGER.info("SigHelper: Reconnected to SigHelper!") - rescue ex - LOGGER.debug("SigHelper: Reconnection to helper unsuccessful with error '#{ex.message}'. Retrying") - sleep 500.milliseconds - next - end - break if !@conn.closed? - end - end - Fiber.yield - end - end - end - - def send(request : Request) - transaction = Transaction.new - transaction_id = @prng.rand(TransactionID) - - # Add transaction to queue - @mutex.synchronize do - # On a 32-bits random integer, this should never happen. Though, just in case, ... - if @queue[transaction_id]? - raise Exception.new("SigHelper: Duplicate transaction ID! You got a shiny pokemon!") - end - - @queue[transaction_id] = transaction - end - - write_packet(transaction_id, request) - - return transaction.channel - end - - def receive_data - transaction_id, slice = read_packet - - @mutex.synchronize do - if transaction = @queue.delete(transaction_id) - # Remove transaction from queue and send data to the channel - transaction.channel.send(slice) - LOGGER.trace("SigHelper: Transaction unqueued and data sent to channel") - else - raise Exception.new("SigHelper: Received transaction was not in queue") - end - end - end - - # Read a single packet from the socket - private def read_packet : {TransactionID, Bytes} - # Header - transaction_id = @conn.read_bytes(UInt32, NetworkEndian) - length = @conn.read_bytes(UInt32, NetworkEndian) - - LOGGER.trace("SigHelper: Recv transaction 0x#{transaction_id.to_s(base: 16)} / length #{length}") - - if length > 67_000 - raise Exception.new("SigHelper: Packet longer than expected (#{length})") - end - - # Payload - slice = Bytes.new(length) - @conn.read(slice) if length > 0 - - LOGGER.trace("SigHelper: payload = #{slice}") - LOGGER.trace("SigHelper: Recv transaction 0x#{transaction_id.to_s(base: 16)} - Done") - - return transaction_id, slice - end - - # Write a single packet to the socket - private def write_packet(transaction_id : TransactionID, request : Request) - LOGGER.trace("SigHelper: Send transaction 0x#{transaction_id.to_s(base: 16)} / opcode #{request.opcode}") - - io = IO::Memory.new(1024) - io.write_bytes(request.opcode.to_u8, NetworkEndian) - io.write_bytes(transaction_id, NetworkEndian) - - if payload = request.payload - payload.to_io(io) - end - - @conn.send(io) - @conn.flush - - LOGGER.trace("SigHelper: Send transaction 0x#{transaction_id.to_s(base: 16)} - Done") - end - end - - class Connection - @socket : UNIXSocket | TCPSocket - - {% if flag?(:advanced_debug) %} - @io : IO::Hexdump - {% end %} - - def initialize(host_or_path : String) - case host_or_path - when .starts_with?('/') - # Make sure that the file exists - if File.exists?(host_or_path) - @socket = UNIXSocket.new(host_or_path) - else - raise Exception.new("SigHelper: '#{host_or_path}' no such file") - end - when .starts_with?("tcp://") - uri = URI.parse(host_or_path) - @socket = TCPSocket.new(uri.host.not_nil!, uri.port.not_nil!) - else - uri = URI.parse("tcp://#{host_or_path}") - @socket = TCPSocket.new(uri.host.not_nil!, uri.port.not_nil!) - end - LOGGER.info("SigHelper: Using helper at '#{host_or_path}'") - - {% if flag?(:advanced_debug) %} - @io = IO::Hexdump.new(@socket, output: STDERR, read: true, write: true) - {% end %} - - @socket.sync = false - @socket.blocking = false - end - - def closed? : Bool - return @socket.closed? - end - - def close : Nil - @socket.close if !@socket.closed? - end - - def flush(*args, **options) - @socket.flush(*args, **options) - end - - def send(*args, **options) - @socket.send(*args, **options) - end - - # Wrap IO functions, with added debug tooling if needed - {% for function in %w(read read_bytes write write_bytes) %} - def {{function.id}}(*args, **options) - {% if flag?(:advanced_debug) %} - @io.{{function.id}}(*args, **options) - {% else %} - @socket.{{function.id}}(*args, **options) - {% end %} - end - {% end %} - end -end diff --git a/src/invidious/helpers/signatures.cr b/src/invidious/helpers/signatures.cr deleted file mode 100644 index 82a28fc0..00000000 --- a/src/invidious/helpers/signatures.cr +++ /dev/null @@ -1,53 +0,0 @@ -require "http/params" -require "./sig_helper" - -class Invidious::DecryptFunction - @last_update : Time = Time.utc - 42.days - - def initialize(uri_or_path) - @client = SigHelper::Client.new(uri_or_path) - self.check_update - end - - def check_update - # If we have updated in the last 5 minutes, do nothing - return if (Time.utc - @last_update) < 5.minutes - - # Get the amount of time elapsed since when the player was updated, in the - # event where multiple invidious processes are run in parallel. - update_time_elapsed = (@client.get_player_timestamp || 301).seconds - - if update_time_elapsed > 5.minutes - LOGGER.debug("Signature: Player might be outdated, updating") - @client.force_update - @last_update = Time.utc - end - end - - def decrypt_nsig(n : String) : String? - self.check_update - return @client.decrypt_n_param(n) - rescue ex - LOGGER.debug(ex.message || "Signature: Unknown error") - LOGGER.trace(ex.inspect_with_backtrace) - return nil - end - - def decrypt_signature(str : String) : String? - self.check_update - return @client.decrypt_sig(str) - rescue ex - LOGGER.debug(ex.message || "Signature: Unknown error") - LOGGER.trace(ex.inspect_with_backtrace) - return nil - end - - def get_sts : UInt64? - self.check_update - return @client.get_signature_timestamp - rescue ex - LOGGER.debug(ex.message || "Signature: Unknown error") - LOGGER.trace(ex.inspect_with_backtrace) - return nil - end -end diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index 5637e533..24b20ed9 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -1,3 +1,5 @@ +PUBSUB_URL = URI.parse("https://pubsubhubbub.appspot.com") + # See http://www.evanmiller.org/how-not-to-sort-by-average-rating.html def ci_lower_bound(pos, n) if n == 0 diff --git a/src/invidious/http_server/static_assets_handler.cr b/src/invidious/http_server/static_assets_handler.cr new file mode 100644 index 00000000..7902c95b --- /dev/null +++ b/src/invidious/http_server/static_assets_handler.cr @@ -0,0 +1,120 @@ +{% skip_file if compare_versions(Crystal::VERSION, "1.17.0-dev") < 0 %} + +module Invidious::HttpServer + class StaticAssetsHandler < HTTP::StaticFileHandler + # In addition to storing the actual data of a file, it also implements the required + # getters needed for the object to imitate a `File::Stat` within `StaticFileHandler`. + # + # Since the `File::Stat` is created once in `#call` and then passed around to the + # rest of the class's methods, imitating the object allows us to only lookup + # the cache hash once for every request. + # + private record CachedFile, data : Bytes, size : Int64, modification_time : Time do + def directory? + false + end + + def file? + true + end + end + + CACHE_LIMIT = 5_000_000 # 5MB + @@current_cache_size = 0 + @@cached_files = {} of Path => CachedFile + + # Returns metadata for the requested file + # + # If the requested file is cached, a `CachedFile` is returned instead of a `File::Stat`. + # This represents the metadata info of a cached file and implements all the methods of `File::Stat` that + # is used by the `StaticAssetsHandler`. + # + # The `CachedFile` also stores the raw bytes of the cached file, and this method serves as the place where + # the cached file is retrieved if it exists. Though the data will only be read in `#serve_file` + private def file_info(expanded_path : Path) + file_path = @public_dir.join(expanded_path.to_kind(Path::Kind.native)) + {@@cached_files[file_path]? || File.info?(file_path), file_path} + end + + # Add "Cache-Control" header to the response + private def add_cache_headers(response_headers : HTTP::Headers, last_modified : Time) : Nil + super; response_headers["Cache-Control"] = "max-age=2629800" + end + + # Serves and caches the file at the given path. + # + # This is an override of `serve_file` to allow serving a file from memory, and to cache it + # it as needed. + private def serve_file(context : HTTP::Server::Context, file_info, file_path : Path, original_file_path : Path, last_modified : Time) + context.response.content_type = MIME.from_filename(original_file_path.to_s, "application/octet-stream") + + range_header = context.request.headers["Range"]? + + # If the file is cached we can just directly serve it + if file_info.is_a? CachedFile + return dispatch_serve(context, file_info.data, file_info, range_header) + end + + # Otherwise we'll need to read from disk and cache it + retrieve_bytes_from = IO::Memory.new + File.open(file_path) do |file| + # We cannot cache partial data so we'll rewind and read from the start + if range_header + dispatch_serve(context, file, file_info, range_header) + IO.copy(file.rewind, retrieve_bytes_from) + else + context.response.output = IO::MultiWriter.new(context.response.output, retrieve_bytes_from, sync_close: true) + dispatch_serve(context, file, file_info, range_header) + end + end + + return flush_io_to_cache(retrieve_bytes_from, file_path, file_info) + end + + # Writes file data to the cache + private def flush_io_to_cache(io, file_path, file_info) + if (@@current_cache_size += file_info.size) <= CACHE_LIMIT + @@cached_files[file_path] = CachedFile.new(io.to_slice, file_info.size, file_info.modification_time) + end + end + + # Either send the file in full, or just fragments of it depending on the request + private def dispatch_serve(context, file, file_info, range_header) + if range_header + # an IO is needed for `serve_file_range` + file = file.is_a?(Bytes) ? IO::Memory.new(file, writeable: false) : file + serve_file_range(context, file, range_header, file_info) + else + context.response.headers["Accept-Ranges"] = "bytes" + serve_file_full(context, file, file_info) + end + end + + # If we're serving the full file right away then there's no need for an IO at all. + private def serve_file_full(context : HTTP::Server::Context, file : Bytes, file_info) + context.response.status = :ok + context.response.content_length = file_info.size + context.response.write file + end + + # Serves segments of a file based on the `Range header` + # + # An override of `serve_file_range` to allow using a generic IO rather than a `File`. + # Literally the same code as what we inherited but just with the `file` argument's type + # being set to `IO` rather than `File` + # + # Can be removed once https://github.com/crystal-lang/crystal/issues/15817 is fixed. + private def serve_file_range(context : HTTP::Server::Context, file : IO, range_header : String, file_info) + # Paste in the body of inherited serve_file_range + {{@type.superclass.methods.select(&.name.==("serve_file_range"))[0].body}} + end + + # Clear cached files. + # + # This is only used in the specs to clear the cache before each handler test + def self.clear_cache + @@current_cache_size = 0 + return @@cached_files.clear + end + end +end diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index 7c584d15..eb084331 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -107,7 +107,11 @@ struct Playlist json.field "author", self.author json.field "authorId", self.ucid - json.field "authorUrl", "/channel/#{self.ucid}" + if !self.ucid.empty? + json.field "authorUrl", "/channel/#{self.ucid}" + else + json.field "authorUrl", "" + end json.field "subtitle", self.subtitle json.field "authorThumbnails" do @@ -359,6 +363,9 @@ def fetch_playlist(plid : String) thumbnail = playlist_info.dig?( "thumbnailRenderer", "playlistVideoThumbnailRenderer", "thumbnail", "thumbnails", 0, "url" + ).try &.as_s || playlist_info.dig?( + "thumbnailRenderer", "playlistCustomThumbnailRenderer", + "thumbnail", "thumbnails", 0, "url" ).try &.as_s views = 0_i64 diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index 503b8c05..f8060342 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -104,6 +104,7 @@ module Invidious::Routes::API::V1::Channels json.field "tabs", channel.tabs json.field "tags", channel.tags json.field "authorVerified", channel.verified + json.field "pronouns", channel.pronouns json.field "latestVideos" do json.array do diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index 6a3eb8ae..fc3de695 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -1,6 +1,9 @@ require "html" module Invidious::Routes::API::V1::Videos + private INTERNET_ARCHIVE_URL = URI.parse("https://archive.org") + private CHARS_SAFE = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" + def self.videos(env) locale = env.get("preferences").as(Preferences).locale @@ -279,7 +282,7 @@ module Invidious::Routes::API::V1::Videos file = URI.encode_www_form("#{id[0, 3]}/#{id}.xml") - location = make_client(ARCHIVE_URL, &.get("/download/youtubeannotations_#{index}/#{id[0, 2]}.tar/#{file}")) + location = make_client(INTERNET_ARCHIVE_URL, &.get("/download/youtubeannotations_#{index}/#{id[0, 2]}.tar/#{file}")) if !location.headers["Location"]? env.response.status_code = location.status_code diff --git a/src/invidious/routes/before_all.cr b/src/invidious/routes/before_all.cr index 7b2ffdd6..90958934 100644 --- a/src/invidious/routes/before_all.cr +++ b/src/invidious/routes/before_all.cr @@ -1,4 +1,16 @@ module Invidious::Routes::BeforeAll + struct CompanionCSP + property companion_urls : String = "" + + def initialize + self.companion_urls = CONFIG.invidious_companion.reject(&.builtin_proxy).map do |companion| + "#{companion.public_url.scheme}://#{companion.public_url.host}#{companion.public_url.port ? ":#{companion.public_url.port}" : ""}" + end.join(" ") + end + end + + private COMPANION_CSP = CompanionCSP.new + def self.handle(env) preferences = Preferences.from_json("{}") @@ -35,9 +47,9 @@ module Invidious::Routes::BeforeAll "style-src 'self' 'unsafe-inline'", "img-src 'self' data:", "font-src 'self' data:", - "connect-src 'self'", + "connect-src 'self' " + COMPANION_CSP.companion_urls, "manifest-src 'self'", - "media-src 'self' blob:", + "media-src 'self' blob: " + COMPANION_CSP.companion_urls, "child-src 'self' blob:", "frame-src 'self'", "frame-ancestors " + frame_ancestors, @@ -94,8 +106,8 @@ module Invidious::Routes::BeforeAll end dark_mode = convert_theme(env.params.query["dark_mode"]?) || preferences.dark_mode.to_s - thin_mode = env.params.query["thin_mode"]? || preferences.thin_mode.to_s - thin_mode = thin_mode == "true" + thin_mode = env.params.query["thin_mode"]? + thin_mode = (thin_mode == "true") || preferences.thin_mode locale = env.params.query["hl"]? || preferences.locale preferences.dark_mode = dark_mode diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index 6d2b4465..968d38dc 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -231,8 +231,10 @@ module Invidious::Routes::Channels env.redirect "/post/#{URI.encode_www_form(lb)}?ucid=#{URI.encode_www_form(ucid)}" end - thin_mode = env.params.query["thin_mode"]? || env.get("preferences").as(Preferences).thin_mode - thin_mode = thin_mode == "true" + preferences = env.get("preferences").as(Preferences) + + thin_mode = env.params.query["thin_mode"]? + thin_mode = (thin_mode == "true") || preferences.thin_mode continuation = env.params.query["continuation"]? @@ -264,11 +266,11 @@ module Invidious::Routes::Channels id = env.params.url["id"] ucid = env.params.query["ucid"]? - prefs = env.get("preferences").as(Preferences) + preferences = env.get("preferences").as(Preferences) - locale = prefs.locale + locale = preferences.locale - thin_mode = env.params.query["thin_mode"]? || prefs.thin_mode + thin_mode = env.params.query["thin_mode"]? || preferences.thin_mode thin_mode = thin_mode == "true" nojs = env.params.query["nojs"]? diff --git a/src/invidious/routes/companion.cr b/src/invidious/routes/companion.cr index 11c2e3f5..811393ab 100644 --- a/src/invidious/routes/companion.cr +++ b/src/invidious/routes/companion.cr @@ -1,5 +1,5 @@ module Invidious::Routes::Companion - # /companion + # GET /companion def self.get_companion(env) url = env.request.path if env.request.query @@ -16,6 +16,23 @@ module Invidious::Routes::Companion end end + # POST /companion + def self.post_companion(env) + url = env.request.path + if env.request.query + url += "?#{env.request.query}" + end + + begin + COMPANION_POOL.client do |wrapper| + wrapper.client.post(url, env.request.headers, env.request.body) do |resp| + return self.proxy_companion(env, resp) + end + end + rescue ex + end + end + def self.options_companion(env) url = env.request.path if env.request.query diff --git a/src/invidious/routes/embed.cr b/src/invidious/routes/embed.cr index 6b0887d5..ec5a5804 100644 --- a/src/invidious/routes/embed.cr +++ b/src/invidious/routes/embed.cr @@ -33,7 +33,8 @@ module Invidious::Routes::Embed end def self.show(env) - locale = env.get("preferences").as(Preferences).locale + preferences = env.get("preferences").as(Preferences) + locale = preferences.locale id = env.params.url["id"] plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "") @@ -45,8 +46,6 @@ module Invidious::Routes::Embed env.params.query.delete("playlist") end - preferences = env.get("preferences").as(Preferences) - if id.includes?("%20") || id.includes?("+") || env.params.query.to_s.includes?("%20") || env.params.query.to_s.includes?("+") id = env.params.url["id"].gsub("%20", "").delete("+") @@ -209,17 +208,6 @@ module Invidious::Routes::Embed if CONFIG.invidious_companion.present? invidious_companion = CONFIG.invidious_companion.sample - invidious_companion_urls = CONFIG.invidious_companion.reject(&.builtin_proxy).map do |companion| - uri = - "#{companion.public_url.scheme}://#{companion.public_url.host}#{companion.public_url.port ? ":#{companion.public_url.port}" : ""}" - end.join(" ") - - if !invidious_companion_urls.empty? - env.response.headers["Content-Security-Policy"] = - env.response.headers["Content-Security-Policy"] - .gsub("media-src", "media-src #{invidious_companion_urls}") - .gsub("connect-src", "connect-src #{invidious_companion_urls}") - end end rendered "embed" diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr index 1e0a4085..99ab19f1 100644 --- a/src/invidious/routes/feeds.cr +++ b/src/invidious/routes/feeds.cr @@ -37,12 +37,14 @@ module Invidious::Routes::Feeds end def self.trending(env) - locale = env.get("preferences").as(Preferences).locale + preferences = env.get("preferences").as(Preferences) + locale = preferences.locale + trending_type = env.params.query["type"]? trending_type ||= "Default" region = env.params.query["region"]? - region ||= env.get("preferences").as(Preferences).region + region ||= preferences.region begin trending, plid = fetch_trending(trending_type, region, locale) diff --git a/src/invidious/routes/login.cr b/src/invidious/routes/login.cr index e7de5018..674f0a46 100644 --- a/src/invidious/routes/login.cr +++ b/src/invidious/routes/login.cr @@ -98,6 +98,8 @@ module Invidious::Routes::Login begin validate_request(tokens[0], answer, env.request, HMAC_KEY, locale) + rescue ex : InfoException + return error_template(400, InfoException.new("Erroneous CAPTCHA")) rescue ex return error_template(400, ex) end diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr index f2213da4..56e529b2 100644 --- a/src/invidious/routes/playlists.cr +++ b/src/invidious/routes/playlists.cr @@ -225,10 +225,10 @@ module Invidious::Routes::Playlists end def self.add_playlist_items_page(env) - prefs = env.get("preferences").as(Preferences) - locale = prefs.locale + preferences = env.get("preferences").as(Preferences) + locale = preferences.locale - region = env.params.query["region"]? || prefs.region + region = env.params.query["region"]? || preferences.region user = env.get? "user" sid = env.get? "sid" diff --git a/src/invidious/routes/preferences.cr b/src/invidious/routes/preferences.cr index 5ff56b0b..1c4f4c9d 100644 --- a/src/invidious/routes/preferences.cr +++ b/src/invidious/routes/preferences.cr @@ -2,12 +2,11 @@ module Invidious::Routes::PreferencesRoute def self.show(env) - locale = env.get("preferences").as(Preferences).locale + preferences = env.get("preferences").as(Preferences) + locale = preferences.locale referer = get_referer(env) - preferences = env.get("preferences").as(Preferences) - templated "user/preferences" end diff --git a/src/invidious/routes/routes.cr b/src/invidious/routes/routes.cr new file mode 100644 index 00000000..68b1ff82 --- /dev/null +++ b/src/invidious/routes/routes.cr @@ -0,0 +1,20 @@ +module Invidious::Routes + private REQUEST_HEADERS_WHITELIST = { + "accept", + "accept-encoding", + "cache-control", + "content-length", + "if-none-match", + "range", + } + private RESPONSE_HEADERS_BLACKLIST = { + "access-control-allow-origin", + "alt-svc", + "server", + "cross-origin-opener-policy-report-only", + "report-to", + "cross-origin", + "timing-allow-origin", + "cross-origin-resource-policy", + } +end diff --git a/src/invidious/routes/search.cr b/src/invidious/routes/search.cr index 5b10887c..f8babb72 100644 --- a/src/invidious/routes/search.cr +++ b/src/invidious/routes/search.cr @@ -37,11 +37,11 @@ module Invidious::Routes::Search end def self.search(env) - prefs = env.get("preferences").as(Preferences) - locale = prefs.locale + preferences = env.get("preferences").as(Preferences) + locale = preferences.locale + + region = env.params.query["region"]? || preferences.region - # otherwise, do a normal search - region = env.params.query["region"]? || prefs.region query = Invidious::Search::Query.new(env.params.query, :regular, region) # empty query → show homepage diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index 083087a9..7c01aa36 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -1,4 +1,6 @@ module Invidious::Routes::VideoPlayback + private HTTP_CHUNK_SIZE = 10485760 # ~10MB + # /videoplayback def self.get_video_playback(env) locale = env.get("preferences").as(Preferences).locale diff --git a/src/invidious/routes/watch.cr b/src/invidious/routes/watch.cr index 8a4fa246..b829b0f5 100644 --- a/src/invidious/routes/watch.cr +++ b/src/invidious/routes/watch.cr @@ -2,7 +2,8 @@ module Invidious::Routes::Watch def self.handle(env) - locale = env.get("preferences").as(Preferences).locale + preferences = env.get("preferences").as(Preferences) + locale = preferences.locale region = env.params.query["region"]? if env.params.query.to_s.includes?("%20") || env.params.query.to_s.includes?("+") @@ -38,8 +39,6 @@ module Invidious::Routes::Watch nojs ||= "0" nojs = nojs == "1" - preferences = env.get("preferences").as(Preferences) - user = env.get?("user").try &.as(User) if user subscriptions = user.subscriptions @@ -194,17 +193,6 @@ module Invidious::Routes::Watch if CONFIG.invidious_companion.present? invidious_companion = CONFIG.invidious_companion.sample - invidious_companion_urls = CONFIG.invidious_companion.reject(&.builtin_proxy).map do |companion| - uri = - "#{companion.public_url.scheme}://#{companion.public_url.host}#{companion.public_url.port ? ":#{companion.public_url.port}" : ""}" - end.join(" ") - - if !invidious_companion_urls.empty? - env.response.headers["Content-Security-Policy"] = - env.response.headers["Content-Security-Policy"] - .gsub("media-src", "media-src #{invidious_companion_urls}") - .gsub("connect-src", "connect-src #{invidious_companion_urls}") - end end templated "watch" diff --git a/src/invidious/routing.cr b/src/invidious/routing.cr index a51bb4b6..32e8554c 100644 --- a/src/invidious/routing.cr +++ b/src/invidious/routing.cr @@ -227,6 +227,7 @@ module Invidious::Routing def register_companion_routes if CONFIG.invidious_companion.present? get "/companion/*", Routes::Companion, :get_companion + post "/companion/*", Routes::Companion, :post_companion options "/companion/*", Routes::Companion, :options_companion end end diff --git a/src/invidious/search/filters.cr b/src/invidious/search/filters.cr index bc2715cf..d94bfc30 100644 --- a/src/invidious/search/filters.cr +++ b/src/invidious/search/filters.cr @@ -57,8 +57,6 @@ module Invidious::Search # Values correspond to { "1:varint": } enum Sort Relevance = 0 - Rating = 1 - Date = 2 Views = 3 end diff --git a/src/invidious/trending.cr b/src/invidious/trending.cr index e289ed5b..622fe517 100644 --- a/src/invidious/trending.cr +++ b/src/invidious/trending.cr @@ -4,20 +4,21 @@ def fetch_trending(trending_type, region, locale) plid = nil - browse_id = "FEtrending" + browse_id = "" case trending_type.try &.downcase - when "music" - params = "4gINGgt5dG1hX2NoYXJ0cw%3D%3D" when "gaming" - params = "4gIcGhpnYW1pbmdfY29ycHVzX21vc3RfcG9wdWxhcg%3D%3D" - when "movies" - params = "4gIKGgh0cmFpbGVycw%3D%3D" + browse_id = "UCOpNcN46UbXVtpKMrmU4Abg" + params = "Egh0cmVuZGluZw%3D%3D" when "livestreams" browse_id = "UC4R8DWoMoI7CAwX8_LjQHig" params = "EgdsaXZldGFikgEDCKEK" - else # Default - params = "" + else + # Livestreams is the default one as Youtube removed + # the aggregated trending page + # https://github.com/iv-org/invidious/issues/5397#issuecomment-3218928458 + browse_id = "UC4R8DWoMoI7CAwX8_LjQHig" + params = "EgdsaXZldGFikgEDCKEK" end client_config = YoutubeAPI::ClientConfig.new(region: region) diff --git a/src/invidious/user/imports.cr b/src/invidious/user/imports.cr index 007eb666..7c4101cc 100644 --- a/src/invidious/user/imports.cr +++ b/src/invidious/user/imports.cr @@ -30,28 +30,24 @@ struct Invidious::User return subscriptions end - def parse_playlist_export_csv(user : User, raw_input : String) + # Parse a CSV Google Takeout - Youtube Playlist file + def parse_playlist_export_csv(user : User, playlist_name : String, raw_input : String) # Split the input into head and body content - raw_head, raw_body = raw_input.strip('\n').split("\n\n", limit: 2, remove_empty: true) + raw_head, raw_body = raw_input.split("\n", limit: 2, remove_empty: true) # Create the playlist from the head content csv_head = CSV.new(raw_head.strip('\n'), headers: true) csv_head.next - title = csv_head[4] - description = csv_head[5] - visibility = csv_head[6] + title = playlist_name - if visibility.compare("Public", case_insensitive: true) == 0 - privacy = PlaylistPrivacy::Public - else - privacy = PlaylistPrivacy::Private - end + description = "This is the default description of an imported playlist. Feel Free to change it as you see fit." + privacy = PlaylistPrivacy::Private playlist = create_playlist(title, privacy, user) Invidious::Database::Playlists.update_description(playlist.id, description) # Add each video to the playlist from the body content - csv_body = CSV.new(raw_body.strip('\n'), headers: true) + csv_body = CSV.new(raw_body.strip('\n'), headers: false) csv_body.each do |row| video_id = row[0] if playlist @@ -204,10 +200,12 @@ struct Invidious::User end def from_youtube_pl(user : User, body : String, filename : String, type : String) : Bool - extension = filename.split(".").last + filename_array = filename.split(".") + playlist_name = filename_array.first + extension = filename_array.last if extension == "csv" || type == "text/csv" - playlist = parse_playlist_export_csv(user, body) + playlist = parse_playlist_export_csv(user, playlist_name, body) if playlist return true else diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 348a0a66..0446922f 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -326,6 +326,14 @@ end def fetch_video(id, region) info = extract_video_info(video_id: id) + if info.nil? + raise InfoException.new("Invidious companion is not available. \ + Video playback cannot continue. \ + If you are the administrator of this instance, install Invidious companion \ + following the installation instructions \ + https://docs.invidious.io/installation/") + end + if reason = info["reason"]? if reason == "Video unavailable" raise NotFoundException.new(reason.as_s || "") diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 6038dfcf..8114ad68 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -53,11 +53,12 @@ def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)? end def extract_video_info(video_id : String) - # Init client config for the API - client_config = YoutubeAPI::ClientConfig.new - # Fetch data from the player endpoint - player_response = YoutubeAPI.player(video_id: video_id, params: "2AMB", client_config: client_config) + player_response = YoutubeAPI.player(video_id: video_id) + + if player_response.nil? + return nil + end playability_status = player_response.dig?("playabilityStatus", "status").try &.as_s @@ -105,37 +106,6 @@ def extract_video_info(video_id : String) params = parse_video_info(video_id, player_response) params["reason"] = JSON::Any.new(reason) if reason - if !CONFIG.invidious_companion.present? - if player_response.dig?("streamingData", "adaptiveFormats", 0, "url").nil? - LOGGER.warn("Missing URLs for adaptive formats, falling back to other YT clients.") - players_fallback = {YoutubeAPI::ClientType::TvSimply, YoutubeAPI::ClientType::WebMobile} - - players_fallback.each do |player_fallback| - client_config.client_type = player_fallback - - next if !(player_fallback_response = try_fetch_streaming_data(video_id, client_config)) - - adaptive_formats = player_fallback_response.dig?("streamingData", "adaptiveFormats") - if adaptive_formats && (adaptive_formats.dig?(0, "url") || adaptive_formats.dig?(0, "signatureCipher")) - streaming_data = player_response["streamingData"].as_h - streaming_data["adaptiveFormats"] = adaptive_formats - player_response["streamingData"] = JSON::Any.new(streaming_data) - break - end - rescue InfoException - next LOGGER.warn("Failed to fetch streams with #{player_fallback}") - end - end - - # Seems like video page can still render even without playable streams. - # its better than nothing. - # - # # Were we able to find playable video streams? - # if player_response.dig?("streamingData", "adaptiveFormats", 0, "url").nil? - # # No :( - # end - end - {"captions", "playabilityStatus", "playerConfig", "storyboards"}.each do |f| params[f] = player_response[f] if player_response[f]? end @@ -163,7 +133,7 @@ end def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig) : Hash(String, JSON::Any)? LOGGER.debug("try_fetch_streaming_data: [#{id}] Using #{client_config.client_type} client.") - response = YoutubeAPI.player(video_id: id, params: "2AMB", client_config: client_config) + response = YoutubeAPI.player(video_id: id) playability_status = response["playabilityStatus"]["status"] LOGGER.debug("try_fetch_streaming_data: [#{id}] Got playabilityStatus == #{playability_status}.") @@ -475,26 +445,15 @@ end private def convert_url(fmt) if cfr = fmt["signatureCipher"]?.try { |json| HTTP::Params.parse(json.as_s) } - sp = cfr["sp"] url = URI.parse(cfr["url"]) params = url.query_params LOGGER.debug("convert_url: Decoding '#{cfr}'") - - unsig = DECRYPT_FUNCTION.try &.decrypt_signature(cfr["s"]) - params[sp] = unsig if unsig else url = URI.parse(fmt["url"].as_s) params = url.query_params end - n = DECRYPT_FUNCTION.try &.decrypt_nsig(params["n"]) - params["n"] = n if n - - if token = CONFIG.po_token - params["pot"] = token - end - url.query_params = params LOGGER.trace("convert_url: new url is '#{url}'") diff --git a/src/invidious/views/components/channel_info.ecr b/src/invidious/views/components/channel_info.ecr index f4164f31..97a2d7da 100644 --- a/src/invidious/views/components/channel_info.ecr +++ b/src/invidious/views/components/channel_info.ecr @@ -12,7 +12,10 @@
- <%= author %><% if !channel.verified.nil? && channel.verified %> <% end %> +
+ <%= author %><% if !channel.verified.nil? && channel.verified %> <% end %> + <% if !channel.pronouns.nil? %>
<%= channel.pronouns %><% end %> +
diff --git a/src/invidious/views/components/player.ecr b/src/invidious/views/components/player.ecr index 85fa4373..fbd472e0 100644 --- a/src/invidious/views/components/player.ecr +++ b/src/invidious/views/components/player.ecr @@ -1,3 +1,6 @@ +<% + invidious_companion_check_id = invidious_companion_encrypt(video.id) if invidious_companion +%> diff --git a/src/invidious/views/embed.ecr b/src/invidious/views/embed.ecr index 1bf5cc3e..5551cd0a 100644 --- a/src/invidious/views/embed.ecr +++ b/src/invidious/views/embed.ecr @@ -1,5 +1,5 @@ -"> + diff --git a/src/invidious/views/feeds/trending.ecr b/src/invidious/views/feeds/trending.ecr index 69483f30..46d02ad4 100644 --- a/src/invidious/views/feeds/trending.ecr +++ b/src/invidious/views/feeds/trending.ecr @@ -21,7 +21,7 @@
- <% {"Default", "Music", "Gaming", "Movies", "Livestreams"}.each do |option| %> + <% {"Livestreams", "Gaming"}.each do |option| %>
<% if trending_type == option %> <%= translate(locale, option) %> diff --git a/src/invidious/views/post.ecr b/src/invidious/views/post.ecr index fb03a44c..f644d634 100644 --- a/src/invidious/views/post.ecr +++ b/src/invidious/views/post.ecr @@ -38,7 +38,7 @@ "params" => { "comments": ["youtube"] }, - "preferences" => prefs, + "preferences" => preferences, "base_url" => "/api/v1/post/#{URI.encode_www_form(id)}/comments", "ucid" => ucid }.to_pretty_json diff --git a/src/invidious/views/template.ecr b/src/invidious/views/template.ecr index 9904b4fc..40f5544f 100644 --- a/src/invidious/views/template.ecr +++ b/src/invidious/views/template.ecr @@ -1,6 +1,7 @@ <% - locale = env.get("preferences").as(Preferences).locale - dark_mode = env.get("preferences").as(Preferences).dark_mode + preferences = env.get("preferences").as(Preferences) + locale = preferences.locale + dark_mode = preferences.dark_mode %> @@ -149,7 +150,24 @@ <%= translate(locale, "footer_donate_page") %> - <%= translate(locale, "Current version: ") %> <%= CURRENT_VERSION %>-<%= CURRENT_COMMIT %> @ <%= CURRENT_BRANCH %> + + <%= translate(locale, "Current version: ") %> + <% if CONFIG.modified_source_code_url %> + <%= CURRENT_VERSION %>-<%= CURRENT_COMMIT %> + <% else %> + <%= CURRENT_VERSION %>-<%= CURRENT_COMMIT %> + <% end %> + @ <%= CURRENT_BRANCH %> + <% if CURRENT_TAG != "" %> + ( + <% if CONFIG.modified_source_code_url %> + <%= CURRENT_TAG %> + <% else %> + <%= CURRENT_TAG %> + <% end %> + ) + <% end %> +
diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr index 89632dc5..7cf6c51c 100644 --- a/src/invidious/views/watch.ecr +++ b/src/invidious/views/watch.ecr @@ -79,11 +79,11 @@ we're going to need to do it here in order to allow for translations.

<%= title %> <% if params.listen %> - " href="/watch?<%= env.params.query %>&listen=0"> + " id="link-iv-listen" data-base-url="/watch?<%= env.params.query %>&listen=0" href="/watch?<%= env.params.query %>&listen=0"> <% else %> - " href="/watch?<%= env.params.query %>&listen=1"> + " id="link-iv-listen" data-base-url="/watch?<%= env.params.query %>&listen=1" href="/watch?<%= env.params.query %>&listen=1"> <% end %> @@ -125,7 +125,7 @@ we're going to need to do it here in order to allow for translations. end -%> <%= translate(locale, "videoinfo_watch_on_youTube") %> - (<%= translate(locale, "videoinfo_youTube_embed_link") %>) + (<%= translate(locale, "videoinfo_youTube_embed_link") %>)

@@ -230,7 +230,7 @@ we're going to need to do it here in order to allow for translations. <% if !video.author_thumbnail.empty? %> <% end %> - <%= author %><% if !video.author_verified.nil? && video.author_verified %> <% end %> + <%= author %><% if !video.author_verified.nil? && video.author_verified %> <% end %>

diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr index 85f6caa5..04e00f20 100644 --- a/src/invidious/yt_backend/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -442,6 +442,7 @@ private module Parsers if content_container = special_category_container["horizontalListRenderer"]? elsif content_container = special_category_container["expandedShelfContentsRenderer"]? elsif content_container = special_category_container["verticalListRenderer"]? + elsif content_container = special_category_container["gridRenderer"]? else # Anything else, such as `horizontalMovieListRenderer` is currently unsupported. return diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 6fa8ae0e..dd709920 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -199,10 +199,6 @@ module YoutubeAPI # conf_1 = ClientConfig.new(region: "NO") # YoutubeAPI::search("Kollektivet", params: "", client_config: conf_1) # - # # Use the Android client to request video streams URLs - # conf_2 = ClientConfig.new(client_type: ClientType::Android) - # YoutubeAPI::player(video_id: "dQw4w9WgXcQ", client_config: conf_2) - # # struct ClientConfig # Type of client to emulate. @@ -335,10 +331,6 @@ module YoutubeAPI client_context["client"]["platform"] = platform end - if CONFIG.visitor_data.is_a?(String) - client_context["client"]["visitorData"] = CONFIG.visitor_data.as(String) - end - return client_context end @@ -455,61 +447,23 @@ module YoutubeAPI end #################################################################### - # player(video_id, params, client_config?) + # player(video_id) # - # Requests the youtubei/v1/player endpoint with the required headers - # and POST data in order to get a JSON reply. + # Requests the youtubei/v1/player Invidious Companion endpoint with + # the requested video ID. # - # The requested data is a video ID (`v=` parameter), with some - # additional parameters, formatted as a base64 string. + # The requested data is a video ID (`v=` parameter). # - # An optional ClientConfig parameter can be passed, too (see - # `struct ClientConfig` above for more details). - # - def player( - video_id : String, - *, # Force the following parameters to be passed by name - params : String, - client_config : ClientConfig | Nil = nil, - ) - # Playback context, separate because it can be different between clients - playback_ctx = { - "html5Preference" => "HTML5_PREF_WANTS", - "referer" => "https://www.youtube.com/watch?v=#{video_id}", - } of String => String | Int64 - - if {"WEB", "TVHTML5"}.any? { |s| client_config.name.starts_with? s } - if sts = DECRYPT_FUNCTION.try &.get_sts - playback_ctx["signatureTimestamp"] = sts.to_i64 - end - end - - # JSON Request data, required by the API + def player(video_id : String) + # JSON Request data, required by Invidious Companion data = { - "contentCheckOk" => true, - "videoId" => video_id, - "context" => self.make_context(client_config, video_id), - "racyCheckOk" => true, - "user" => { - "lockedSafetyMode" => false, - }, - "playbackContext" => { - "contentPlaybackContext" => playback_ctx, - }, - "serviceIntegrityDimensions" => { - "poToken" => CONFIG.po_token, - }, + "videoId" => video_id, } - # Append the additional parameters if those were provided - if params != "" - data["params"] = params - end - if CONFIG.invidious_companion.present? return self._post_invidious_companion("/youtubei/v1/player", data) else - return self._post_json("/youtubei/v1/player", data, client_config) + return nil end end @@ -635,10 +589,6 @@ module YoutubeAPI headers["User-Agent"] = user_agent end - if CONFIG.visitor_data.is_a?(String) - headers["X-Goog-Visitor-Id"] = CONFIG.visitor_data.as(String) - end - # Logging LOGGER.debug("YoutubeAPI: Using endpoint: \"#{endpoint}\"") LOGGER.trace("YoutubeAPI: ClientConfig: #{client_config}")