VeritableValor/invidious
1
0
Fork 0
mirror of https://github.com/iv-org/invidious.git synced 2026-09-26 11:35:27 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge 4a5a6da0b709939716e380465fbbe3d72cfea8e4 into 248c315b4e8fa67c1b6c3b7bba6d1e77ba32b3b4

Browse Source
This commit is contained in:
Fijxu 2026-09-15 21:16:00 +00:00 committed by GitHub
commit 286041849d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
58 changed files with 2270 additions and 2007 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
spec/invidious/channel/community_spec.cr Normal file
View File

File diff suppressed because one or more lines are too long

26
spec/invidious/helpers/tokens_spec.cr Normal file
View File

@ -0,0 +1,26 @@
require "../../spec_helper"
Spectator.describe Invidious::Helpers::Tokens do
describe ".sign_token" do
it "correctly signs a given hash" do
token = {
"session" => "v1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
"expires" => 1554680038,
"scopes" => [
":notifications",
":subscriptions/*",
"GET:tokens*",
],
"signature" => "f__2hS20th8pALF305PJFK-D2aVtvefNnQheILHD2vU=",
}
expect(Invidious::Helpers::Tokens.sign_token("SECRET_KEY", token)).to eq(token["signature"])
token = {
"session" => "v1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
"scopes" => [":notifications", "POST:subscriptions/*"],
"signature" => "fNvXoT0MRAL9eE6lTE33CEg8HitYJDOL9a22rSN2Ihg=",
}
expect(Invidious::Helpers::Tokens.sign_token("SECRET_KEY", token)).to eq(token["signature"])
end
end
end

56
spec/invidious/helpers_spec.cr
View File

File diff suppressed because one or more lines are too long

11
spec/invidious/search/ctoken_spec.cr Normal file
View File

@ -0,0 +1,11 @@
require "../../spec_helper"
Spectator.describe Invidious::Search::CToken do
describe ".produce_channel_search_continuation" do
it "correctly produces token for searching a specific channel" do
expect(Invidious::Search::CToken.produce_channel_search_continuation("UCXuqSBlHAE6Xw-yeJA0Tunw", "", 100)).to eq("4qmFsgJqEhhVQ1h1cVNCbEhBRTZYdy15ZUpBMFR1bncaIEVnWnpaV0Z5WTJnd0FUZ0JZQUY2QkVkS2IxaTRBUUE9WgCaAilicm93c2UtZmVlZFVDWHVxU0JsSEFFNlh3LXllSkEwVHVud3NlYXJjaA%3D%3D")
expect(Invidious::Search::CToken.produce_channel_search_continuation("UCXuqSBlHAE6Xw-yeJA0Tunw", "По ожиशुपतिरपि子而時ஸ்றீனி", 0)).to eq("4qmFsgKoARIYVUNYdXFTQmxIQUU2WHcteWVKQTBUdW53GiBFZ1p6WldGeVkyZ3dBVGdCWUFGNkJFZEJRVDI0QVFBPVo-0J_QviDQvtC20LjgpLbgpYHgpKrgpKTgpL_gpLDgpKrgpL_lrZDogIzmmYLgrrjgr43grrHgr4Dgrqngrr-aAilicm93c2UtZmVlZFVDWHVxU0JsSEFFNlh3LXllSkEwVHVud3NlYXJjaA%3D%3D")
end
end
end

2
spec/invidious/hashtag_spec.cr → spec/invidious/search/hashtag_spec.cr
View File

@ -1,4 +1,4 @@
require "../parsers_helper.cr" require "../../parsers_helper.cr"
Spectator.describe Invidious::Hashtag do Spectator.describe Invidious::Hashtag do
it "parses richItemRenderer containers (test 1)" do it "parses richItemRenderer containers (test 1)" do

1
spec/spec_helper.cr
View File

@ -9,7 +9,6 @@ require "../src/invidious/videos/caption"
require "../src/invidious/videos" require "../src/invidious/videos"
require "../src/invidious/playlists" require "../src/invidious/playlists"
require "../src/invidious/search/ctoken" require "../src/invidious/search/ctoken"
require "../src/invidious/trending"
require "spectator" require "spectator"
Spectator.configure do |config| Spectator.configure do |config|

4
src/invidious.cr
View File

@ -44,6 +44,8 @@ require "./invidious/jsonify/**"
require "./invidious/*" require "./invidious/*"
require "./invidious/comments/*" require "./invidious/comments/*"
require "./invidious/channels/*" require "./invidious/channels/*"
require "./invidious/playlists/*"
require "./invidious/feeds/*"
require "./invidious/user/*" require "./invidious/user/*"
require "./invidious/search/*" require "./invidious/search/*"
require "./invidious/routes/**" require "./invidious/routes/**"
@ -230,7 +232,7 @@ error 404 do |env|
end end
error 500 do |env, exception| error 500 do |env, exception|
error_template(500, exception) Errors.error_template(500, exception)
end end
# Init Kemal # Init Kemal

340
src/invidious/channels/about.cr
View File

@ -19,202 +19,206 @@ record AboutChannel,
verified : Bool, verified : Bool,
is_age_gated : Bool is_age_gated : Bool
def get_about_info(ucid) : AboutChannel module Invidious::Channels::About
begin extend self
# Fetch channel information from channel home page
initdata = YoutubeAPI.browse(browse_id: ucid, params: "")
rescue
raise InfoException.new("Could not get channel info.")
end
if initdata.dig?("alerts", 0, "alertRenderer", "type") == "ERROR" def get_about_info(ucid) : AboutChannel
error_message = initdata["alerts"][0]["alertRenderer"]["text"]["simpleText"].as_s begin
if error_message == "This channel does not exist." # Fetch channel information from channel home page
raise NotFoundException.new(error_message) initdata = YoutubeAPI.browse(browse_id: ucid, params: "")
else rescue
raise InfoException.new(error_message) raise InfoException.new("Could not get channel info.")
end end
end
if browse_endpoint = initdata["onResponseReceivedActions"]?.try &.[0]?.try &.["navigateAction"]?.try &.["endpoint"]?.try &.["browseEndpoint"]? if initdata.dig?("alerts", 0, "alertRenderer", "type") == "ERROR"
raise ChannelRedirect.new(channel_id: browse_endpoint["browseId"].to_s) error_message = initdata["alerts"][0]["alertRenderer"]["text"]["simpleText"].as_s
end if error_message == "This channel does not exist."
raise NotFoundException.new(error_message)
else
raise InfoException.new(error_message)
end
end
auto_generated = false if browse_endpoint = initdata["onResponseReceivedActions"]?.try &.[0]?.try &.["navigateAction"]?.try &.["endpoint"]?.try &.["browseEndpoint"]?
# Check for special auto generated gaming channels raise ChannelRedirect.new(channel_id: browse_endpoint["browseId"].to_s)
if !initdata.has_key?("metadata") end
auto_generated = true
end
tags = [] of String
tab_names = [] of String
total_views = 0_i64
joined = Time.unix(0)
if age_gate_renderer = initdata.dig?("contents", "twoColumnBrowseResultsRenderer", "tabs", 0, "tabRenderer", "content", "sectionListRenderer", "contents", 0, "channelAgeGateRenderer")
description_node = nil
author = age_gate_renderer["channelTitle"].as_s
ucid = initdata.dig("responseContext", "serviceTrackingParams", 0, "params", 0, "value").as_s
author_url = "https://www.youtube.com/channel/#{ucid}"
author_thumbnail = age_gate_renderer.dig("avatar", "thumbnails", 0, "url").as_s
banner = nil
is_family_friendly = false
is_age_gated = true
tab_names = ["videos", "shorts", "streams"]
auto_generated = false auto_generated = false
else # Check for special auto generated gaming channels
if auto_generated if !initdata.has_key?("metadata")
author = initdata["header"]["interactiveTabbedHeaderRenderer"]["title"]["simpleText"].as_s auto_generated = true
author_url = initdata["microformat"]["microformatDataRenderer"]["urlCanonical"].as_s end
author_thumbnail = initdata["header"]["interactiveTabbedHeaderRenderer"]["boxArt"]["thumbnails"][0]["url"].as_s
# Raises a KeyError on failure. tags = [] of String
banners = initdata["header"]["interactiveTabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? tab_names = [] of String
banner = banners.try &.[-1]?.try &.["url"].as_s? total_views = 0_i64
joined = Time.unix(0)
description_base_node = initdata["header"]["interactiveTabbedHeaderRenderer"]["description"] if age_gate_renderer = initdata.dig?("contents", "twoColumnBrowseResultsRenderer", "tabs", 0, "tabRenderer", "content", "sectionListRenderer", "contents", 0, "channelAgeGateRenderer")
# some channels have the description in a simpleText description_node = nil
# ex: https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/ author = age_gate_renderer["channelTitle"].as_s
description_node = description_base_node.dig?("simpleText") || description_base_node ucid = initdata.dig("responseContext", "serviceTrackingParams", 0, "params", 0, "value").as_s
author_url = "https://www.youtube.com/channel/#{ucid}"
tags = initdata.dig?("header", "interactiveTabbedHeaderRenderer", "badges") author_thumbnail = age_gate_renderer.dig("avatar", "thumbnails", 0, "url").as_s
.try &.as_a.map(&.["metadataBadgeRenderer"]["label"].as_s) || [] of String banner = nil
is_family_friendly = false
is_age_gated = true
tab_names = ["videos", "shorts", "streams"]
auto_generated = false
else else
author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s if auto_generated
author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s author = initdata["header"]["interactiveTabbedHeaderRenderer"]["title"]["simpleText"].as_s
author_thumbnail = initdata["metadata"]["channelMetadataRenderer"]["avatar"]["thumbnails"][0]["url"].as_s author_url = initdata["microformat"]["microformatDataRenderer"]["urlCanonical"].as_s
author_badge = initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "title", "dynamicTextViewModel", "text", "attachmentRuns", 0, "element", "type", "imageType", "image", "sources", 0, "clientResource", "imageName") author_thumbnail = initdata["header"]["interactiveTabbedHeaderRenderer"]["boxArt"]["thumbnails"][0]["url"].as_s
.try &.as_s
# CHECK_CIRCLE_FILLED is used for normal channels and AUDIO_BADGE if used For
# music/artist channels
# TODO: Maybe separate verified author from verified artist?
author_verified = author_badge.try { |badge| badge == "CHECK_CIRCLE_FILLED" || badge == "AUDIO_BADGE" } || false
ucid = initdata["metadata"]["channelMetadataRenderer"]["externalId"].as_s
# Raises a KeyError on failure. # Raises a KeyError on failure.
# TODO: Check if `c4TabbedHeaderRenderer` still exists on some channels. banners = initdata["header"]["interactiveTabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]?
banners = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]? banner = banners.try &.[-1]?.try &.["url"].as_s?
banners ||= initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "banner", "imageBannerViewModel", "image", "sources")
banner = banners.try &.[-1]?.try &.["url"].as_s?
# if banner.includes? "channels/c4/default_banner" description_base_node = initdata["header"]["interactiveTabbedHeaderRenderer"]["description"]
# banner = nil # some channels have the description in a simpleText
# end # ex: https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/
description_node = description_base_node.dig?("simpleText") || description_base_node
description_node = initdata["metadata"]["channelMetadataRenderer"]?.try &.["description"]? tags = initdata.dig?("header", "interactiveTabbedHeaderRenderer", "badges")
tags = initdata.dig?("microformat", "microformatDataRenderer", "tags").try &.as_a.map(&.as_s) || [] of String .try &.as_a.map(&.["metadataBadgeRenderer"]["label"].as_s) || [] of String
end else
author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s
author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s
author_thumbnail = initdata["metadata"]["channelMetadataRenderer"]["avatar"]["thumbnails"][0]["url"].as_s
author_badge = initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "title", "dynamicTextViewModel", "text", "attachmentRuns", 0, "element", "type", "imageType", "image", "sources", 0, "clientResource", "imageName")
.try &.as_s
# CHECK_CIRCLE_FILLED is used for normal channels and AUDIO_BADGE if used For
# music/artist channels
# TODO: Maybe separate verified author from verified artist?
author_verified = author_badge.try { |badge| badge == "CHECK_CIRCLE_FILLED" || badge == "AUDIO_BADGE" } || false
ucid = initdata["metadata"]["channelMetadataRenderer"]["externalId"].as_s
is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool # Raises a KeyError on failure.
if tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]? # TODO: Check if `c4TabbedHeaderRenderer` still exists on some channels.
# Get the name of the tabs available on this channel banners = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]?
tab_names = tabs_json.as_a.compact_map do |entry| banners ||= initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "banner", "imageBannerViewModel", "image", "sources")
name = entry.dig?("tabRenderer", "title").try &.as_s.downcase banner = banners.try &.[-1]?.try &.["url"].as_s?
# This is a small fix to not add extra code on the HTML side # if banner.includes? "channels/c4/default_banner"
# I.e, the URL for the "live" tab is .../streams, so use "streams" # banner = nil
# everywhere for the sake of simplicity # end
(name == "live") ? "streams" : name
description_node = initdata["metadata"]["channelMetadataRenderer"]?.try &.["description"]?
tags = initdata.dig?("microformat", "microformatDataRenderer", "tags").try &.as_a.map(&.as_s) || [] of String
end end
# Get the currently active tab ("About") is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool
about_tab = extract_selected_tab(tabs_json) if tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]?
# Get the name of the tabs available on this channel
tab_names = tabs_json.as_a.compact_map do |entry|
name = entry.dig?("tabRenderer", "title").try &.as_s.downcase
# Try to find the about metadata section # This is a small fix to not add extra code on the HTML side
channel_about_meta = about_tab.dig?( # I.e, the URL for the "live" tab is .../streams, so use "streams"
"content", # everywhere for the sake of simplicity
"sectionListRenderer", "contents", 0, (name == "live") ? "streams" : name
"itemSectionRenderer", "contents", 0, end
"channelAboutFullMetadataRenderer"
)
if !channel_about_meta.nil? # Get the currently active tab ("About")
total_views = channel_about_meta.dig?("viewCountText", "simpleText").try &.as_s.gsub(/\D/, "").to_i64? || 0_i64 about_tab = extract_selected_tab(tabs_json)
# The joined text is split to several sub strings. The reduce joins those strings before parsing the date. # Try to find the about metadata section
joined = extract_text(channel_about_meta["joinedDateText"]?) channel_about_meta = about_tab.dig?(
.try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0) "content",
"sectionListRenderer", "contents", 0,
# Normal Auto-generated channels "itemSectionRenderer", "contents", 0,
# https://support.google.com/youtube/answer/2579942 "channelAboutFullMetadataRenderer"
# For auto-generated channels, channel_about_meta only has
# ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"]
auto_generated = (
(channel_about_meta["primaryLinks"]?.try &.size) == 1 && \
extract_text(channel_about_meta.dig?("primaryLinks", 0, "title")) == "Auto-generated by YouTube" ||
channel_about_meta.dig?("links", 0, "channelExternalLinkViewModel", "title", "content").try &.as_s == "Auto-generated by YouTube"
) )
if !channel_about_meta.nil?
total_views = channel_about_meta.dig?("viewCountText", "simpleText").try &.as_s.gsub(/\D/, "").to_i64? || 0_i64
# The joined text is split to several sub strings. The reduce joins those strings before parsing the date.
joined = extract_text(channel_about_meta["joinedDateText"]?)
.try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0)
# Normal Auto-generated channels
# https://support.google.com/youtube/answer/2579942
# For auto-generated channels, channel_about_meta only has
# ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"]
auto_generated = (
(channel_about_meta["primaryLinks"]?.try &.size) == 1 && \
extract_text(channel_about_meta.dig?("primaryLinks", 0, "title")) == "Auto-generated by YouTube" ||
channel_about_meta.dig?("links", 0, "channelExternalLinkViewModel", "title", "content").try &.as_s == "Auto-generated by YouTube"
)
end
end end
end end
end
allowed_regions = initdata allowed_regions = initdata
.dig?("microformat", "microformatDataRenderer", "availableCountries") .dig?("microformat", "microformatDataRenderer", "availableCountries")
.try &.as_a.map(&.as_s) || [] of String .try &.as_a.map(&.as_s) || [] of String
description = !description_node.nil? ? description_node.as_s : "" description = !description_node.nil? ? description_node.as_s : ""
description_html = HTML.escape(description) description_html = HTML.escape(description)
if !description_node.nil? if !description_node.nil?
if description_node.as_h?.nil? if description_node.as_h?.nil?
description_node = text_to_parsed_content(description_node.as_s) description_node = text_to_parsed_content(description_node.as_s)
end
description_html = parse_content(description_node)
if description_html == "" && description != ""
description_html = HTML.escape(description)
end
end
sub_count = 0
pronouns = nil
if (metadata_rows = initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "metadata", "contentMetadataViewModel", "metadataRows").try &.as_a)
metadata_rows.each do |row|
subscribe_metadata_part = row.dig?("metadataParts").try &.as_a.find { |i| i.dig?("text", "content").try &.as_s.includes?("subscribers") }
if !subscribe_metadata_part.nil?
sub_count = short_text_to_number(subscribe_metadata_part.dig("text", "content").as_s.split(" ")[0]).to_i32
end end
description_html = parse_content(description_node)
pronoun_metadata_part = row.dig?("metadataParts").try &.as_a.find { |i| i.dig?("tooltip").try &.as_s.includes?("Pronouns") } if description_html == "" && description != ""
if !pronoun_metadata_part.nil? description_html = HTML.escape(description)
pronouns = pronoun_metadata_part.dig("text", "content").as_s
end end
break if sub_count != 0 && !pronouns.nil?
end end
sub_count = 0
pronouns = nil
if (metadata_rows = initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "metadata", "contentMetadataViewModel", "metadataRows").try &.as_a)
metadata_rows.each do |row|
subscribe_metadata_part = row.dig?("metadataParts").try &.as_a.find { |i| i.dig?("text", "content").try &.as_s.includes?("subscribers") }
if !subscribe_metadata_part.nil?
sub_count = short_text_to_number(subscribe_metadata_part.dig("text", "content").as_s.split(" ")[0]).to_i32
end
pronoun_metadata_part = row.dig?("metadataParts").try &.as_a.find { |i| i.dig?("tooltip").try &.as_s.includes?("Pronouns") }
if !pronoun_metadata_part.nil?
pronouns = pronoun_metadata_part.dig("text", "content").as_s
end
break if sub_count != 0 && !pronouns.nil?
end
end
AboutChannel.new(
ucid: ucid,
author: author,
auto_generated: auto_generated,
author_url: author_url,
author_thumbnail: author_thumbnail,
banner: banner,
description: description,
description_html: description_html,
total_views: total_views,
sub_count: sub_count,
joined: joined,
is_family_friendly: is_family_friendly,
pronouns: pronouns,
allowed_regions: allowed_regions,
tabs: tab_names,
tags: tags,
verified: author_verified || false,
is_age_gated: is_age_gated || false,
)
end end
AboutChannel.new( def fetch_related_channels(about_channel : AboutChannel, continuation : String? = nil) : {Array(SearchChannel), String?}
ucid: ucid, if continuation.nil?
author: author, # params is {"2:string":"channels"} encoded
auto_generated: auto_generated, initial_data = YoutubeAPI.browse(browse_id: about_channel.ucid, params: "EghjaGFubmVscw%3D%3D")
author_url: author_url, else
author_thumbnail: author_thumbnail, initial_data = YoutubeAPI.browse(continuation)
banner: banner, end
description: description,
description_html: description_html, items, continuation = extract_items(initial_data)
total_views: total_views,
sub_count: sub_count, return items.select(SearchChannel), continuation
joined: joined, end
is_family_friendly: is_family_friendly,
pronouns: pronouns,
allowed_regions: allowed_regions,
tabs: tab_names,
tags: tags,
verified: author_verified || false,
is_age_gated: is_age_gated || false,
)
end
def fetch_related_channels(about_channel : AboutChannel, continuation : String? = nil) : {Array(SearchChannel), String?}
if continuation.nil?
# params is {"2:string":"channels"} encoded
initial_data = YoutubeAPI.browse(browse_id: about_channel.ucid, params: "EghjaGFubmVscw%3D%3D")
else
initial_data = YoutubeAPI.browse(continuation)
end
items, continuation = extract_items(initial_data)
return items.select(SearchChannel), continuation
end end

258
src/invidious/channels/channels.cr
View File

@ -123,7 +123,7 @@ def get_batch_channels(channels)
active_threads += 1 active_threads += 1
spawn do spawn do
begin begin
get_channel(ucid) Invidious::Channels::Channels.get_channel(ucid)
finished_channel.send(ucid) finished_channel.send(ucid)
rescue ex rescue ex
finished_channel.send(nil) finished_channel.send(nil)
@ -144,153 +144,157 @@ def get_batch_channels(channels)
return final return final
end end
def get_channel(id) : InvidiousChannel module Invidious::Channels::Channels
channel = Invidious::Database::Channels.select(id) extend self
if channel.nil? || (Time.utc - channel.updated) > 2.days def get_channel(id) : InvidiousChannel
channel = fetch_channel(id, pull_all_videos: false) channel = Invidious::Database::Channels.select(id)
Invidious::Database::Channels.insert(channel, update_on_conflict: true)
if channel.nil? || (Time.utc - channel.updated) > 2.days
channel = self.fetch_channel(id, pull_all_videos: false)
Invidious::Database::Channels.insert(channel, update_on_conflict: true)
end
return channel
end end
return channel def fetch_channel(ucid, pull_all_videos : Bool)
end LOGGER.debug("fetch_channel: #{ucid}")
LOGGER.trace("fetch_channel: #{ucid} : pull_all_videos = #{pull_all_videos}")
def fetch_channel(ucid, pull_all_videos : Bool) namespaces = {
LOGGER.debug("fetch_channel: #{ucid}") "yt" => "http://www.youtube.com/xml/schemas/2015",
LOGGER.trace("fetch_channel: #{ucid} : pull_all_videos = #{pull_all_videos}") "media" => "http://search.yahoo.com/mrss/",
"default" => "http://www.w3.org/2005/Atom",
}
namespaces = { LOGGER.trace("fetch_channel: #{ucid} : Downloading RSS feed")
"yt" => "http://www.youtube.com/xml/schemas/2015", rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}").body
"media" => "http://search.yahoo.com/mrss/", LOGGER.trace("fetch_channel: #{ucid} : Parsing RSS feed")
"default" => "http://www.w3.org/2005/Atom", rss = XML.parse(rss)
}
LOGGER.trace("fetch_channel: #{ucid} : Downloading RSS feed") author = rss.xpath_node("//default:feed/default:title", namespaces)
rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}").body if !author
LOGGER.trace("fetch_channel: #{ucid} : Parsing RSS feed") raise InfoException.new("Deleted or invalid channel")
rss = XML.parse(rss) end
author = rss.xpath_node("//default:feed/default:title", namespaces) author = author.content
if !author
raise InfoException.new("Deleted or invalid channel")
end
author = author.content # Auto-generated channels
# https://support.google.com/youtube/answer/2579942
if author.ends_with?(" - Topic") ||
{"Popular on YouTube", "Music", "Sports", "Gaming"}.includes? author
auto_generated = true
end
# Auto-generated channels LOGGER.trace("fetch_channel: #{ucid} : author = #{author}, auto_generated = #{auto_generated}")
# https://support.google.com/youtube/answer/2579942
if author.ends_with?(" - Topic") ||
{"Popular on YouTube", "Music", "Sports", "Gaming"}.includes? author
auto_generated = true
end
LOGGER.trace("fetch_channel: #{ucid} : author = #{author}, auto_generated = #{auto_generated}") channel = InvidiousChannel.new({
id: ucid,
channel = InvidiousChannel.new({ author: author,
id: ucid, updated: Time.utc,
author: author, deleted: false,
updated: Time.utc, subscribed: nil,
deleted: false,
subscribed: nil,
})
LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos page")
videos, continuation = IV::Channel::Tabs.get_videos(channel)
LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel RSS feed")
rss.xpath_nodes("//default:feed/default:entry", namespaces).each do |entry|
video_id = entry.xpath_node("yt:videoId", namespaces).not_nil!.content
title = entry.xpath_node("default:title", namespaces).not_nil!.content
published = Time.parse_rfc3339(
entry.xpath_node("default:published", namespaces).not_nil!.content
)
updated = Time.parse_rfc3339(
entry.xpath_node("default:updated", namespaces).not_nil!.content
)
author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content
ucid = entry.xpath_node("yt:channelId", namespaces).not_nil!.content
views = entry
.xpath_node("media:group/media:community/media:statistics", namespaces)
.try &.["views"]?.try &.to_i64? || 0_i64
channel_video = videos
.select(SearchVideo)
.select(&.id.== video_id)[0]?
length_seconds = channel_video.try &.length_seconds
length_seconds ||= 0
live_now = channel_video.try &.badges.live_now?
live_now ||= false
premiere_timestamp = channel_video.try &.premiere_timestamp
video = ChannelVideo.new({
id: video_id,
title: title,
published: published,
updated: updated,
ucid: ucid,
author: author,
length_seconds: length_seconds,
live_now: live_now,
premiere_timestamp: premiere_timestamp,
views: views,
}) })
LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updating or inserting video") LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos page")
videos, continuation = IV::Channel::Tabs.get_videos(channel)
# We don't include the 'premiere_timestamp' here because channel pages don't include them, LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel RSS feed")
# meaning the above timestamp is always null rss.xpath_nodes("//default:feed/default:entry", namespaces).each do |entry|
was_insert = Invidious::Database::ChannelVideos.insert(video) video_id = entry.xpath_node("yt:videoId", namespaces).not_nil!.content
title = entry.xpath_node("default:title", namespaces).not_nil!.content
if was_insert published = Time.parse_rfc3339(
LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Inserted, updating subscriptions") entry.xpath_node("default:published", namespaces).not_nil!.content
NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video)) )
else updated = Time.parse_rfc3339(
LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updated") entry.xpath_node("default:updated", namespaces).not_nil!.content
)
author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content
ucid = entry.xpath_node("yt:channelId", namespaces).not_nil!.content
views = entry
.xpath_node("media:group/media:community/media:statistics", namespaces)
.try &.["views"]?.try &.to_i64? || 0_i64
channel_video = videos
.select(SearchVideo)
.select(&.id.== video_id)[0]?
length_seconds = channel_video.try &.length_seconds
length_seconds ||= 0
live_now = channel_video.try &.badges.live_now?
live_now ||= false
premiere_timestamp = channel_video.try &.premiere_timestamp
video = ChannelVideo.new({
id: video_id,
title: title,
published: published,
updated: updated,
ucid: ucid,
author: author,
length_seconds: length_seconds,
live_now: live_now,
premiere_timestamp: premiere_timestamp,
views: views,
})
LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updating or inserting video")
# We don't include the 'premiere_timestamp' here because channel pages don't include them,
# meaning the above timestamp is always null
was_insert = Invidious::Database::ChannelVideos.insert(video)
if was_insert
LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Inserted, updating subscriptions")
NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video))
else
LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updated")
end
end end
end
if pull_all_videos if pull_all_videos
loop do loop do
# Keep fetching videos using the continuation token retrieved earlier # Keep fetching videos using the continuation token retrieved earlier
videos, continuation = IV::Channel::Tabs.get_videos(channel, continuation: continuation) videos, continuation = IV::Channel::Tabs.get_videos(channel, continuation: continuation)
count = 0 count = 0
videos.select(SearchVideo).each do |video| videos.select(SearchVideo).each do |video|
count += 1 count += 1
video = ChannelVideo.new({ video = ChannelVideo.new({
id: video.id, id: video.id,
title: video.title, title: video.title,
published: video.published, published: video.published,
updated: Time.utc, updated: Time.utc,
ucid: video.ucid, ucid: video.ucid,
author: video.author, author: video.author,
length_seconds: video.length_seconds, length_seconds: video.length_seconds,
live_now: video.badges.live_now?, live_now: video.badges.live_now?,
premiere_timestamp: video.premiere_timestamp, premiere_timestamp: video.premiere_timestamp,
views: video.views, views: video.views,
}) })
# We are notified of Red videos elsewhere (PubSub), which includes a correct published date, # We are notified of Red videos elsewhere (PubSub), which includes a correct published date,
# so since they don't provide a published date here we can safely ignore them. # so since they don't provide a published date here we can safely ignore them.
if Time.utc - video.published > 1.minute if Time.utc - video.published > 1.minute
was_insert = Invidious::Database::ChannelVideos.insert(video) was_insert = Invidious::Database::ChannelVideos.insert(video)
if was_insert if was_insert
NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video)) NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video))
end
end end
end end
break if count < 25
sleep 500.milliseconds
end end
break if count < 25
sleep 500.milliseconds
end end
end
channel.updated = Time.utc channel.updated = Time.utc
return channel return channel
end
end end

519
src/invidious/channels/community.cr
View File

@ -1,198 +1,201 @@
private IMAGE_QUALITIES = {320, 560, 640, 1280, 2000} module Invidious::Channels::Community
extend self
private IMAGE_QUALITIES = {320, 560, 640, 1280, 2000}
# TODO: Add "sort_by" # TODO: Add "sort_by"
def fetch_channel_community(ucid, cursor, locale, format, thin_mode) def fetch_channel_community(ucid, cursor, locale, format, thin_mode)
if cursor.nil? if cursor.nil?
# EgVwb3N0c_IGBAoCSgA%3D is the protobuf object to load "posts" # EgVwb3N0c_IGBAoCSgA%3D is the protobuf object to load "posts"
initial_data = YoutubeAPI.browse(ucid, params: "EgVwb3N0c_IGBAoCSgA%3D") initial_data = YoutubeAPI.browse(ucid, params: "EgVwb3N0c_IGBAoCSgA%3D")
items = [] of JSON::Any
extract_items(initial_data) do |item|
items << item
end
else
continuation = self.produce_channel_community_continuation(ucid, cursor)
initial_data = YoutubeAPI.browse(continuation: continuation)
container = initial_data.dig?("continuationContents", "itemSectionContinuation", "contents")
raise InfoException.new("Can't extract community data") if container.nil?
items = container.as_a
end
return extract_channel_community(items, ucid: ucid, locale: locale, format: format, thin_mode: thin_mode)
end
def decode_ucid_from_post_protobuf(params)
decoded_protobuf = params.try { |i| URI.decode_www_form(i) }
.try { |i| Base64.decode(i) }
.try { |i| IO::Memory.new(i) }
.try { |i| Protodec::Any.parse(i) }
return decoded_protobuf.try(&.["56:0:embedded"]["2:0:string"].as_s)
end
def fetch_channel_community_post(ucid, post_id, locale, format, thin_mode)
object = {
"56:embedded" => {
"2:string" => ucid,
"3:string" => post_id.to_s,
"11:string" => ucid,
},
}
params = object.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
initial_data = YoutubeAPI.browse("FEpost_detail", params: params)
items = [] of JSON::Any items = [] of JSON::Any
extract_items(initial_data) do |item| extract_items(initial_data) do |item|
items << item items << item
end end
else
continuation = produce_channel_community_continuation(ucid, cursor)
initial_data = YoutubeAPI.browse(continuation: continuation)
container = initial_data.dig?("continuationContents", "itemSectionContinuation", "contents") return extract_channel_community(items, ucid: ucid, locale: locale, format: format, thin_mode: thin_mode, is_single_post: true)
raise InfoException.new("Can't extract community data") if container.nil?
items = container.as_a
end end
return extract_channel_community(items, ucid: ucid, locale: locale, format: format, thin_mode: thin_mode) def extract_channel_community(items, *, ucid, locale, format, thin_mode, is_single_post : Bool = false)
end if message = items[0]["messageRenderer"]?
error_message = (message["text"]["simpleText"]? ||
def decode_ucid_from_post_protobuf(params) message["text"]["runs"]?.try &.[0]?.try &.["text"]?)
decoded_protobuf = params.try { |i| URI.decode_www_form(i) } .try &.as_s || ""
.try { |i| Base64.decode(i) } if error_message == "This channel does not exist."
.try { |i| IO::Memory.new(i) } raise NotFoundException.new(error_message)
.try { |i| Protodec::Any.parse(i) } else
raise InfoException.new(error_message)
return decoded_protobuf.try(&.["56:0:embedded"]["2:0:string"].as_s)
end
def fetch_channel_community_post(ucid, post_id, locale, format, thin_mode)
object = {
"56:embedded" => {
"2:string" => ucid,
"3:string" => post_id.to_s,
"11:string" => ucid,
},
}
params = object.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
initial_data = YoutubeAPI.browse("FEpost_detail", params: params)
items = [] of JSON::Any
extract_items(initial_data) do |item|
items << item
end
return extract_channel_community(items, ucid: ucid, locale: locale, format: format, thin_mode: thin_mode, is_single_post: true)
end
def extract_channel_community(items, *, ucid, locale, format, thin_mode, is_single_post : Bool = false)
if message = items[0]["messageRenderer"]?
error_message = (message["text"]["simpleText"]? ||
message["text"]["runs"]?.try &.[0]?.try &.["text"]?)
.try &.as_s || ""
if error_message == "This channel does not exist."
raise NotFoundException.new(error_message)
else
raise InfoException.new(error_message)
end
end
response = JSON.build do |json|
json.object do
json.field "authorId", ucid
if is_single_post
json.field "singlePost", true
end end
json.field "comments" do end
json.array do
items.each do |post|
comments = post["backstagePostThreadRenderer"]?.try &.["comments"]? ||
post["backstageCommentsContinuation"]?
post = post["backstagePostThreadRenderer"]?.try &.["post"]["backstagePostRenderer"]? || response = JSON.build do |json|
post["commentThreadRenderer"]?.try &.["comment"]["commentRenderer"]? json.object do
json.field "authorId", ucid
if is_single_post
json.field "singlePost", true
end
json.field "comments" do
json.array do
items.each do |post|
comments = post["backstagePostThreadRenderer"]?.try &.["comments"]? ||
post["backstageCommentsContinuation"]?
next if !post post = post["backstagePostThreadRenderer"]?.try &.["post"]["backstagePostRenderer"]? ||
post["commentThreadRenderer"]?.try &.["comment"]["commentRenderer"]?
content_html = post["contentText"]?.try { |t| parse_content(t) } || "" next if !post
author = post["authorText"]["runs"]?.try &.[0]?.try &.["text"]? || ""
json.object do content_html = post["contentText"]?.try { |t| parse_content(t) } || ""
json.field "author", author author = post["authorText"]["runs"]?.try &.[0]?.try &.["text"]? || ""
json.field "authorThumbnails" do
json.array do
qualities = {32, 48, 76, 100, 176, 512}
author_thumbnail = post["authorThumbnail"]["thumbnails"].as_a[0]["url"].as_s
qualities.each do |quality| json.object do
json.object do json.field "author", author
json.field "url", author_thumbnail.gsub(/s\d+-/, "s#{quality}-") json.field "authorThumbnails" do
json.field "width", quality json.array do
json.field "height", quality qualities = {32, 48, 76, 100, 176, 512}
author_thumbnail = post["authorThumbnail"]["thumbnails"].as_a[0]["url"].as_s
qualities.each do |quality|
json.object do
json.field "url", author_thumbnail.gsub(/s\d+-/, "s#{quality}-")
json.field "width", quality
json.field "height", quality
end
end end
end end
end end
end
if post["authorEndpoint"]? if post["authorEndpoint"]?
json.field "authorId", post["authorEndpoint"]["browseEndpoint"]["browseId"] json.field "authorId", post["authorEndpoint"]["browseEndpoint"]["browseId"]
json.field "authorUrl", post["authorEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"].as_s json.field "authorUrl", post["authorEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"].as_s
else else
json.field "authorId", "" json.field "authorId", ""
json.field "authorUrl", "" json.field "authorUrl", ""
end end
published_text = post["publishedTimeText"]["runs"][0]["text"].as_s published_text = post["publishedTimeText"]["runs"][0]["text"].as_s
published = decode_date(published_text.rchop(" (edited)")) published = decode_date(published_text.rchop(" (edited)"))
if published_text.includes?(" (edited)") if published_text.includes?(" (edited)")
json.field "isEdited", true json.field "isEdited", true
else else
json.field "isEdited", false json.field "isEdited", false
end end
like_count = post["actionButtons"]["commentActionButtonsRenderer"]["likeButton"]["toggleButtonRenderer"]["accessibilityData"]["accessibilityData"]["label"] like_count = post["actionButtons"]["commentActionButtonsRenderer"]["likeButton"]["toggleButtonRenderer"]["accessibilityData"]["accessibilityData"]["label"]
.try &.as_s.gsub(/\D/, "").to_i? || 0 .try &.as_s.gsub(/\D/, "").to_i? || 0
reply_count = short_text_to_number(post.dig?("actionButtons", "commentActionButtonsRenderer", "replyButton", "buttonRenderer", "text", "simpleText").try &.as_s || "0") reply_count = short_text_to_number(post.dig?("actionButtons", "commentActionButtonsRenderer", "replyButton", "buttonRenderer", "text", "simpleText").try &.as_s || "0")
json.field "content", Helpers.html_to_content(content_html) json.field "content", Invidious::Helpers.html_to_content(content_html)
json.field "contentHtml", content_html json.field "contentHtml", content_html
json.field "published", published.to_unix json.field "published", published.to_unix
json.field "publishedText", I18n.translate(locale, "`x` ago", recode_date(published, locale)) json.field "publishedText", I18n.translate(locale, "`x` ago", recode_date(published, locale))
json.field "likeCount", like_count json.field "likeCount", like_count
json.field "replyCount", reply_count json.field "replyCount", reply_count
json.field "commentId", post["postId"]? || post["commentId"]? || "" json.field "commentId", post["postId"]? || post["commentId"]? || ""
json.field "authorIsChannelOwner", post["authorEndpoint"]["browseEndpoint"]["browseId"] == ucid json.field "authorIsChannelOwner", post["authorEndpoint"]["browseEndpoint"]["browseId"] == ucid
if attachment = post["backstageAttachment"]? if attachment = post["backstageAttachment"]?
json.field "attachment" do json.field "attachment" do
case attachment.as_h case attachment.as_h
when .has_key?("videoRenderer") when .has_key?("videoRenderer")
parse_item(attachment) parse_item(attachment)
.as(SearchVideo | ProblematicTimelineItem) .as(SearchVideo | ProblematicTimelineItem)
.to_json(locale, json) .to_json(locale, json)
when .has_key?("backstageImageRenderer") when .has_key?("backstageImageRenderer")
json.object do json.object do
attachment = attachment["backstageImageRenderer"] attachment = attachment["backstageImageRenderer"]
json.field "type", "image" json.field "type", "image"
json.field "imageThumbnails" do json.field "imageThumbnails" do
json.array do json.array do
thumbnail = attachment["image"]["thumbnails"][0].as_h thumbnail = attachment["image"]["thumbnails"][0].as_h
width = thumbnail["width"].as_i width = thumbnail["width"].as_i
height = thumbnail["height"].as_i height = thumbnail["height"].as_i
aspect_ratio = (width.to_f / height.to_f) aspect_ratio = (width.to_f / height.to_f)
url = thumbnail["url"].as_s.gsub(/=w\d+-h\d+(-p)?(-nd)?(-df)?(-rwa)?/, "=s640") url = thumbnail["url"].as_s.gsub(/=w\d+-h\d+(-p)?(-nd)?(-df)?(-rwa)?/, "=s640")
IMAGE_QUALITIES.each do |quality| IMAGE_QUALITIES.each do |quality|
json.object do json.object do
json.field "url", url.gsub(/=s\d+/, "=s#{quality}") json.field "url", url.gsub(/=s\d+/, "=s#{quality}")
json.field "width", quality json.field "width", quality
json.field "height", (quality / aspect_ratio).ceil.to_i json.field "height", (quality / aspect_ratio).ceil.to_i
end
end end
end end
end end
end end
end when .has_key?("pollRenderer")
when .has_key?("pollRenderer") json.object do
json.object do attachment = attachment["pollRenderer"]
attachment = attachment["pollRenderer"] json.field "type", "poll"
json.field "type", "poll" json.field "totalVotes", short_text_to_number(attachment["totalVotes"]["simpleText"].as_s.split(" ")[0])
json.field "totalVotes", short_text_to_number(attachment["totalVotes"]["simpleText"].as_s.split(" ")[0]) json.field "choices" do
json.field "choices" do json.array do
json.array do attachment["choices"].as_a.each do |choice|
attachment["choices"].as_a.each do |choice| json.object do
json.object do json.field "text", choice.dig("text", "runs", 0, "text").as_s
json.field "text", choice.dig("text", "runs", 0, "text").as_s # A choice can have an image associated with it.
# A choice can have an image associated with it. # Ex post: https://www.youtube.com/post/UgkxD4XavXUD4NQiddJXXdohbwOwcVqrH9Re
# Ex post: https://www.youtube.com/post/UgkxD4XavXUD4NQiddJXXdohbwOwcVqrH9Re if choice["image"]?
if choice["image"]? thumbnail = choice["image"]["thumbnails"][0].as_h
thumbnail = choice["image"]["thumbnails"][0].as_h width = thumbnail["width"].as_i
width = thumbnail["width"].as_i height = thumbnail["height"].as_i
height = thumbnail["height"].as_i aspect_ratio = (width.to_f / height.to_f)
aspect_ratio = (width.to_f / height.to_f) url = thumbnail["url"].as_s.gsub(/=w\d+-h\d+(-p)?(-nd)?(-df)?(-rwa)?/, "=s640")
url = thumbnail["url"].as_s.gsub(/=w\d+-h\d+(-p)?(-nd)?(-df)?(-rwa)?/, "=s640") json.field "image" do
json.field "image" do json.array do
json.array do IMAGE_QUALITIES.each do |quality|
IMAGE_QUALITIES.each do |quality| json.object do
json.object do json.field "url", url.gsub(/=s\d+/, "=s#{quality}")
json.field "url", url.gsub(/=s\d+/, "=s#{quality}") json.field "width", quality
json.field "width", quality json.field "height", (quality / aspect_ratio).ceil.to_i
json.field "height", (quality / aspect_ratio).ceil.to_i end
end end
end end
end end
@ -202,137 +205,137 @@ def extract_channel_community(items, *, ucid, locale, format, thin_mode, is_sing
end end
end end
end end
end when .has_key?("postMultiImageRenderer")
when .has_key?("postMultiImageRenderer") json.object do
json.object do attachment = attachment["postMultiImageRenderer"]
attachment = attachment["postMultiImageRenderer"] json.field "type", "multiImage"
json.field "type", "multiImage" json.field "images" do
json.field "images" do json.array do
json.array do attachment["images"].as_a.each do |image|
attachment["images"].as_a.each do |image| json.array do
json.array do thumbnail = image["backstageImageRenderer"]["image"]["thumbnails"][0].as_h
thumbnail = image["backstageImageRenderer"]["image"]["thumbnails"][0].as_h width = thumbnail["width"].as_i
width = thumbnail["width"].as_i height = thumbnail["height"].as_i
height = thumbnail["height"].as_i aspect_ratio = (width.to_f / height.to_f)
aspect_ratio = (width.to_f / height.to_f) url = thumbnail["url"].as_s.gsub(/=w\d+-h\d+(-p)?(-nd)?(-df)?(-rwa)?/, "=s640")
url = thumbnail["url"].as_s.gsub(/=w\d+-h\d+(-p)?(-nd)?(-df)?(-rwa)?/, "=s640")
IMAGE_QUALITIES.each do |quality| IMAGE_QUALITIES.each do |quality|
json.object do json.object do
json.field "url", url.gsub(/=s\d+/, "=s#{quality}") json.field "url", url.gsub(/=s\d+/, "=s#{quality}")
json.field "width", quality json.field "width", quality
json.field "height", (quality / aspect_ratio).ceil.to_i json.field "height", (quality / aspect_ratio).ceil.to_i
end
end end
end end
end end
end end
end end
end end
end when .has_key?("playlistRenderer")
when .has_key?("playlistRenderer") parse_item(attachment)
parse_item(attachment) .as(SearchPlaylist)
.as(SearchPlaylist) .to_json(locale, json)
.to_json(locale, json) when .has_key?("quizRenderer")
when .has_key?("quizRenderer") json.object do
json.object do attachment = attachment["quizRenderer"]
attachment = attachment["quizRenderer"] json.field "type", "quiz"
json.field "type", "quiz" json.field "totalVotes", short_text_to_number(attachment["totalVotes"]["simpleText"].as_s.split(" ")[0])
json.field "totalVotes", short_text_to_number(attachment["totalVotes"]["simpleText"].as_s.split(" ")[0]) json.field "choices" do
json.field "choices" do json.array do
json.array do attachment["choices"].as_a.each do |choice|
attachment["choices"].as_a.each do |choice| json.object do
json.object do json.field "text", choice.dig("text", "runs", 0, "text").as_s
json.field "text", choice.dig("text", "runs", 0, "text").as_s json.field "isCorrect", choice["isCorrect"].as_bool
json.field "isCorrect", choice["isCorrect"].as_bool end
end end
end end
end end
end end
end else
else json.object do
json.object do json.field "type", "unknown"
json.field "type", "unknown" json.field "error", "Unrecognized attachment type."
json.field "error", "Unrecognized attachment type." end
end end
end end
end end
end
if comments && (reply_count = (comments["backstageCommentsRenderer"]["moreText"]["simpleText"]? || if comments && (reply_count = (comments["backstageCommentsRenderer"]["moreText"]["simpleText"]? ||
comments["backstageCommentsRenderer"]["moreText"]["runs"]?.try &.[0]?.try &.["text"]?) comments["backstageCommentsRenderer"]["moreText"]["runs"]?.try &.[0]?.try &.["text"]?)
.try &.as_s.gsub(/\D/, "").to_i?) .try &.as_s.gsub(/\D/, "").to_i?)
continuation = comments["backstageCommentsRenderer"]["continuations"]?.try &.as_a[0]["nextContinuationData"]["continuation"].as_s continuation = comments["backstageCommentsRenderer"]["continuations"]?.try &.as_a[0]["nextContinuationData"]["continuation"].as_s
continuation ||= "" continuation ||= ""
json.field "replies" do json.field "replies" do
json.object do json.object do
json.field "replyCount", reply_count json.field "replyCount", reply_count
json.field "continuation", extract_channel_community_cursor(continuation) json.field "continuation", extract_channel_community_cursor(continuation)
end
end end
end end
end end
end end
end end
end end
end if !is_single_post
if !is_single_post if cont = items.dig?(-1, "continuationItemRenderer", "continuationEndpoint", "continuationCommand", "token")
if cont = items.dig?(-1, "continuationItemRenderer", "continuationEndpoint", "continuationCommand", "token") json.field "continuation", extract_channel_community_cursor(cont.as_s)
json.field "continuation", extract_channel_community_cursor(cont.as_s) end
end end
end end
end end
end
if format == "html" if format == "html"
response = JSON.parse(response) response = JSON.parse(response)
content_html = IV::Frontend::Comments.template_youtube(response, locale, thin_mode) content_html = IV::Frontend::Comments.template_youtube(response, locale, thin_mode)
response = JSON.build do |json| response = JSON.build do |json|
json.object do json.object do
json.field "contentHtml", content_html json.field "contentHtml", content_html
end
end end
end end
return response
end end
return response def produce_channel_community_continuation(ucid, cursor)
end object = {
"80226972:embedded" => {
"2:string" => ucid,
"3:string" => cursor || "",
},
}
def produce_channel_community_continuation(ucid, cursor) continuation = object.try { |i| Protodec::Any.cast_json(i) }
object = {
"80226972:embedded" => {
"2:string" => ucid,
"3:string" => cursor || "",
},
}
continuation = object.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
return continuation
end
def extract_channel_community_cursor(continuation)
object = URI.decode_www_form(continuation)
.try { |i| Base64.decode(i) }
.try { |i| IO::Memory.new(i) }
.try { |i| Protodec::Any.parse(i) }
.try(&.["80226972:0:embedded"]["3:1:base64"].as_h)
if object["53:2:embedded"]?.try &.["3:0:embedded"]?
object["53:2:embedded"]["3:0:embedded"]["2:0:string"] = object["53:2:embedded"]["3:0:embedded"]
.try(&.["2:0:base64"].as_h)
.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) } .try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i, padding: false) } .try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
object["53:2:embedded"]["3:0:embedded"].as_h.delete("2:0:base64") return continuation
end end
cursor = Protodec::Any.cast_json(object) def extract_channel_community_cursor(continuation)
.try { |i| Protodec::Any.from_json(i) } object = URI.decode_www_form(continuation)
.try { |i| Base64.urlsafe_encode(i) } .try { |i| Base64.decode(i) }
.try { |i| IO::Memory.new(i) }
.try { |i| Protodec::Any.parse(i) }
.try(&.["80226972:0:embedded"]["3:1:base64"].as_h)
cursor if object["53:2:embedded"]?.try &.["3:0:embedded"]?
object["53:2:embedded"]["3:0:embedded"]["2:0:string"] = object["53:2:embedded"]["3:0:embedded"]
.try(&.["2:0:base64"].as_h)
.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i, padding: false) }
object["53:2:embedded"]["3:0:embedded"].as_h.delete("2:0:base64")
end
cursor = Protodec::Any.cast_json(object)
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
cursor
end
end end

96
src/invidious/channels/playlists.cr
View File

@ -1,55 +1,59 @@
def fetch_channel_playlists(ucid, author, continuation, sort_by) module Invidious::Channels::Playlists
if continuation extend self
initial_data = YoutubeAPI.browse(continuation)
else
params =
case sort_by
when "last", "last_added"
# Equivalent to "&sort=lad"
# {"2:string": "playlists", "3:varint": 4, "4:varint": 1, "6:varint": 1, "110:embedded": {"1:embedded": {"8:string": ""}}}
"EglwbGF5bGlzdHMYBCABMAHyBgQKAkIA"
when "oldest", "oldest_created"
# formerly "&sort=da"
# Not available anymore :c or maybe ??
# {"2:string": "playlists", "3:varint": 2, "4:varint": 1, "6:varint": 1, "110:embedded": {"1:embedded": {"8:string": ""}}}
"EglwbGF5bGlzdHMYAiABMAHyBgQKAkIA"
# {"2:string": "playlists", "3:varint": 1, "4:varint": 1, "6:varint": 1}
# "EglwbGF5bGlzdHMYASABMAE%3D"
when "newest", "newest_created"
# Formerly "&sort=dd"
# {"2:string": "playlists", "3:varint": 3, "4:varint": 1, "6:varint": 1, "110:embedded": {"1:embedded": {"8:string": ""}}}
"EglwbGF5bGlzdHMYAyABMAHyBgQKAkIA"
end
initial_data = YoutubeAPI.browse(ucid, params: params || "") def fetch_channel_playlists(ucid, author, continuation, sort_by)
if continuation
initial_data = YoutubeAPI.browse(continuation)
else
params =
case sort_by
when "last", "last_added"
# Equivalent to "&sort=lad"
# {"2:string": "playlists", "3:varint": 4, "4:varint": 1, "6:varint": 1, "110:embedded": {"1:embedded": {"8:string": ""}}}
"EglwbGF5bGlzdHMYBCABMAHyBgQKAkIA"
when "oldest", "oldest_created"
# formerly "&sort=da"
# Not available anymore :c or maybe ??
# {"2:string": "playlists", "3:varint": 2, "4:varint": 1, "6:varint": 1, "110:embedded": {"1:embedded": {"8:string": ""}}}
"EglwbGF5bGlzdHMYAiABMAHyBgQKAkIA"
# {"2:string": "playlists", "3:varint": 1, "4:varint": 1, "6:varint": 1}
# "EglwbGF5bGlzdHMYASABMAE%3D"
when "newest", "newest_created"
# Formerly "&sort=dd"
# {"2:string": "playlists", "3:varint": 3, "4:varint": 1, "6:varint": 1, "110:embedded": {"1:embedded": {"8:string": ""}}}
"EglwbGF5bGlzdHMYAyABMAHyBgQKAkIA"
end
initial_data = YoutubeAPI.browse(ucid, params: params || "")
end
return extract_items(initial_data, author, ucid)
end end
return extract_items(initial_data, author, ucid) def fetch_channel_podcasts(ucid, author, continuation)
end if continuation
initial_data = YoutubeAPI.browse(continuation)
def fetch_channel_podcasts(ucid, author, continuation) else
if continuation initial_data = YoutubeAPI.browse(ucid, params: "Eghwb2RjYXN0c_IGBQoDugEA")
initial_data = YoutubeAPI.browse(continuation) end
else return extract_items(initial_data, author, ucid)
initial_data = YoutubeAPI.browse(ucid, params: "Eghwb2RjYXN0c_IGBQoDugEA")
end end
return extract_items(initial_data, author, ucid)
end
def fetch_channel_releases(ucid, author, continuation) def fetch_channel_releases(ucid, author, continuation)
if continuation if continuation
initial_data = YoutubeAPI.browse(continuation) initial_data = YoutubeAPI.browse(continuation)
else else
initial_data = YoutubeAPI.browse(ucid, params: "EghyZWxlYXNlc_IGBQoDsgEA") initial_data = YoutubeAPI.browse(ucid, params: "EghyZWxlYXNlc_IGBQoDsgEA")
end
return extract_items(initial_data, author, ucid)
end end
return extract_items(initial_data, author, ucid)
end
def fetch_channel_courses(ucid, author, continuation) def fetch_channel_courses(ucid, author, continuation)
if continuation if continuation
initial_data = YoutubeAPI.browse(continuation) initial_data = YoutubeAPI.browse(continuation)
else else
initial_data = YoutubeAPI.browse(ucid, params: "Egdjb3Vyc2Vz8gYFCgPCAQA%3D") initial_data = YoutubeAPI.browse(ucid, params: "Egdjb3Vyc2Vz8gYFCgPCAQA%3D")
end
return extract_items(initial_data, author, ucid)
end end
return extract_items(initial_data, author, ucid)
end end

0
src/invidious/channels/videos.cr → src/invidious/channels/tabs.cr
View File

2
src/invidious/comments/youtube.cr
View File

@ -246,7 +246,7 @@ module Invidious::Comments
end end
content_html = html_content || "" content_html = html_content || ""
json.field "content", Helpers.html_to_content(content_html) json.field "content", Invidious::Helpers.html_to_content(content_html)
json.field "contentHtml", content_html json.field "contentHtml", content_html
if published_text != nil if published_text != nil

216
src/invidious/ext/kemal_static_file_handler.cr Normal file
View File

@ -0,0 +1,216 @@
{% if compare_versions(Crystal::VERSION, "1.17.0-dev") >= 0 %}
# Strip StaticFileHandler from the binary
#
# This allows us to compile on 1.17.0 as the compiler won't try to
# semantically check the outdated upstream code.
class Kemal::Config
private def setup_static_file_handler
end
end
# Nullify `Kemal::StaticFileHandler`
#
# Needed until the next release of Kemal after 1.7
class Kemal::StaticFileHandler < HTTP::StaticFileHandler
def call(context : HTTP::Server::Context)
end
end
{% skip_file %}
{% end %}
# Since systems have a limit on number of open files (`ulimit -a`),
# we serve them from memory to avoid 'Too many open files' without needing
# to modify ulimit.
#
# Very heavily re-used:
# https://github.com/kemalcr/kemal/blob/master/src/kemal/helpers/helpers.cr
# https://github.com/kemalcr/kemal/blob/master/src/kemal/static_file_handler.cr
#
# Changes:
# - A `send_file` overload is added which supports sending a Slice, file_path, filestat
# - `StaticFileHandler` is patched to cache to and serve from @cached_files
private def multipart(file, env : HTTP::Server::Context)
# See http://httpwg.org/specs/rfc7233.html
fileb = file.size
startb = endb = 0
if match = env.request.headers["Range"].match /bytes=(\d{1,})-(\d{0,})/
startb = match[1].to_i { 0 } if match.size >= 2
endb = match[2].to_i { 0 } if match.size >= 3
end
endb = fileb - 1 if endb == 0
if startb < endb < fileb
content_length = 1 + endb - startb
env.response.status_code = 206
env.response.content_length = content_length
env.response.headers["Accept-Ranges"] = "bytes"
env.response.headers["Content-Range"] = "bytes #{startb}-#{endb}/#{fileb}" # MUST
if startb > 1024
skipped = 0
# file.skip only accepts values less or equal to 1024 (buffer size, undocumented)
until (increase_skipped = skipped + 1024) > startb
file.skip(1024)
skipped = increase_skipped
end
if (skipped_minus_startb = skipped - startb) > 0
file.skip skipped_minus_startb
end
else
file.skip(startb)
end
IO.copy(file, env.response, content_length)
else
env.response.content_length = fileb
env.response.status_code = 200 # Range not satisfable, see 4.4 Note
IO.copy(file, env.response)
end
end
# Set the Content-Disposition to "attachment" with the specified filename,
# instructing the user agents to prompt to save.
private def attachment(env : HTTP::Server::Context, filename : String? = nil, disposition : String? = nil)
disposition = "attachment" if disposition.nil? && filename
if disposition && filename
env.response.headers["Content-Disposition"] = "#{disposition}; filename=\"#{File.basename(filename)}\""
end
end
def send_file(env : HTTP::Server::Context, file_path : String, data : Slice(UInt8), filestat : File::Info, filename : String? = nil, disposition : String? = nil)
config = Kemal.config.serve_static
mime_type = MIME.from_filename(file_path, "application/octet-stream")
env.response.content_type = mime_type
env.response.headers["Accept-Ranges"] = "bytes"
env.response.headers["X-Content-Type-Options"] = "nosniff"
minsize = 860 # http://webmasters.stackexchange.com/questions/31750/what-is-recommended-minimum-object-size-for-gzip-performance-benefits ??
request_headers = env.request.headers
filesize = data.bytesize
attachment(env, filename, disposition)
Kemal.config.static_headers.try(&.call(env, file_path, filestat))
file = IO::Memory.new(data)
if env.request.method == "GET" && env.request.headers.has_key?("Range")
return multipart(file, env)
end
condition = config.is_a?(Hash) && config["gzip"]? == true && filesize > minsize && Kemal::Utils.zip_types(file_path)
if condition && request_headers.includes_word?("Accept-Encoding", "gzip")
env.response.headers["Content-Encoding"] = "gzip"
Compress::Gzip::Writer.open(env.response) do |deflate|
IO.copy(file, deflate)
end
elsif condition && request_headers.includes_word?("Accept-Encoding", "deflate")
env.response.headers["Content-Encoding"] = "deflate"
Compress::Deflate::Writer.open(env.response) do |deflate|
IO.copy(file, deflate)
end
else
env.response.content_length = filesize
IO.copy(file, env.response)
end
return
end
module Kemal
class StaticFileHandler < HTTP::StaticFileHandler
CACHE_LIMIT = 5_000_000 # 5MB
@cached_files = {} of String => {data: Bytes, filestat: File::Info}
def call(context : HTTP::Server::Context)
return call_next(context) if context.request.path.not_nil! == "/"
case context.request.method
when "GET", "HEAD"
else
if @fallthrough
call_next(context)
else
context.response.status = HTTP::Status::METHOD_NOT_ALLOWED
context.response.headers.add("Allow", "GET, HEAD")
end
return
end
config = Kemal.config.serve_static
original_path = context.request.path.not_nil!
request_path = URI.decode_www_form(original_path)
# File path cannot contains '\0' (NUL) because all filesystem I know
# don't accept '\0' character as file name.
if request_path.includes? '\0'
context.response.status = HTTP::Status::BAD_REQUEST
return
end
expanded_path = File.expand_path(request_path, "/")
is_dir_path = if original_path.ends_with?('/') && !expanded_path.ends_with? '/'
expanded_path = expanded_path + '/'
true
else
expanded_path.ends_with? '/'
end
file_path = File.join(@public_dir, expanded_path)
if file = @cached_files[file_path]?
last_modified = file[:filestat].modification_time
add_cache_headers(context.response.headers, last_modified)
if cache_request?(context, last_modified)
context.response.status = HTTP::Status::NOT_MODIFIED
return
end
send_file(context, file_path, file[:data], file[:filestat])
else
file_info = File.info?(file_path)
is_dir = file_info.try &.directory? || false
is_file = file_info.try &.file? || false
if request_path != expanded_path
redirect_to context, expanded_path
elsif is_dir && !is_dir_path
redirect_to context, expanded_path + '/'
end
return call_next(context) if file_info.nil?
if is_dir
if config.is_a?(Hash) && config["dir_listing"] == true
context.response.content_type = "text/html"
directory_listing(context.response, request_path, file_path)
else
call_next(context)
end
elsif is_file
last_modified = file_info.modification_time
add_cache_headers(context.response.headers, last_modified)
if cache_request?(context, last_modified)
context.response.status = HTTP::Status::NOT_MODIFIED
return
end
if @cached_files.sum(&.[1][:data].bytesize) + (size = File.size(file_path)) < CACHE_LIMIT
data = Bytes.new(size)
File.open(file_path, &.read(data))
@cached_files[file_path] = {data: data, filestat: file_info}
send_file(context, file_path, data, file_info)
else
send_file(context, file_path)
end
else # Not a normal file (FIFO/device/socket)
call_next(context)
end
end
end
end
end

52
src/invidious/feeds/trending.cr Normal file
View File

@ -0,0 +1,52 @@
module Invidious::Feeds::Trending
extend self
def fetch(trending_type, region, locale)
region ||= "US"
region = region.upcase
plid = nil
browse_id = ""
case trending_type.try &.downcase
when "gaming"
browse_id = "UCOpNcN46UbXVtpKMrmU4Abg"
params = "Egh0cmVuZGluZw%3D%3D"
when "livestreams"
browse_id = "UC4R8DWoMoI7CAwX8_LjQHig"
params = "EgdsaXZldGFikgEDCKEK"
else
# Livestreams is the default one as Youtube removed
# the aggregated trending page
# https://github.com/iv-org/invidious/issues/5397#issuecomment-3218928458
browse_id = "UC4R8DWoMoI7CAwX8_LjQHig"
params = "EgdsaXZldGFikgEDCKEK"
end
client_config = YoutubeAPI::ClientConfig.new(region: region)
initial_data = YoutubeAPI.browse(browse_id, params: params, client_config: client_config)
items, _ = extract_items(initial_data)
extracted = [] of SearchItem
deduplicate = items.size > 1
items.each do |itm|
if itm.is_a?(Category)
# Ignore the smaller categories, as they generally contain a sponsored
# channel, which brings a lot of noise on the trending page.
# See: https://github.com/iv-org/invidious/issues/2989
next if (itm.contents.size < 24 && deduplicate)
extracted.concat itm.contents.select(SearchItem)
else
extracted << itm
end
end
# Deduplicate items before returning results
return extracted.select(SearchVideo | ProblematicTimelineItem).uniq!(&.id), plid
end
end

270
src/invidious/helpers/errors.cr
View File

@ -2,26 +2,29 @@
# Issue template # Issue template
# ------------------- # -------------------
macro error_template(*args) module Errors
error_template_helper(env, {{args.splat}}) extend self
end
def github_details(summary : String, content : String) macro error_template(*args)
details = %(\n<details>) Errors.error_template_helper(env, {{args.splat}})
details += %(\n<summary>#{summary}</summary>) end
details += %(\n<p>)
details += %(\n \n```\n)
details += content.strip
details += %(\n```)
details += %(\n</p>)
details += %(\n</details>)
return HTML.escape(details)
end
def get_issue_template(env : HTTP::Server::Context, exception : Exception) : Tuple(String, String) def github_details(summary : String, content : String)
issue_title = "#{exception.message} (#{exception.class})" details = %(\n<details>)
details += %(\n<summary>#{summary}</summary>)
details += %(\n<p>)
details += %(\n \n```\n)
details += content.strip
details += %(\n```)
details += %(\n</p>)
details += %(\n</details>)
return HTML.escape(details)
end
issue_template = <<-TEXT def get_issue_template(env : HTTP::Server::Context, exception : Exception) : Tuple(String, String)
issue_title = "#{exception.message} (#{exception.class})"
issue_template = <<-TEXT
Title: `#{HTML.escape(issue_title)}` Title: `#{HTML.escape(issue_title)}`
Date: `#{Time::Format::ISO_8601_DATE_TIME.format(Time.utc)}` Date: `#{Time::Format::ISO_8601_DATE_TIME.format(Time.utc)}`
Route: `#{HTML.escape(env.request.resource)}` Route: `#{HTML.escape(env.request.resource)}`
@ -29,39 +32,39 @@ def get_issue_template(env : HTTP::Server::Context, exception : Exception) : Tup
TEXT TEXT
issue_template += github_details("Backtrace", exception.inspect_with_backtrace) issue_template += github_details("Backtrace", exception.inspect_with_backtrace)
return issue_title, issue_template return issue_title, issue_template
end
def error_template_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception)
if exception.is_a?(InfoException)
return error_template_helper(env, status_code, exception.message || "")
end end
locale = env.get("preferences").as(Preferences).locale def error_template_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception)
if exception.is_a?(InfoException)
return error_template_helper(env, status_code, exception.message || "")
end
env.response.content_type = "text/html" locale = env.get("preferences").as(Preferences).locale
env.response.status_code = status_code
# Unpacking into issue_title, issue_template directly causes a compiler error env.response.content_type = "text/html"
# I have no idea why. env.response.status_code = status_code
issue_template_components = get_issue_template(env, exception)
issue_title, issue_template = issue_template_components
# URLs for the error message below # Unpacking into issue_title, issue_template directly causes a compiler error
url_faq = "https://github.com/iv-org/documentation/blob/master/docs/faq.md" # I have no idea why.
url_search_issues = "https://github.com/iv-org/invidious/issues" issue_template_components = self.get_issue_template(env, exception)
url_search_issues += "?q=is:issue+is:open+" issue_title, issue_template = issue_template_components
url_search_issues += URI.encode_www_form("[Bug] #{issue_title}")
url_switch = "https://redirect.invidious.io" + env.request.resource # URLs for the error message below
url_faq = "https://github.com/iv-org/documentation/blob/master/docs/faq.md"
url_search_issues = "https://github.com/iv-org/invidious/issues"
url_search_issues += "?q=is:issue+is:open+"
url_search_issues += URI.encode_www_form("[Bug] #{issue_title}")
url_new_issue = "https://github.com/iv-org/invidious/issues/new" url_switch = "https://redirect.invidious.io" + env.request.resource
url_new_issue += "?labels=bug&template=bug_report.md&title="
url_new_issue += URI.encode_www_form("[Bug] " + issue_title)
error_message = <<-END_HTML url_new_issue = "https://github.com/iv-org/invidious/issues/new"
url_new_issue += "?labels=bug&template=bug_report.md&title="
url_new_issue += URI.encode_www_form("[Bug] " + issue_title)
error_message = <<-END_HTML
<div class="error_message"> <div class="error_message">
<h2>#{I18n.translate(locale, "crash_page_you_found_a_bug")}</h2> <h2>#{I18n.translate(locale, "crash_page_you_found_a_bug")}</h2>
<br/><br/> <br/><br/>
@ -82,116 +85,116 @@ def error_template_helper(env : HTTP::Server::Context, status_code : Int32, exce
</div> </div>
END_HTML END_HTML
# Don't show the usual "next steps" widget. The same options are # Don't show the usual "next steps" widget. The same options are
# proposed above the error message, just worded differently. # proposed above the error message, just worded differently.
next_steps = "" next_steps = ""
return templated "error" return templated "error"
end
def error_template_helper(env : HTTP::Server::Context, status_code : Int32, message : String)
env.response.content_type = "text/html"
env.response.status_code = status_code
locale = env.get("preferences").as(Preferences).locale
error_message = I18n.translate(locale, message)
next_steps = error_redirect_helper(env)
return templated "error"
end
# -------------------
# Atom feeds
# -------------------
macro error_atom(*args)
error_atom_helper(env, {{args.splat}})
end
def error_atom_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception)
if exception.is_a?(InfoException)
return error_atom_helper(env, status_code, exception.message || "")
end end
env.response.content_type = "application/atom+xml" def error_template_helper(env : HTTP::Server::Context, status_code : Int32, message : String)
env.response.status_code = status_code env.response.content_type = "text/html"
env.response.status_code = status_code
return "<error>#{exception.inspect_with_backtrace}</error>" locale = env.get("preferences").as(Preferences).locale
end
def error_atom_helper(env : HTTP::Server::Context, status_code : Int32, message : String) error_message = I18n.translate(locale, message)
env.response.content_type = "application/atom+xml" next_steps = error_redirect_helper(env)
env.response.status_code = status_code
return "<error>#{message}</error>" return templated "error"
end
# -------------------
# JSON
# -------------------
macro error_json(*args)
error_json_helper(env, {{args.splat}})
end
def error_json_helper(
env : HTTP::Server::Context,
status_code : Int32,
exception : Exception,
additional_fields : Hash(String, Object) | Nil = nil,
)
if exception.is_a?(InfoException)
return error_json_helper(env, status_code, exception.message || "", additional_fields)
end end
env.response.content_type = "application/json" # -------------------
env.response.status_code = status_code # Atom feeds
# -------------------
error_message = {"error" => exception.message, "errorBacktrace" => exception.inspect_with_backtrace} macro error_atom(*args)
Errors.error_atom_helper(env, {{args.splat}})
if additional_fields
error_message = error_message.merge(additional_fields)
end end
return error_message.to_json def error_atom_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception)
end if exception.is_a?(InfoException)
return error_atom_helper(env, status_code, exception.message || "")
end
def error_json_helper( env.response.content_type = "application/atom+xml"
env : HTTP::Server::Context, env.response.status_code = status_code
status_code : Int32,
message : String,
additional_fields : Hash(String, Object) | Nil = nil,
)
env.response.content_type = "application/json"
env.response.status_code = status_code
error_message = {"error" => message} return "<error>#{exception.inspect_with_backtrace}</error>"
if additional_fields
error_message = error_message.merge(additional_fields)
end end
return error_message.to_json def error_atom_helper(env : HTTP::Server::Context, status_code : Int32, message : String)
end env.response.content_type = "application/atom+xml"
env.response.status_code = status_code
# ------------------- return "<error>#{message}</error>"
# Redirect end
# -------------------
def error_redirect_helper(env : HTTP::Server::Context) # -------------------
request_path = env.request.path # JSON
# -------------------
locale = env.get("preferences").as(Preferences).locale macro error_json(*args)
Errors.error_json_helper(env, {{args.splat}})
end
if request_path.starts_with?("/search") || request_path.starts_with?("/watch") || def error_json_helper(
request_path.starts_with?("/channel") || request_path.starts_with?("/playlist?list=PL") env : HTTP::Server::Context,
next_steps_text = I18n.translate(locale, "next_steps_error_message") status_code : Int32,
refresh = I18n.translate(locale, "next_steps_error_message_refresh") exception : Exception,
go_to_youtube = I18n.translate(locale, "next_steps_error_message_go_to_youtube") additional_fields : Hash(String, Object) | Nil = nil,
switch_instance = I18n.translate(locale, "Switch Invidious Instance") )
if exception.is_a?(InfoException)
return error_json_helper(env, status_code, exception.message || "", additional_fields)
end
return <<-END_HTML env.response.content_type = "application/json"
env.response.status_code = status_code
error_message = {"error" => exception.message, "errorBacktrace" => exception.inspect_with_backtrace}
if additional_fields
error_message = error_message.merge(additional_fields)
end
return error_message.to_json
end
def error_json_helper(
env : HTTP::Server::Context,
status_code : Int32,
message : String,
additional_fields : Hash(String, Object) | Nil = nil,
)
env.response.content_type = "application/json"
env.response.status_code = status_code
error_message = {"error" => message}
if additional_fields
error_message = error_message.merge(additional_fields)
end
return error_message.to_json
end
# -------------------
# Redirect
# -------------------
def error_redirect_helper(env : HTTP::Server::Context)
request_path = env.request.path
locale = env.get("preferences").as(Preferences).locale
if request_path.starts_with?("/search") || request_path.starts_with?("/watch") ||
request_path.starts_with?("/channel") || request_path.starts_with?("/playlist?list=PL")
next_steps_text = I18n.translate(locale, "next_steps_error_message")
refresh = I18n.translate(locale, "next_steps_error_message_refresh")
go_to_youtube = I18n.translate(locale, "next_steps_error_message_go_to_youtube")
switch_instance = I18n.translate(locale, "Switch Invidious Instance")
return <<-END_HTML
<p style="margin-bottom: 4px;">#{next_steps_text}</p> <p style="margin-bottom: 4px;">#{next_steps_text}</p>
<ul> <ul>
<li> <li>
@ -205,7 +208,8 @@ def error_redirect_helper(env : HTTP::Server::Context)
</li> </li>
</ul> </ul>
END_HTML END_HTML
else else
return "" return ""
end
end end
end end

2
src/invidious/helpers/handlers.cr
View File

@ -83,7 +83,7 @@ class AuthHandler < Kemal::Handler
if token = env.request.headers["Authorization"]? if token = env.request.headers["Authorization"]?
token = JSON.parse(URI.decode_www_form(token.lchop("Bearer "))) token = JSON.parse(URI.decode_www_form(token.lchop("Bearer ")))
session = URI.decode_www_form(token["session"].as_s) session = URI.decode_www_form(token["session"].as_s)
scopes, _, _ = validate_request(token, session, env.request, HMAC_KEY, nil) scopes, _, _ = Invidious::Helpers::Tokens.validate_request(token, session, env.request, HMAC_KEY, nil)
if email = Invidious::Database::SessionIDs.select_email(session) if email = Invidious::Database::SessionIDs.select_email(session)
user = Invidious::Database::Users.select!(email: email) user = Invidious::Database::Users.select!(email: email)

4
src/invidious/helpers/helpers.cr
View File

@ -22,7 +22,7 @@ struct Annotation
property annotations : String property annotations : String
end end
module Helpers module Invidious::Helpers
extend self extend self
private TEST_IDS = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"} private TEST_IDS = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"}
@ -62,7 +62,7 @@ module Helpers
end end
def create_notification_stream(env, topics, connection_channel) def create_notification_stream(env, topics, connection_channel)
connection = Channel(PQ::Notification).new(8) connection = ::Channel(PQ::Notification).new(8)
connection_channel.send({true, connection}) connection_channel.send({true, connection})
locale = env.get("preferences").as(Preferences).locale locale = env.get("preferences").as(Preferences).locale

10
src/invidious/helpers/serialized_yt_data.cr
View File

@ -53,7 +53,7 @@ struct SearchVideo
xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg") xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg")
end end
xml.element("p", style: "word-break:break-word;white-space:pre-wrap") { xml.text Helpers.html_to_content(self.description_html) } xml.element("p", style: "word-break:break-word;white-space:pre-wrap") { xml.text Invidious::Helpers.html_to_content(self.description_html) }
end end
end end
@ -63,7 +63,7 @@ struct SearchVideo
xml.element("media:title") { xml.text self.title } xml.element("media:title") { xml.text self.title }
xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg", xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg",
width: "320", height: "180") width: "320", height: "180")
xml.element("media:description") { xml.text Helpers.html_to_content(self.description_html) } xml.element("media:description") { xml.text Invidious::Helpers.html_to_content(self.description_html) }
end end
xml.element("media:community") do xml.element("media:community") do
@ -111,7 +111,7 @@ struct SearchVideo
Invidious::JSONify::APIv1.thumbnails(json, self.id) Invidious::JSONify::APIv1.thumbnails(json, self.id)
end end
json.field "description", Helpers.html_to_content(self.description_html) json.field "description", Invidious::Helpers.html_to_content(self.description_html)
json.field "descriptionHtml", self.description_html json.field "descriptionHtml", self.description_html
json.field "viewCount", self.views json.field "viewCount", self.views
@ -255,7 +255,7 @@ struct SearchChannel
json.field "videoCount", self.video_count json.field "videoCount", self.video_count
json.field "channelHandle", self.channel_handle json.field "channelHandle", self.channel_handle
json.field "description", Helpers.html_to_content(self.description_html) json.field "description", Invidious::Helpers.html_to_content(self.description_html)
json.field "descriptionHtml", self.description_html json.field "descriptionHtml", self.description_html
end end
end end
@ -332,7 +332,7 @@ struct ProblematicTimelineItem
end end
xml.element("pre") do xml.element("pre") do
get_issue_template(env, @parse_exception) Errors.get_issue_template(env, @parse_exception)
end end
end end
end end

250
src/invidious/helpers/tokens.cr
View File

@ -1,145 +1,149 @@
require "crypto/subtle" require "crypto/subtle"
def generate_token(email, scopes, expire, key) module Invidious::Helpers::Tokens
session = "v1:#{Base64.urlsafe_encode(Random::Secure.random_bytes(32))}" extend self
Invidious::Database::SessionIDs.insert(session, email)
token = { def generate_token(email, scopes, expire, key)
"session" => session, session = "v1:#{Base64.urlsafe_encode(Random::Secure.random_bytes(32))}"
"scopes" => scopes, Invidious::Database::SessionIDs.insert(session, email)
"expire" => expire,
}
if !expire token = {
token.delete("expire") "session" => session,
end "scopes" => scopes,
"expire" => expire,
}
token["signature"] = sign_token(key, token) if !expire
token.delete("expire")
return token.to_json
end
def generate_response(session, scopes, key, expire = 6.hours, use_nonce = false)
expire = Time.utc + expire
token = {
"session" => session,
"expire" => expire.to_unix,
"scopes" => scopes,
}
if use_nonce
nonce = Random::Secure.hex(16)
Invidious::Database::Nonces.insert(nonce, expire)
token["nonce"] = nonce
end
token["signature"] = sign_token(key, token)
return token.to_json
end
def sign_token(key, hash)
string_to_sign = [] of String
# TODO: figure out which "key" variable is used
# Ameba reports a warning for "Lint/ShadowingOuterLocalVar" on this
# variable, but it's preferable to not touch that (works fine atm).
hash.each do |key, value|
next if key == "signature"
if value.is_a?(JSON::Any) && value.as_a?
value = value.as_a.map(&.as_s)
end end
case value token["signature"] = sign_token(key, token)
when Array
string_to_sign << "#{key}=#{value.sort.join(",")}" return token.to_json
when Tuple end
string_to_sign << "#{key}=#{value.to_a.sort.join(",")}"
else def generate_response(session, scopes, key, expire = 6.hours, use_nonce = false)
string_to_sign << "#{key}=#{value}" expire = Time.utc + expire
token = {
"session" => session,
"expire" => expire.to_unix,
"scopes" => scopes,
}
if use_nonce
nonce = Random::Secure.hex(16)
Invidious::Database::Nonces.insert(nonce, expire)
token["nonce"] = nonce
end end
token["signature"] = sign_token(key, token)
return token.to_json
end end
string_to_sign = string_to_sign.sort.join("\n") def sign_token(key, hash)
return Base64.urlsafe_encode(OpenSSL::HMAC.digest(:sha256, key, string_to_sign)).strip string_to_sign = [] of String
end
def validate_request(token, session, request, key, locale = nil) # TODO: figure out which "key" variable is used
case token # Ameba reports a warning for "Lint/ShadowingOuterLocalVar" on this
when String # variable, but it's preferable to not touch that (works fine atm).
token = JSON.parse(URI.decode_www_form(token)).as_h hash.each do |key, value|
when JSON::Any next if key == "signature"
token = token.as_h
when Nil if value.is_a?(JSON::Any) && value.as_a?
raise InfoException.new("Hidden field \"token\" is a required field") value = value.as_a.map(&.as_s)
end
case value
when Array
string_to_sign << "#{key}=#{value.sort.join(",")}"
when Tuple
string_to_sign << "#{key}=#{value.to_a.sort.join(",")}"
else
string_to_sign << "#{key}=#{value}"
end
end
string_to_sign = string_to_sign.sort.join("\n")
return Base64.urlsafe_encode(OpenSSL::HMAC.digest(:sha256, key, string_to_sign)).strip
end end
expire = token["expire"]?.try &.as_i def validate_request(token, session, request, key, locale = nil)
if expire.try &.< Time.utc.to_unix case token
raise InfoException.new("Token is expired, please try again") when String
end token = JSON.parse(URI.decode_www_form(token)).as_h
when JSON::Any
token = token.as_h
when Nil
raise InfoException.new("Hidden field \"token\" is a required field")
end
if token["session"] != session expire = token["expire"]?.try &.as_i
raise InfoException.new("Erroneous token") if expire.try &.< Time.utc.to_unix
end raise InfoException.new("Token is expired, please try again")
end
scopes = token["scopes"].as_a.map(&.as_s) if token["session"] != session
scope = "#{request.method}:#{request.path.lchop("/api/v1/auth/").lstrip("/")}"
if !scopes_include_scope(scopes, scope)
raise InfoException.new("Invalid scope")
end
if !Crypto::Subtle.constant_time_compare(token["signature"].to_s, sign_token(key, token))
raise InfoException.new("Invalid signature")
end
if token["nonce"]? && (nonce = Invidious::Database::Nonces.select(token["nonce"].as_s))
if nonce[1] > Time.utc
Invidious::Database::Nonces.update_set_expired(nonce[0])
else
raise InfoException.new("Erroneous token") raise InfoException.new("Erroneous token")
end end
end
return {scopes, expire, token["signature"].as_s} scopes = token["scopes"].as_a.map(&.as_s)
end scope = "#{request.method}:#{request.path.lchop("/api/v1/auth/").lstrip("/")}"
if !scopes_include_scope(scopes, scope)
def scope_includes_scope(scope, subset) raise InfoException.new("Invalid scope")
methods, endpoint = scope.split(":")
methods = methods.split(";").map(&.upcase).reject(&.empty?).sort!
endpoint = endpoint.downcase
subset_methods, subset_endpoint = subset.split(":")
subset_methods = subset_methods.split(";").map(&.upcase).sort!
subset_endpoint = subset_endpoint.downcase
if methods.empty?
methods = %w(GET POST PUT HEAD DELETE PATCH OPTIONS)
end
if methods & subset_methods != subset_methods
return false
end
if endpoint.ends_with?("*") && !subset_endpoint.starts_with? endpoint.rchop("*")
return false
end
if !endpoint.ends_with?("*") && subset_endpoint != endpoint
return false
end
return true
end
def scopes_include_scope(scopes, subset)
scopes.each do |scope|
if scope_includes_scope(scope, subset)
return true
end end
if !Crypto::Subtle.constant_time_compare(token["signature"].to_s, sign_token(key, token))
raise InfoException.new("Invalid signature")
end
if token["nonce"]? && (nonce = Invidious::Database::Nonces.select(token["nonce"].as_s))
if nonce[1] > Time.utc
Invidious::Database::Nonces.update_set_expired(nonce[0])
else
raise InfoException.new("Erroneous token")
end
end
return {scopes, expire, token["signature"].as_s}
end end
return false def scope_includes_scope(scope, subset)
methods, endpoint = scope.split(":")
methods = methods.split(";").map(&.upcase).reject(&.empty?).sort!
endpoint = endpoint.downcase
subset_methods, subset_endpoint = subset.split(":")
subset_methods = subset_methods.split(";").map(&.upcase).sort!
subset_endpoint = subset_endpoint.downcase
if methods.empty?
methods = %w(GET POST PUT HEAD DELETE PATCH OPTIONS)
end
if methods & subset_methods != subset_methods
return false
end
if endpoint.ends_with?("*") && !subset_endpoint.starts_with? endpoint.rchop("*")
return false
end
if !endpoint.ends_with?("*") && subset_endpoint != endpoint
return false
end
return true
end
def scopes_include_scope(scopes, subset)
scopes.each do |scope|
if self.scope_includes_scope(scope, subset)
return true
end
end
return false
end
end end

2
src/invidious/jobs/refresh_channels_job.cr
View File

@ -30,7 +30,7 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob
spawn do spawn do
begin begin
LOGGER.trace("RefreshChannelsJob: #{id} fiber : Fetching channel") LOGGER.trace("RefreshChannelsJob: #{id} fiber : Fetching channel")
channel = fetch_channel(id, pull_all_videos: CONFIG.full_refresh) channel = Invidious::Channels::Channels.fetch_channel(id, pull_all_videos: CONFIG.full_refresh)
lim_fibers = max_fibers lim_fibers = max_fibers

2
src/invidious/jobs/refresh_feeds_job.cr
View File

@ -54,7 +54,7 @@ class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob
# While iterating through, we may have an email stored from a deleted account # While iterating through, we may have an email stored from a deleted account
if db.query_one?("SELECT true FROM users WHERE email = $1", email, as: Bool) if db.query_one?("SELECT true FROM users WHERE email = $1", email, as: Bool)
LOGGER.info("RefreshFeedsJob: CREATE #{view_name}") LOGGER.info("RefreshFeedsJob: CREATE #{view_name}")
db.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(email)}") db.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{Invidious::User::Users::MATERIALIZED_VIEW_SQL.call(email)}")
db.exec("UPDATE users SET feed_needs_update = false WHERE email = $1", email) db.exec("UPDATE users SET feed_needs_update = false WHERE email = $1", email)
end end
rescue ex rescue ex

579
src/invidious/playlists.cr
View File

@ -1,579 +0,0 @@
struct PlaylistVideo
include DB::Serializable
property title : String
property id : String
property author : String
property ucid : String
property length_seconds : Int32
property published : Time
property plid : String
property index : Int64
property live_now : Bool
def to_xml(xml : XML::Builder)
xml.element("entry") do
xml.element("id") { xml.text "yt:video:#{self.id}" }
xml.element("yt:videoId") { xml.text self.id }
xml.element("yt:channelId") { xml.text self.ucid }
xml.element("title") { xml.text self.title }
xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?v=#{self.id}")
xml.element("author") do
xml.element("name") { xml.text self.author }
xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" }
end
xml.element("content", type: "xhtml") do
xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do
xml.element("a", href: "#{HOST_URL}/watch?v=#{self.id}") do
xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg")
end
end
end
xml.element("published") { xml.text self.published.to_s("%Y-%m-%dT%H:%M:%S%:z") }
xml.element("media:group") do
xml.element("media:title") { xml.text self.title }
xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg",
width: "320", height: "180")
end
end
end
def to_xml(_xml : Nil = nil)
XML.build { |xml| to_xml(xml) }
end
def to_json(locale : String?, json : JSON::Builder)
to_json(json)
end
def to_json(json : JSON::Builder, index : Int32? = nil)
json.object do
json.field "type", "video"
json.field "title", self.title
json.field "videoId", self.id
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "videoThumbnails" do
Invidious::JSONify::APIv1.thumbnails(json, self.id)
end
if index
json.field "index", index
json.field "indexId", self.index.to_u64.to_s(16).upcase
else
json.field "index", self.index
end
json.field "lengthSeconds", self.length_seconds
json.field "liveNow", self.live_now
end
end
def to_json(_json : Nil, index : Int32? = nil)
JSON.build { |json| to_json(json, index: index) }
end
end
struct Playlist
include DB::Serializable
property title : String
property id : String
property author : String
property author_thumbnail : String
property ucid : String
property description : String
property description_html : String
property video_count : Int32
property views : Int64
property updated : Time
property thumbnail : String?
property subtitle : String?
def to_json(offset, json : JSON::Builder, video_id : String? = nil)
json.object do
json.field "type", "playlist"
json.field "title", self.title
json.field "playlistId", self.id
json.field "playlistThumbnail", self.thumbnail
json.field "author", self.author
json.field "authorId", self.ucid
if !self.ucid.empty?
json.field "authorUrl", "/channel/#{self.ucid}"
else
json.field "authorUrl", ""
end
json.field "subtitle", self.subtitle
json.field "authorThumbnails" do
json.array do
qualities = {32, 48, 76, 100, 176, 512}
qualities.each do |quality|
json.object do
json.field "url", self.author_thumbnail.not_nil!.gsub(/=\d+/, "=s#{quality}")
json.field "width", quality
json.field "height", quality
end
end
end
end
json.field "description", self.description
json.field "descriptionHtml", self.description_html
json.field "videoCount", self.video_count
json.field "viewCount", self.views
json.field "updated", self.updated.to_unix
json.field "isListed", self.privacy.public?
json.field "videos" do
json.array do
videos = get_playlist_videos(self, offset: offset, video_id: video_id)
videos.each do |video|
video.to_json(json)
end
end
end
end
end
def to_json(offset, _json : Nil = nil, video_id : String? = nil)
JSON.build do |json|
to_json(offset, json, video_id: video_id)
end
end
def privacy
PlaylistPrivacy::Public
end
end
enum PlaylistPrivacy
Public = 0
Unlisted = 1
Private = 2
end
struct InvidiousPlaylist
include DB::Serializable
property title : String
property id : String
property author : String
property description : String = ""
property video_count : Int32
property created : Time
property updated : Time
@[DB::Field(converter: InvidiousPlaylist::PlaylistPrivacyConverter)]
property privacy : PlaylistPrivacy = PlaylistPrivacy::Private
property index : Array(Int64)
@[DB::Field(ignore: true)]
property thumbnail_id : String?
module PlaylistPrivacyConverter
def self.from_rs(rs)
return PlaylistPrivacy.parse(String.new(rs.read(Slice(UInt8))))
end
end
def to_json(offset, json : JSON::Builder, video_id : String? = nil)
json.object do
json.field "type", "invidiousPlaylist"
json.field "title", self.title
json.field "playlistId", self.id
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", nil
json.field "authorThumbnails", [] of String
json.field "description", Helpers.html_to_content(self.description_html)
json.field "descriptionHtml", self.description_html
json.field "videoCount", self.video_count
json.field "viewCount", self.views
json.field "updated", self.updated.to_unix
json.field "isListed", self.privacy.public?
json.field "videos" do
json.array do
if (!offset || offset == 0) && !video_id.nil?
index = Invidious::Database::PlaylistVideos.select_index(self.id, video_id)
offset = self.index.index(index) || 0
end
videos = get_playlist_videos(self, offset: offset, video_id: video_id)
videos.each_with_index do |video, idx|
video.to_json(json, offset + idx)
end
end
end
end
end
def to_json(offset, _json : Nil = nil, video_id : String? = nil)
JSON.build do |json|
to_json(offset, json, video_id: video_id)
end
end
def thumbnail
# TODO: Get playlist thumbnail from playlist data rather than first video
@thumbnail_id ||= Invidious::Database::PlaylistVideos.select_one_id(self.id, self.index) || "-----------"
"/vi/#{@thumbnail_id}/mqdefault.jpg"
end
def author_thumbnail
nil
end
def ucid
nil
end
def views
0_i64
end
def description_html
HTML.escape(self.description)
end
end
def create_playlist(title, privacy, user)
plid = "IVPL#{Random::Secure.urlsafe_base64(24)[0, 31]}"
playlist = InvidiousPlaylist.new({
title: title.byte_slice(0, 150),
id: plid,
author: user.email,
description: "", # Max 5000 characters
video_count: 0,
created: Time.utc,
updated: Time.utc,
privacy: privacy,
index: [] of Int64,
})
Invidious::Database::Playlists.insert(playlist)
return playlist
end
def subscribe_playlist(user, playlist)
playlist = InvidiousPlaylist.new({
title: playlist.title[..150],
id: playlist.id,
author: user.email,
description: "", # Max 5000 characters
video_count: playlist.video_count,
created: Time.utc,
updated: playlist.updated,
privacy: PlaylistPrivacy::Private,
index: [] of Int64,
})
Invidious::Database::Playlists.insert(playlist)
return playlist
end
def produce_playlist_continuation(id, index)
if id.starts_with? "UC"
id = "UU" + id.lchop("UC")
end
plid = "VL" + id
# Emulate a "request counter" increment, to make perfectly valid
# ctokens, even if at the time of writing, it's ignored by youtube.
request_count = (index / 100).to_i64 || 1_i64
data = {"1:varint" => index.to_i64}
.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i, padding: false) }
object = {
"80226972:embedded" => {
"2:string" => plid,
"3:base64" => {
"1:varint" => request_count,
"15:string" => "PT:#{data}",
"104:embedded" => {"1:0:varint" => 0_i64},
},
"35:string" => id,
},
}
continuation = object.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
return continuation
end
def get_playlist(plid : String)
if plid.starts_with? "IV"
if playlist = Invidious::Database::Playlists.select(id: plid)
return playlist
else
raise NotFoundException.new("Playlist does not exist.")
end
else
return fetch_playlist(plid)
end
end
def fetch_playlist(plid : String)
if plid.starts_with? "UC"
plid = "UU#{plid.lchop("UC")}"
end
initial_data = YoutubeAPI.browse("VL" + plid, params: "")
playlist_sidebar_renderer = initial_data.dig?("sidebar", "playlistSidebarRenderer", "items")
raise InfoException.new("Could not extract playlistSidebarRenderer.") if !playlist_sidebar_renderer
playlist_info = playlist_sidebar_renderer.dig?(0, "playlistSidebarPrimaryInfoRenderer")
raise InfoException.new("Could not extract playlist info") if !playlist_info
title = playlist_info.dig?("title", "runs", 0, "text").try &.as_s || ""
desc_item = playlist_info["description"]?
description_txt = desc_item.try &.["runs"]?.try &.as_a
.map(&.["text"].as_s).join("") || desc_item.try &.["simpleText"]?.try &.as_s || ""
description_html = desc_item.try &.["runs"]?.try &.as_a
.try { |run| content_to_comment_html(run).try &.to_s } || "<p></p>"
thumbnail = playlist_info.dig?(
"thumbnailRenderer", "playlistVideoThumbnailRenderer",
"thumbnail", "thumbnails", 0, "url"
).try &.as_s || playlist_info.dig?(
"thumbnailRenderer", "playlistCustomThumbnailRenderer",
"thumbnail", "thumbnails", 0, "url"
).try &.as_s
views = 0_i64
updated = Time.utc
video_count = 0
subtitle = extract_text(initial_data.dig?("header", "playlistHeaderRenderer", "subtitle"))
playlist_info["stats"]?.try &.as_a.each do |stat|
text = stat["runs"]?.try &.as_a.map(&.["text"].as_s).join("") || stat["simpleText"]?.try &.as_s
next if !text
if text.includes? "video"
video_count = text.gsub(/\D/, "").to_i? || 0
elsif text.includes? "episode"
video_count = text.gsub(/\D/, "").to_i? || 0
elsif text.includes? "view"
views = text.gsub(/\D/, "").to_i64? || 0_i64
elsif !text.includes? "Pay to watch"
updated = decode_date(text.lchop("Last updated on ").lchop("Updated "))
end
end
if playlist_sidebar_renderer.size < 2
author = ""
author_thumbnail = ""
ucid = ""
else
author_info = playlist_sidebar_renderer[1].dig?(
"playlistSidebarSecondaryInfoRenderer", "videoOwner", "videoOwnerRenderer"
)
raise InfoException.new("Could not extract author info") if !author_info
author = author_info.dig?("title", "runs", 0, "text").try &.as_s || ""
author_thumbnail = author_info.dig?("thumbnail", "thumbnails", 0, "url").try &.as_s || ""
ucid = author_info.dig?("title", "runs", 0, "navigationEndpoint", "browseEndpoint", "browseId").try &.as_s || ""
end
return Playlist.new({
title: title,
id: plid,
author: author,
author_thumbnail: author_thumbnail,
ucid: ucid,
description: description_txt,
description_html: description_html,
video_count: video_count,
views: views,
updated: updated,
thumbnail: thumbnail,
subtitle: subtitle,
})
end
def get_playlist_videos(playlist : InvidiousPlaylist | Playlist, offset : Int32, video_id = nil)
# Show empty playlist if requested page is out of range
# (e.g, when a new playlist has been created, offset will be negative)
if offset >= playlist.video_count || offset < 0
return [] of PlaylistVideo
end
if playlist.is_a? InvidiousPlaylist
Invidious::Database::PlaylistVideos.select(playlist.id, playlist.index, offset, limit: 100)
else
if video_id
initial_data = YoutubeAPI.next({
"videoId" => video_id,
"playlistId" => playlist.id,
})
offset = initial_data.dig?("contents", "twoColumnWatchNextResults", "playlist", "playlist", "currentIndex").try &.as_i || offset
end
videos = [] of PlaylistVideo | ProblematicTimelineItem
until videos.size >= 200 || videos.size == playlist.video_count || offset >= playlist.video_count
# 100 videos per request
ctoken = produce_playlist_continuation(playlist.id, offset)
initial_data = YoutubeAPI.browse(ctoken)
videos += extract_playlist_videos(playlist.id, initial_data)
offset += 100
end
return videos
end
end
# TODO (2026-06-24): Migrate this function to use parsers instead, as it uses,
# the same LockupViewModel used in Channel videos and Youtube playlists that
# appears on searches (Invidious /search endpoint).
# Related to https://github.com/iv-org/invidious/pull/5736
def extract_playlist_videos(playlist_id : String, initial_data : Hash(String, JSON::Any))
videos = [] of PlaylistVideo | ProblematicTimelineItem
if initial_data["contents"]?
tabs = initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]
tabs_renderer = tabs.as_a.select(&.["tabRenderer"]["selected"]?.try &.as_bool)[0]["tabRenderer"]
# Watch out the two versions, with and without "s"
if tabs_renderer["contents"]? || tabs_renderer["content"]?
# Initial playlist data
tabs_contents = tabs_renderer.["contents"]? || tabs_renderer.["content"]
list_renderer = tabs_contents.["sectionListRenderer"]["contents"][0]
contents = list_renderer.["itemSectionRenderer"]["contents"].as_a
else
# Continuation data
contents = initial_data["onResponseReceivedActions"][0]?
.try &.["appendContinuationItemsAction"]["continuationItems"].as_a
end
else
contents = initial_data["response"]?.try &.["continuationContents"]["playlistVideoListContinuation"]["contents"].as_a
end
contents.try &.each do |item|
if i = item["lockupViewModel"]?
thumbnail_view_model = i.dig?(
"contentImage", "thumbnailViewModel"
)
watch_endpoint = i.dig?("rendererContext", "commandContext", "onTap", "innertubeCommand", "watchEndpoint")
video_id = watch_endpoint.try &.["videoId"]?.try &.as_s
plid = watch_endpoint.try &.["playlistId"]?.try &.as_s || playlist_id
index = watch_endpoint.try &.["index"]?.try &.as_i64
metadata = i["metadata"]?
lockup_metadata_view_model = metadata.try &.dig?("lockupMetadataViewModel")
title = lockup_metadata_view_model.try &.dig?("title", "content").try &.as_s
lockup_metadata = lockup_metadata_view_model.try &.dig?("metadata")
metadata_rows = lockup_metadata.try &.dig?("contentMetadataViewModel", "metadataRows").try &.as_a
# Find the metadataParts with commandRuns inside, which contains author
# information.
metadata_parts = metadata_rows.try &.find { |row|
parts = row["metadataParts"]?.try &.as_a
parts && parts.any? { |item2| item2.dig?("text", "commandRuns").try &.as_a }
}.try &.["metadataParts"].as_a
if author_info = metadata_parts.try &.find(&.dig?("text", "commandRuns"))
.try &.["text"]
author = author_info["content"].as_s
ucid = author_info.dig?("commandRuns", 0, "onTap", "innertubeCommand", "browseEndpoint", "browseId")
.try &.as_s
end
length = thumbnail_view_model.try &.dig?("overlays", 0, "thumbnailBottomOverlayViewModel", "badges", 0, "thumbnailBadgeViewModel", "text").try &.as_s
length_seconds = decode_length_seconds(length) if length
live = false
if !length_seconds
live = true
length_seconds = 0
end
videos << PlaylistVideo.new({
title: title || "",
id: video_id || "",
author: author || "",
ucid: ucid || "",
length_seconds: length_seconds,
published: Time.utc,
plid: plid,
live_now: live,
index: index || -1_i64,
})
end
rescue ex
videos << ProblematicTimelineItem.new(parse_exception: ex)
end
return videos
end
def template_playlist(playlist, listen)
html = <<-END_HTML
<h3>
<a href="/playlist?list=#{playlist["playlistId"]}">
#{playlist["title"]}
</a>
</h3>
<div class="pure-menu pure-menu-scrollable playlist-restricted">
<ol class="pure-menu-list">
END_HTML
playlist["videos"].as_a.each do |video|
html += <<-END_HTML
<li class="pure-menu-item" id="#{video["videoId"]}">
<a href="/watch?v=#{video["videoId"]}&list=#{playlist["playlistId"]}&index=#{video["index"]}#{listen ? "&listen=1" : ""}">
<div class="thumbnail">
<img loading="lazy" class="thumbnail" src="/vi/#{video["videoId"]}/mqdefault.jpg" alt="" />
<p class="length">#{recode_length_seconds(video["lengthSeconds"].as_i)}</p>
</div>
<p style="width:100%">#{video["title"]}</p>
<p>
<b style="width:100%">#{video["author"]}</b>
</p>
</a>
</li>
END_HTML
end
html += <<-END_HTML
</ol>
</div>
<hr>
END_HTML
html
end

2
src/invidious/mixes.cr → src/invidious/playlists/mixes.cr
View File

@ -27,7 +27,7 @@ def fetch_mix(rdid, video_id, cookies = nil, locale = nil)
video_id = "CvFH_6DNRCY" if rdid.starts_with? "OLAK5uy_" video_id = "CvFH_6DNRCY" if rdid.starts_with? "OLAK5uy_"
response = YT_POOL.client &.get("/watch?v=#{video_id}&list=#{rdid}&gl=US&hl=en", headers) response = YT_POOL.client &.get("/watch?v=#{video_id}&list=#{rdid}&gl=US&hl=en", headers)
initial_data = Helpers.extract_initial_data(response.body) initial_data = Invidious::Helpers.extract_initial_data(response.body)
if !initial_data["contents"]["twoColumnWatchNextResults"]["playlist"]? if !initial_data["contents"]["twoColumnWatchNextResults"]["playlist"]?
raise InfoException.new("Could not create mix.") raise InfoException.new("Could not create mix.")

583
src/invidious/playlists/playlists.cr Normal file
View File

@ -0,0 +1,583 @@
struct PlaylistVideo
include DB::Serializable
property title : String
property id : String
property author : String
property ucid : String
property length_seconds : Int32
property published : Time
property plid : String
property index : Int64
property live_now : Bool
def to_xml(xml : XML::Builder)
xml.element("entry") do
xml.element("id") { xml.text "yt:video:#{self.id}" }
xml.element("yt:videoId") { xml.text self.id }
xml.element("yt:channelId") { xml.text self.ucid }
xml.element("title") { xml.text self.title }
xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?v=#{self.id}")
xml.element("author") do
xml.element("name") { xml.text self.author }
xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" }
end
xml.element("content", type: "xhtml") do
xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do
xml.element("a", href: "#{HOST_URL}/watch?v=#{self.id}") do
xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg")
end
end
end
xml.element("published") { xml.text self.published.to_s("%Y-%m-%dT%H:%M:%S%:z") }
xml.element("media:group") do
xml.element("media:title") { xml.text self.title }
xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg",
width: "320", height: "180")
end
end
end
def to_xml(_xml : Nil = nil)
XML.build { |xml| to_xml(xml) }
end
def to_json(locale : String?, json : JSON::Builder)
to_json(json)
end
def to_json(json : JSON::Builder, index : Int32? = nil)
json.object do
json.field "type", "video"
json.field "title", self.title
json.field "videoId", self.id
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "videoThumbnails" do
Invidious::JSONify::APIv1.thumbnails(json, self.id)
end
if index
json.field "index", index
json.field "indexId", self.index.to_u64.to_s(16).upcase
else
json.field "index", self.index
end
json.field "lengthSeconds", self.length_seconds
json.field "liveNow", self.live_now
end
end
def to_json(_json : Nil, index : Int32? = nil)
JSON.build { |json| to_json(json, index: index) }
end
end
struct Playlist
include DB::Serializable
property title : String
property id : String
property author : String
property author_thumbnail : String
property ucid : String
property description : String
property description_html : String
property video_count : Int32
property views : Int64
property updated : Time
property thumbnail : String?
property subtitle : String?
def to_json(offset, json : JSON::Builder, video_id : String? = nil)
json.object do
json.field "type", "playlist"
json.field "title", self.title
json.field "playlistId", self.id
json.field "playlistThumbnail", self.thumbnail
json.field "author", self.author
json.field "authorId", self.ucid
if !self.ucid.empty?
json.field "authorUrl", "/channel/#{self.ucid}"
else
json.field "authorUrl", ""
end
json.field "subtitle", self.subtitle
json.field "authorThumbnails" do
json.array do
qualities = {32, 48, 76, 100, 176, 512}
qualities.each do |quality|
json.object do
json.field "url", self.author_thumbnail.not_nil!.gsub(/=\d+/, "=s#{quality}")
json.field "width", quality
json.field "height", quality
end
end
end
end
json.field "description", self.description
json.field "descriptionHtml", self.description_html
json.field "videoCount", self.video_count
json.field "viewCount", self.views
json.field "updated", self.updated.to_unix
json.field "isListed", self.privacy.public?
json.field "videos" do
json.array do
videos = Invidious::Playlists::Playlists.get_playlist_videos(self, offset: offset, video_id: video_id)
videos.each do |video|
video.to_json(json)
end
end
end
end
end
def to_json(offset, _json : Nil = nil, video_id : String? = nil)
JSON.build do |json|
to_json(offset, json, video_id: video_id)
end
end
def privacy
PlaylistPrivacy::Public
end
end
enum PlaylistPrivacy
Public = 0
Unlisted = 1
Private = 2
end
struct InvidiousPlaylist
include DB::Serializable
property title : String
property id : String
property author : String
property description : String = ""
property video_count : Int32
property created : Time
property updated : Time
@[DB::Field(converter: InvidiousPlaylist::PlaylistPrivacyConverter)]
property privacy : PlaylistPrivacy = PlaylistPrivacy::Private
property index : Array(Int64)
@[DB::Field(ignore: true)]
property thumbnail_id : String?
module PlaylistPrivacyConverter
def self.from_rs(rs)
return PlaylistPrivacy.parse(String.new(rs.read(Slice(UInt8))))
end
end
def to_json(offset, json : JSON::Builder, video_id : String? = nil)
json.object do
json.field "type", "invidiousPlaylist"
json.field "title", self.title
json.field "playlistId", self.id
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", nil
json.field "authorThumbnails", [] of String
json.field "description", Invidious::Helpers.html_to_content(self.description_html)
json.field "descriptionHtml", self.description_html
json.field "videoCount", self.video_count
json.field "viewCount", self.views
json.field "updated", self.updated.to_unix
json.field "isListed", self.privacy.public?
json.field "videos" do
json.array do
if (!offset || offset == 0) && !video_id.nil?
index = Invidious::Database::PlaylistVideos.select_index(self.id, video_id)
offset = self.index.index(index) || 0
end
videos = Invidious::Playlists::Playlists.get_playlist_videos(self, offset: offset, video_id: video_id)
videos.each_with_index do |video, idx|
video.to_json(json, offset + idx)
end
end
end
end
end
def to_json(offset, _json : Nil = nil, video_id : String? = nil)
JSON.build do |json|
to_json(offset, json, video_id: video_id)
end
end
def thumbnail
# TODO: Get playlist thumbnail from playlist data rather than first video
@thumbnail_id ||= Invidious::Database::PlaylistVideos.select_one_id(self.id, self.index) || "-----------"
"/vi/#{@thumbnail_id}/mqdefault.jpg"
end
def author_thumbnail
nil
end
def ucid
nil
end
def views
0_i64
end
def description_html
HTML.escape(self.description)
end
end
module Invidious::Playlists::Playlists
extend self
def create_playlist(title, privacy, user)
plid = "IVPL#{Random::Secure.urlsafe_base64(24)[0, 31]}"
playlist = InvidiousPlaylist.new({
title: title.byte_slice(0, 150),
id: plid,
author: user.email,
description: "", # Max 5000 characters
video_count: 0,
created: Time.utc,
updated: Time.utc,
privacy: privacy,
index: [] of Int64,
})
Invidious::Database::Playlists.insert(playlist)
return playlist
end
def subscribe_playlist(user, playlist)
playlist = InvidiousPlaylist.new({
title: playlist.title[..150],
id: playlist.id,
author: user.email,
description: "", # Max 5000 characters
video_count: playlist.video_count,
created: Time.utc,
updated: playlist.updated,
privacy: PlaylistPrivacy::Private,
index: [] of Int64,
})
Invidious::Database::Playlists.insert(playlist)
return playlist
end
def produce_playlist_continuation(id, index)
if id.starts_with? "UC"
id = "UU" + id.lchop("UC")
end
plid = "VL" + id
# Emulate a "request counter" increment, to make perfectly valid
# ctokens, even if at the time of writing, it's ignored by youtube.
request_count = (index / 100).to_i64 || 1_i64
data = {"1:varint" => index.to_i64}
.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i, padding: false) }
object = {
"80226972:embedded" => {
"2:string" => plid,
"3:base64" => {
"1:varint" => request_count,
"15:string" => "PT:#{data}",
"104:embedded" => {"1:0:varint" => 0_i64},
},
"35:string" => id,
},
}
continuation = object.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
return continuation
end
def get_playlist(plid : String)
if plid.starts_with? "IV"
if playlist = Invidious::Database::Playlists.select(id: plid)
return playlist
else
raise NotFoundException.new("Playlist does not exist.")
end
else
return fetch_playlist(plid)
end
end
def fetch_playlist(plid : String)
if plid.starts_with? "UC"
plid = "UU#{plid.lchop("UC")}"
end
initial_data = YoutubeAPI.browse("VL" + plid, params: "")
playlist_sidebar_renderer = initial_data.dig?("sidebar", "playlistSidebarRenderer", "items")
raise InfoException.new("Could not extract playlistSidebarRenderer.") if !playlist_sidebar_renderer
playlist_info = playlist_sidebar_renderer.dig?(0, "playlistSidebarPrimaryInfoRenderer")
raise InfoException.new("Could not extract playlist info") if !playlist_info
title = playlist_info.dig?("title", "runs", 0, "text").try &.as_s || ""
desc_item = playlist_info["description"]?
description_txt = desc_item.try &.["runs"]?.try &.as_a
.map(&.["text"].as_s).join("") || desc_item.try &.["simpleText"]?.try &.as_s || ""
description_html = desc_item.try &.["runs"]?.try &.as_a
.try { |run| content_to_comment_html(run).try &.to_s } || "<p></p>"
thumbnail = playlist_info.dig?(
"thumbnailRenderer", "playlistVideoThumbnailRenderer",
"thumbnail", "thumbnails", 0, "url"
).try &.as_s || playlist_info.dig?(
"thumbnailRenderer", "playlistCustomThumbnailRenderer",
"thumbnail", "thumbnails", 0, "url"
).try &.as_s
views = 0_i64
updated = Time.utc
video_count = 0
subtitle = extract_text(initial_data.dig?("header", "playlistHeaderRenderer", "subtitle"))
playlist_info["stats"]?.try &.as_a.each do |stat|
text = stat["runs"]?.try &.as_a.map(&.["text"].as_s).join("") || stat["simpleText"]?.try &.as_s
next if !text
if text.includes? "video"
video_count = text.gsub(/\D/, "").to_i? || 0
elsif text.includes? "episode"
video_count = text.gsub(/\D/, "").to_i? || 0
elsif text.includes? "view"
views = text.gsub(/\D/, "").to_i64? || 0_i64
elsif !text.includes? "Pay to watch"
updated = decode_date(text.lchop("Last updated on ").lchop("Updated "))
end
end
if playlist_sidebar_renderer.size < 2
author = ""
author_thumbnail = ""
ucid = ""
else
author_info = playlist_sidebar_renderer[1].dig?(
"playlistSidebarSecondaryInfoRenderer", "videoOwner", "videoOwnerRenderer"
)
raise InfoException.new("Could not extract author info") if !author_info
author = author_info.dig?("title", "runs", 0, "text").try &.as_s || ""
author_thumbnail = author_info.dig?("thumbnail", "thumbnails", 0, "url").try &.as_s || ""
ucid = author_info.dig?("title", "runs", 0, "navigationEndpoint", "browseEndpoint", "browseId").try &.as_s || ""
end
return Playlist.new({
title: title,
id: plid,
author: author,
author_thumbnail: author_thumbnail,
ucid: ucid,
description: description_txt,
description_html: description_html,
video_count: video_count,
views: views,
updated: updated,
thumbnail: thumbnail,
subtitle: subtitle,
})
end
def get_playlist_videos(playlist : InvidiousPlaylist | Playlist, offset : Int32, video_id = nil)
# Show empty playlist if requested page is out of range
# (e.g, when a new playlist has been created, offset will be negative)
if offset >= playlist.video_count || offset < 0
return [] of PlaylistVideo
end
if playlist.is_a? InvidiousPlaylist
Invidious::Database::PlaylistVideos.select(playlist.id, playlist.index, offset, limit: 100)
else
if video_id
initial_data = YoutubeAPI.next({
"videoId" => video_id,
"playlistId" => playlist.id,
})
offset = initial_data.dig?("contents", "twoColumnWatchNextResults", "playlist", "playlist", "currentIndex").try &.as_i || offset
end
videos = [] of PlaylistVideo | ProblematicTimelineItem
until videos.size >= 200 || videos.size == playlist.video_count || offset >= playlist.video_count
# 100 videos per request
ctoken = produce_playlist_continuation(playlist.id, offset)
initial_data = YoutubeAPI.browse(ctoken)
videos += extract_playlist_videos(playlist.id, initial_data)
offset += 100
end
return videos
end
end
# TODO (2026-06-24): Migrate this function to use parsers instead, as it uses,
# the same LockupViewModel used in Channel videos and Youtube playlists that
# appears on searches (Invidious /search endpoint).
# Related to https://github.com/iv-org/invidious/pull/5736
def extract_playlist_videos(playlist_id : String, initial_data : Hash(String, JSON::Any))
videos = [] of PlaylistVideo | ProblematicTimelineItem
if initial_data["contents"]?
tabs = initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]
tabs_renderer = tabs.as_a.select(&.["tabRenderer"]["selected"]?.try &.as_bool)[0]["tabRenderer"]
# Watch out the two versions, with and without "s"
if tabs_renderer["contents"]? || tabs_renderer["content"]?
# Initial playlist data
tabs_contents = tabs_renderer.["contents"]? || tabs_renderer.["content"]
list_renderer = tabs_contents.["sectionListRenderer"]["contents"][0]
contents = list_renderer.["itemSectionRenderer"]["contents"].as_a
else
# Continuation data
contents = initial_data["onResponseReceivedActions"][0]?
.try &.["appendContinuationItemsAction"]["continuationItems"].as_a
end
else
contents = initial_data["response"]?.try &.["continuationContents"]["playlistVideoListContinuation"]["contents"].as_a
end
contents.try &.each do |item|
if i = item["lockupViewModel"]?
thumbnail_view_model = i.dig?(
"contentImage", "thumbnailViewModel"
)
watch_endpoint = i.dig?("rendererContext", "commandContext", "onTap", "innertubeCommand", "watchEndpoint")
video_id = watch_endpoint.try &.["videoId"]?.try &.as_s
plid = watch_endpoint.try &.["playlistId"]?.try &.as_s || playlist_id
index = watch_endpoint.try &.["index"]?.try &.as_i64
metadata = i["metadata"]?
lockup_metadata_view_model = metadata.try &.dig?("lockupMetadataViewModel")
title = lockup_metadata_view_model.try &.dig?("title", "content").try &.as_s
lockup_metadata = lockup_metadata_view_model.try &.dig?("metadata")
metadata_rows = lockup_metadata.try &.dig?("contentMetadataViewModel", "metadataRows").try &.as_a
# Find the metadataParts with commandRuns inside, which contains author
# information.
metadata_parts = metadata_rows.try &.find { |row|
parts = row["metadataParts"]?.try &.as_a
parts && parts.any? { |item2| item2.dig?("text", "commandRuns").try &.as_a }
}.try &.["metadataParts"].as_a
if author_info = metadata_parts.try &.find(&.dig?("text", "commandRuns"))
.try &.["text"]
author = author_info["content"].as_s
ucid = author_info.dig?("commandRuns", 0, "onTap", "innertubeCommand", "browseEndpoint", "browseId")
.try &.as_s
end
length = thumbnail_view_model.try &.dig?("overlays", 0, "thumbnailBottomOverlayViewModel", "badges", 0, "thumbnailBadgeViewModel", "text").try &.as_s
length_seconds = decode_length_seconds(length) if length
live = false
if !length_seconds
live = true
length_seconds = 0
end
videos << PlaylistVideo.new({
title: title || "",
id: video_id || "",
author: author || "",
ucid: ucid || "",
length_seconds: length_seconds,
published: Time.utc,
plid: plid,
live_now: live,
index: index || -1_i64,
})
end
rescue ex
videos << ProblematicTimelineItem.new(parse_exception: ex)
end
return videos
end
def template_playlist(playlist, listen)
html = <<-END_HTML
<h3>
<a href="/playlist?list=#{playlist["playlistId"]}">
#{playlist["title"]}
</a>
</h3>
<div class="pure-menu pure-menu-scrollable playlist-restricted">
<ol class="pure-menu-list">
END_HTML
playlist["videos"].as_a.each do |video|
html += <<-END_HTML
<li class="pure-menu-item" id="#{video["videoId"]}">
<a href="/watch?v=#{video["videoId"]}&list=#{playlist["playlistId"]}&index=#{video["index"]}#{listen ? "&listen=1" : ""}">
<div class="thumbnail">
<img loading="lazy" class="thumbnail" src="/vi/#{video["videoId"]}/mqdefault.jpg" alt="" />
<p class="length">#{recode_length_seconds(video["lengthSeconds"].as_i)}</p>
</div>
<p style="width:100%">#{video["title"]}</p>
<p>
<b style="width:100%">#{video["author"]}</b>
</p>
</a>
</li>
END_HTML
end
html += <<-END_HTML
</ol>
</div>
<hr>
END_HTML
html
end
end

46
src/invidious/routes/account.cr
View File

@ -21,7 +21,7 @@ module Invidious::Routes::Account
user = user.as(User) user = user.as(User)
sid = sid.as(String) sid = sid.as(String)
csrf_token = generate_response(sid, {":change_password"}, HMAC_KEY) csrf_token = Invidious::Helpers::Tokens.generate_response(sid, {":change_password"}, HMAC_KEY)
templated "user/change_password" templated "user/change_password"
end end
@ -43,33 +43,33 @@ module Invidious::Routes::Account
token = env.params.body["csrf_token"]? token = env.params.body["csrf_token"]?
begin begin
validate_request(token, sid, env.request, HMAC_KEY, locale) Invidious::Helpers::Tokens.validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex rescue ex
return error_template(400, ex) return Errors.error_template(400, ex)
end end
password = env.params.body["password"]? password = env.params.body["password"]?
if password.nil? || password.empty? if password.nil? || password.empty?
return error_template(401, "Password is a required field") return Errors.error_template(401, "Password is a required field")
end end
new_passwords = env.params.body.select { |k, _| k.match(/^new_password\[\d+\]$/) }.map { |_, v| v } new_passwords = env.params.body.select { |k, _| k.match(/^new_password\[\d+\]$/) }.map { |_, v| v }
if new_passwords.size <= 1 || new_passwords.uniq.size != 1 if new_passwords.size <= 1 || new_passwords.uniq.size != 1
return error_template(400, "New passwords must match") return Errors.error_template(400, "New passwords must match")
end end
new_password = new_passwords.uniq[0] new_password = new_passwords.uniq[0]
if new_password.empty? if new_password.empty?
return error_template(401, "Password cannot be empty") return Errors.error_template(401, "Password cannot be empty")
end end
if new_password.bytesize > 55 if new_password.bytesize > 55
return error_template(400, "Password cannot be longer than 55 characters") return Errors.error_template(400, "Password cannot be longer than 55 characters")
end end
if !Crypto::Bcrypt::Password.new(user.password.not_nil!).verify(password.byte_slice(0, 55)) if !Crypto::Bcrypt::Password.new(user.password.not_nil!).verify(password.byte_slice(0, 55))
return error_template(401, "Incorrect password") return Errors.error_template(401, "Incorrect password")
end end
new_password = Crypto::Bcrypt::Password.create(new_password, cost: 10) new_password = Crypto::Bcrypt::Password.create(new_password, cost: 10)
@ -96,7 +96,7 @@ module Invidious::Routes::Account
user = user.as(User) user = user.as(User)
sid = sid.as(String) sid = sid.as(String)
csrf_token = generate_response(sid, {":delete_account"}, HMAC_KEY) csrf_token = Invidious::Helpers::Tokens.generate_response(sid, {":delete_account"}, HMAC_KEY)
templated "user/delete_account" templated "user/delete_account"
end end
@ -118,9 +118,9 @@ module Invidious::Routes::Account
token = env.params.body["csrf_token"]? token = env.params.body["csrf_token"]?
begin begin
validate_request(token, sid, env.request, HMAC_KEY, locale) Invidious::Helpers::Tokens.validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex rescue ex
return error_template(400, ex) return Errors.error_template(400, ex)
end end
view_name = "subscriptions_#{sha256(user.email)}" view_name = "subscriptions_#{sha256(user.email)}"
@ -154,7 +154,7 @@ module Invidious::Routes::Account
user = user.as(User) user = user.as(User)
sid = sid.as(String) sid = sid.as(String)
csrf_token = generate_response(sid, {":clear_watch_history"}, HMAC_KEY) csrf_token = Invidious::Helpers::Tokens.generate_response(sid, {":clear_watch_history"}, HMAC_KEY)
templated "user/clear_watch_history" templated "user/clear_watch_history"
end end
@ -176,9 +176,9 @@ module Invidious::Routes::Account
token = env.params.body["csrf_token"]? token = env.params.body["csrf_token"]?
begin begin
validate_request(token, sid, env.request, HMAC_KEY, locale) Invidious::Helpers::Tokens.validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex rescue ex
return error_template(400, ex) return Errors.error_template(400, ex)
end end
Invidious::Database::Users.clear_watch_history(user) Invidious::Database::Users.clear_watch_history(user)
@ -203,7 +203,7 @@ module Invidious::Routes::Account
user = user.as(User) user = user.as(User)
sid = sid.as(String) sid = sid.as(String)
csrf_token = generate_response(sid, {":authorize_token"}, HMAC_KEY) csrf_token = Invidious::Helpers::Tokens.generate_response(sid, {":authorize_token"}, HMAC_KEY)
scopes = env.params.query["scopes"]?.try &.split(",") scopes = env.params.query["scopes"]?.try &.split(",")
scopes ||= [] of String scopes ||= [] of String
@ -235,16 +235,16 @@ module Invidious::Routes::Account
token = env.params.body["csrf_token"]? token = env.params.body["csrf_token"]?
begin begin
validate_request(token, sid, env.request, HMAC_KEY, locale) Invidious::Helpers::Tokens.validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex rescue ex
return error_template(400, ex) return Errors.error_template(400, ex)
end end
scopes = env.params.body.select { |k, _| k.match(/^scopes\[\d+\]$/) }.map { |_, v| v } scopes = env.params.body.select { |k, _| k.match(/^scopes\[\d+\]$/) }.map { |_, v| v }
callback_url = env.params.body["callbackUrl"]? callback_url = env.params.body["callbackUrl"]?
expire = env.params.body["expire"]?.try &.to_i? expire = env.params.body["expire"]?.try &.to_i?
access_token = generate_token(user.email, scopes, expire, HMAC_KEY) access_token = Invidious::Helpers::Tokens.generate_token(user.email, scopes, expire, HMAC_KEY)
if callback_url if callback_url
access_token = URI.encode_www_form(access_token) access_token = URI.encode_www_form(access_token)
@ -310,7 +310,7 @@ module Invidious::Routes::Account
if redirect if redirect
return env.redirect referer return env.redirect referer
else else
return error_json(403, "No such user") return Errors.error_json(403, "No such user")
end end
end end
@ -319,12 +319,12 @@ module Invidious::Routes::Account
token = env.params.body["csrf_token"]? token = env.params.body["csrf_token"]?
begin begin
validate_request(token, sid, env.request, HMAC_KEY, locale) Invidious::Helpers::Tokens.validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex rescue ex
if redirect if redirect
return error_template(400, ex) return Errors.error_template(400, ex)
else else
return error_json(400, ex) return Errors.error_json(400, ex)
end end
end end
@ -333,7 +333,7 @@ module Invidious::Routes::Account
session = env.params.query["session"] session = env.params.query["session"]
Invidious::Database::SessionIDs.delete(sid: session, email: user.email) Invidious::Database::SessionIDs.delete(sid: session, email: user.email)
else else
return error_json(400, "Unsupported action #{action}") return Errors.error_json(400, "Unsupported action #{action}")
end end
if redirect if redirect

64
src/invidious/routes/api/v1/authenticated.cr
View File

@ -8,7 +8,7 @@ module Invidious::Routes::API::V1::Authenticated
# topics = env.params.body["topics"]?.try &.split(",").uniq.first(1000) # topics = env.params.body["topics"]?.try &.split(",").uniq.first(1000)
# topics ||= [] of String # topics ||= [] of String
# Helpers.create_notification_stream(env, topics, connection_channel) # Invidious::Helpers.create_notification_stream(env, topics, connection_channel)
# end # end
def self.get_preferences(env) def self.get_preferences(env)
@ -78,13 +78,13 @@ module Invidious::Routes::API::V1::Authenticated
user = env.get("user").as(User) user = env.get("user").as(User)
if !user.preferences.watch_history if !user.preferences.watch_history
return error_json(409, "Watch history is disabled in preferences.") return Errors.error_json(409, "Watch history is disabled in preferences.")
end end
# Sanity checks # Sanity checks
id = env.params.url["id"] id = env.params.url["id"]
unless validate_video_id(id) unless validate_video_id(id)
return error_json(400, InvalidVideoID.new(id)) return Errors.error_json(400, InvalidVideoID.new(id))
end end
Invidious::Database::Users.mark_watched(user, id) Invidious::Database::Users.mark_watched(user, id)
@ -95,12 +95,12 @@ module Invidious::Routes::API::V1::Authenticated
user = env.get("user").as(User) user = env.get("user").as(User)
if !user.preferences.watch_history if !user.preferences.watch_history
return error_json(409, "Watch history is disabled in preferences.") return Errors.error_json(409, "Watch history is disabled in preferences.")
end end
video_id = env.params.url["id"] video_id = env.params.url["id"]
unless video_id && validate_video_id(video_id) unless video_id && validate_video_id(video_id)
return error_json(400, InvalidVideoID.new(video_id)) return Errors.error_json(400, InvalidVideoID.new(video_id))
end end
Invidious::Database::Users.mark_unwatched(user, video_id) Invidious::Database::Users.mark_unwatched(user, video_id)
@ -127,7 +127,7 @@ module Invidious::Routes::API::V1::Authenticated
page = env.params.query["page"]?.try &.to_i? page = env.params.query["page"]?.try &.to_i?
page ||= 1 page ||= 1
videos, notifications = get_subscription_feed(user, max_results, page) videos, notifications = Invidious::User::Users.get_subscription_feed(user, max_results, page)
JSON.build do |json| JSON.build do |json|
json.object do json.object do
@ -175,7 +175,7 @@ module Invidious::Routes::API::V1::Authenticated
ucid = env.params.url["ucid"] ucid = env.params.url["ucid"]
if !user.subscriptions.includes? ucid if !user.subscriptions.includes? ucid
get_channel(ucid) Invidious::Channels::Channels.get_channel(ucid)
Invidious::Database::Users.subscribe_channel(user, ucid) Invidious::Database::Users.subscribe_channel(user, ucid)
end end
@ -214,19 +214,19 @@ module Invidious::Routes::API::V1::Authenticated
title = env.params.json["title"]?.try &.as(String).delete("<>").byte_slice(0, 150) title = env.params.json["title"]?.try &.as(String).delete("<>").byte_slice(0, 150)
if !title if !title
return error_json(400, "Invalid title.") return Errors.error_json(400, "Invalid title.")
end end
privacy = env.params.json["privacy"]?.try { |p| PlaylistPrivacy.parse(p.as(String).downcase) } privacy = env.params.json["privacy"]?.try { |p| PlaylistPrivacy.parse(p.as(String).downcase) }
if !privacy if !privacy
return error_json(400, "Invalid privacy setting.") return Errors.error_json(400, "Invalid privacy setting.")
end end
if Invidious::Database::Playlists.count_owned_by(user.email) >= 100 if Invidious::Database::Playlists.count_owned_by(user.email) >= 100
return error_json(400, "User cannot have more than 100 playlists.") return Errors.error_json(400, "User cannot have more than 100 playlists.")
end end
playlist = create_playlist(title, privacy, user) playlist = Invidious::Playlists::Playlists.create_playlist(title, privacy, user)
env.response.headers["Location"] = "#{HOST_URL}/api/v1/auth/playlists/#{playlist.id}" env.response.headers["Location"] = "#{HOST_URL}/api/v1/auth/playlists/#{playlist.id}"
env.response.status_code = 201 env.response.status_code = 201
{ {
@ -241,16 +241,16 @@ module Invidious::Routes::API::V1::Authenticated
plid = env.params.url["plid"]? plid = env.params.url["plid"]?
if !plid || plid.empty? if !plid || plid.empty?
return error_json(400, "A playlist ID is required") return Errors.error_json(400, "A playlist ID is required")
end end
playlist = Invidious::Database::Playlists.select(id: plid) playlist = Invidious::Database::Playlists.select(id: plid)
if !playlist || playlist.author != user.email && playlist.privacy.private? if !playlist || playlist.author != user.email && playlist.privacy.private?
return error_json(404, "Playlist does not exist.") return Errors.error_json(404, "Playlist does not exist.")
end end
if playlist.author != user.email if playlist.author != user.email
return error_json(403, "Invalid user") return Errors.error_json(403, "Invalid user")
end end
title = env.params.json["title"].try &.as(String).delete("<>").byte_slice(0, 150) || playlist.title title = env.params.json["title"].try &.as(String).delete("<>").byte_slice(0, 150) || playlist.title
@ -278,11 +278,11 @@ module Invidious::Routes::API::V1::Authenticated
playlist = Invidious::Database::Playlists.select(id: plid) playlist = Invidious::Database::Playlists.select(id: plid)
if !playlist || playlist.author != user.email && playlist.privacy.private? if !playlist || playlist.author != user.email && playlist.privacy.private?
return error_json(404, "Playlist does not exist.") return Errors.error_json(404, "Playlist does not exist.")
end end
if playlist.author != user.email if playlist.author != user.email
return error_json(403, "Invalid user") return Errors.error_json(403, "Invalid user")
end end
Invidious::Database::Playlists.delete(plid) Invidious::Database::Playlists.delete(plid)
@ -298,29 +298,29 @@ module Invidious::Routes::API::V1::Authenticated
playlist = Invidious::Database::Playlists.select(id: plid) playlist = Invidious::Database::Playlists.select(id: plid)
if !playlist || playlist.author != user.email && playlist.privacy.private? if !playlist || playlist.author != user.email && playlist.privacy.private?
return error_json(404, "Playlist does not exist.") return Errors.error_json(404, "Playlist does not exist.")
end end
if playlist.author != user.email if playlist.author != user.email
return error_json(403, "Invalid user") return Errors.error_json(403, "Invalid user")
end end
if playlist.index.size >= CONFIG.playlist_length_limit if playlist.index.size >= CONFIG.playlist_length_limit
return error_json(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos") return Errors.error_json(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos")
end end
video_id = env.params.json["videoId"].try &.as(String) video_id = env.params.json["videoId"].try &.as(String)
# Sanity checks # Sanity checks
unless video_id && validate_video_id(video_id) unless video_id && validate_video_id(video_id)
return error_json(400, InvalidVideoID.new(video_id)) return Errors.error_json(400, InvalidVideoID.new(video_id))
end end
begin begin
video = get_video(video_id) video = get_video(video_id)
rescue ex : NotFoundException rescue ex : NotFoundException
return error_json(404, ex) return Errors.error_json(404, ex)
rescue ex rescue ex
return error_json(500, ex) return Errors.error_json(500, ex)
end end
playlist_video = PlaylistVideo.new({ playlist_video = PlaylistVideo.new({
@ -355,15 +355,15 @@ module Invidious::Routes::API::V1::Authenticated
playlist = Invidious::Database::Playlists.select(id: plid) playlist = Invidious::Database::Playlists.select(id: plid)
if !playlist || playlist.author != user.email && playlist.privacy.private? if !playlist || playlist.author != user.email && playlist.privacy.private?
return error_json(404, "Playlist does not exist.") return Errors.error_json(404, "Playlist does not exist.")
end end
if playlist.author != user.email if playlist.author != user.email
return error_json(403, "Invalid user") return Errors.error_json(403, "Invalid user")
end end
if !playlist.index.includes? index if !playlist.index.includes? index
return error_json(404, "Playlist does not contain index") return Errors.error_json(404, "Playlist does not contain index")
end end
Invidious::Database::PlaylistVideos.delete(index, plid) Invidious::Database::PlaylistVideos.delete(index, plid)
@ -411,7 +411,7 @@ module Invidious::Routes::API::V1::Authenticated
callback_url = env.params.json["callbackUrl"]?.try &.as(String) callback_url = env.params.json["callbackUrl"]?.try &.as(String)
expire = env.params.json["expire"]?.try &.as(Int64) expire = env.params.json["expire"]?.try &.as(Int64)
else else
return error_json(400, "Invalid or missing header 'Content-Type'") return Errors.error_json(400, "Invalid or missing header 'Content-Type'")
end end
if callback_url && callback_url.empty? if callback_url && callback_url.empty?
@ -427,7 +427,7 @@ module Invidious::Routes::API::V1::Authenticated
# Used by template bellow. # Used by template bellow.
# ameba:disable Lint/UselessAssign # ameba:disable Lint/UselessAssign
csrf_token = generate_response(sid, {":authorize_token"}, HMAC_KEY, use_nonce: true) csrf_token = Invidious::Helpers::Tokens.generate_response(sid, {":authorize_token"}, HMAC_KEY, use_nonce: true)
return templated "user/authorize_token" return templated "user/authorize_token"
else else
env.response.content_type = "application/json" env.response.content_type = "application/json"
@ -436,12 +436,12 @@ module Invidious::Routes::API::V1::Authenticated
authorized_scopes = [] of String authorized_scopes = [] of String
scopes.each do |scope| scopes.each do |scope|
if scopes_include_scope(superset_scopes, scope) if Invidious::Helpers::Tokens.scopes_include_scope(superset_scopes, scope)
authorized_scopes << scope authorized_scopes << scope
end end
end end
access_token = generate_token(user.email, authorized_scopes, expire, HMAC_KEY) access_token = Invidious::Helpers::Tokens.generate_token(user.email, authorized_scopes, expire, HMAC_KEY)
if callback_url if callback_url
access_token = URI.encode_www_form(access_token) access_token = URI.encode_www_form(access_token)
@ -473,10 +473,10 @@ module Invidious::Routes::API::V1::Authenticated
# Allow tokens to revoke other tokens with correct scope # Allow tokens to revoke other tokens with correct scope
if session == env.get("session").as(String) if session == env.get("session").as(String)
Invidious::Database::SessionIDs.delete(sid: session) Invidious::Database::SessionIDs.delete(sid: session)
elsif scopes_include_scope(scopes, "GET:tokens") elsif Invidious::Helpers::Tokens.scopes_include_scope(scopes, "GET:tokens")
Invidious::Database::SessionIDs.delete(sid: session) Invidious::Database::SessionIDs.delete(sid: session)
else else
return error_json(400, "Cannot revoke session #{session}") return Errors.error_json(400, "Cannot revoke session #{session}")
end end
env.response.status_code = 204 env.response.status_code = 204
@ -489,6 +489,6 @@ module Invidious::Routes::API::V1::Authenticated
topics = raw_topics.try &.split(",").uniq!.first(1000) topics = raw_topics.try &.split(",").uniq!.first(1000)
topics ||= [] of String topics ||= [] of String
Helpers.create_notification_stream(env, topics, CONNECTION_CHANNEL) Invidious::Helpers.create_notification_stream(env, topics, CONNECTION_CHANNEL)
end end
end end

62
src/invidious/routes/api/v1/channels.cr
View File

@ -3,14 +3,14 @@ module Invidious::Routes::API::V1::Channels
# This sets the `channel` variable, or handles Exceptions. # This sets the `channel` variable, or handles Exceptions.
private macro get_channel private macro get_channel
begin begin
channel = get_about_info(ucid) channel = Invidious::Channels::About.get_about_info(ucid)
rescue ex : ChannelRedirect rescue ex : ChannelRedirect
env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id) env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id)
return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id}) return Errors.error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id})
rescue ex : NotFoundException rescue ex : NotFoundException
return error_json(404, ex) return Errors.error_json(404, ex)
rescue ex rescue ex
return error_json(500, ex) return Errors.error_json(500, ex)
end end
end end
@ -29,8 +29,8 @@ module Invidious::Routes::API::V1::Channels
if channel.is_age_gated if channel.is_age_gated
begin begin
playlist = get_playlist(channel.ucid.sub("UC", "UULF")) playlist = Invidious::Playlists::Playlists.get_playlist(channel.ucid.sub("UC", "UULF"))
videos = get_playlist_videos(playlist, offset: 0) videos = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: 0)
rescue ex : InfoException rescue ex : InfoException
# playlist doesnt exist. # playlist doesnt exist.
videos = [] of PlaylistVideo videos = [] of PlaylistVideo
@ -39,7 +39,7 @@ module Invidious::Routes::API::V1::Channels
begin begin
videos, _ = Channel::Tabs.get_videos(channel, sort_by: sort_by) videos, _ = Channel::Tabs.get_videos(channel, sort_by: sort_by)
rescue ex rescue ex
return error_json(500, ex) return Errors.error_json(500, ex)
end end
end end
@ -96,7 +96,7 @@ module Invidious::Routes::API::V1::Channels
json.field "autoGenerated", channel.auto_generated json.field "autoGenerated", channel.auto_generated
json.field "ageGated", channel.is_age_gated json.field "ageGated", channel.is_age_gated
json.field "isFamilyFriendly", channel.is_family_friendly json.field "isFamilyFriendly", channel.is_family_friendly
json.field "description", Helpers.html_to_content(channel.description_html) json.field "description", Invidious::Helpers.html_to_content(channel.description_html)
json.field "descriptionHtml", channel.description_html json.field "descriptionHtml", channel.description_html
json.field "allowedRegions", channel.allowed_regions json.field "allowedRegions", channel.allowed_regions
@ -117,7 +117,7 @@ module Invidious::Routes::API::V1::Channels
json.array do json.array do
# Fetch related channels # Fetch related channels
begin begin
related_channels, _ = fetch_related_channels(channel) related_channels, _ = Invidious::Channels::About.fetch_related_channels(channel)
rescue ex rescue ex
related_channels = [] of SearchChannel related_channels = [] of SearchChannel
end end
@ -155,8 +155,8 @@ module Invidious::Routes::API::V1::Channels
if channel.is_age_gated if channel.is_age_gated
begin begin
playlist = get_playlist(channel.ucid.sub("UC", "UULF")) playlist = Invidious::Playlists::Playlists.get_playlist(channel.ucid.sub("UC", "UULF"))
videos = get_playlist_videos(playlist, offset: 0) videos = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: 0)
rescue ex : InfoException rescue ex : InfoException
# playlist doesnt exist. # playlist doesnt exist.
videos = [] of PlaylistVideo videos = [] of PlaylistVideo
@ -168,7 +168,7 @@ module Invidious::Routes::API::V1::Channels
channel, continuation: continuation, sort_by: sort_by channel, continuation: continuation, sort_by: sort_by
) )
rescue ex rescue ex
return error_json(500, ex) return Errors.error_json(500, ex)
end end
end end
@ -201,8 +201,8 @@ module Invidious::Routes::API::V1::Channels
if channel.is_age_gated if channel.is_age_gated
begin begin
playlist = get_playlist(channel.ucid.sub("UC", "UUSH")) playlist = Invidious::Playlists::Playlists.get_playlist(channel.ucid.sub("UC", "UUSH"))
videos = get_playlist_videos(playlist, offset: 0) videos = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: 0)
rescue ex : InfoException rescue ex : InfoException
# playlist doesnt exist. # playlist doesnt exist.
videos = [] of PlaylistVideo videos = [] of PlaylistVideo
@ -214,7 +214,7 @@ module Invidious::Routes::API::V1::Channels
channel, continuation: continuation, sort_by: sort_by channel, continuation: continuation, sort_by: sort_by
) )
rescue ex rescue ex
return error_json(500, ex) return Errors.error_json(500, ex)
end end
end end
@ -247,8 +247,8 @@ module Invidious::Routes::API::V1::Channels
if channel.is_age_gated if channel.is_age_gated
begin begin
playlist = get_playlist(channel.ucid.sub("UC", "UULV")) playlist = Invidious::Playlists::Playlists.get_playlist(channel.ucid.sub("UC", "UULV"))
videos = get_playlist_videos(playlist, offset: 0) videos = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: 0)
rescue ex : InfoException rescue ex : InfoException
# playlist doesnt exist. # playlist doesnt exist.
videos = [] of PlaylistVideo videos = [] of PlaylistVideo
@ -260,7 +260,7 @@ module Invidious::Routes::API::V1::Channels
channel, continuation: continuation, sort_by: sort_by channel, continuation: continuation, sort_by: sort_by
) )
rescue ex rescue ex
return error_json(500, ex) return Errors.error_json(500, ex)
end end
end end
@ -292,7 +292,7 @@ module Invidious::Routes::API::V1::Channels
channel = nil # Make the compiler happy channel = nil # Make the compiler happy
get_channel() get_channel()
items, next_continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by) items, next_continuation = Invidious::Channels::Playlists.fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
JSON.build do |json| JSON.build do |json|
json.object do json.object do
@ -321,7 +321,7 @@ module Invidious::Routes::API::V1::Channels
channel = nil # Make the compiler happy channel = nil # Make the compiler happy
get_channel() get_channel()
items, next_continuation = fetch_channel_podcasts(channel.ucid, channel.author, continuation) items, next_continuation = Invidious::Channels::Playlists.fetch_channel_podcasts(channel.ucid, channel.author, continuation)
JSON.build do |json| JSON.build do |json|
json.object do json.object do
@ -350,7 +350,7 @@ module Invidious::Routes::API::V1::Channels
channel = nil # Make the compiler happy channel = nil # Make the compiler happy
get_channel() get_channel()
items, next_continuation = fetch_channel_releases(channel.ucid, channel.author, continuation) items, next_continuation = Invidious::Channels::Playlists.fetch_channel_releases(channel.ucid, channel.author, continuation)
JSON.build do |json| JSON.build do |json|
json.object do json.object do
@ -379,7 +379,7 @@ module Invidious::Routes::API::V1::Channels
channel = nil # Make the compiler happy channel = nil # Make the compiler happy
get_channel() get_channel()
items, next_continuation = fetch_channel_courses(channel.ucid, channel.author, continuation) items, next_continuation = Invidious::Channels::Playlists.fetch_channel_courses(channel.ucid, channel.author, continuation)
JSON.build do |json| JSON.build do |json|
json.object do json.object do
@ -413,9 +413,9 @@ module Invidious::Routes::API::V1::Channels
# sort_by = env.params.query["sort_by"]?.try &.downcase # sort_by = env.params.query["sort_by"]?.try &.downcase
begin begin
fetch_channel_community(ucid, continuation, locale, format, thin_mode) Invidious::Channels::Community.fetch_channel_community(ucid, continuation, locale, format, thin_mode)
rescue ex rescue ex
return error_json(500, ex) return Errors.error_json(500, ex)
end end
end end
@ -434,16 +434,16 @@ module Invidious::Routes::API::V1::Channels
if ucid.nil? if ucid.nil?
response = YoutubeAPI.resolve_url("https://www.youtube.com/post/#{id}") response = YoutubeAPI.resolve_url("https://www.youtube.com/post/#{id}")
return error_json(400, "Invalid post ID") if response["error"]? return Errors.error_json(400, "Invalid post ID") if response["error"]?
ucid = decode_ucid_from_post_protobuf(response.dig("endpoint", "browseEndpoint", "params").as_s) ucid = Invidious::Channels::Community.decode_ucid_from_post_protobuf(response.dig("endpoint", "browseEndpoint", "params").as_s)
else else
ucid = ucid.to_s ucid = ucid.to_s
end end
begin begin
fetch_channel_community_post(ucid, id, locale, format, thin_mode) Invidious::Channels::Community.fetch_channel_community_post(ucid, id, locale, format, thin_mode)
rescue ex rescue ex
return error_json(500, ex) return Errors.error_json(500, ex)
end end
end end
@ -487,9 +487,9 @@ module Invidious::Routes::API::V1::Channels
continuation = env.params.query["continuation"]? continuation = env.params.query["continuation"]?
begin begin
items, next_continuation = fetch_related_channels(channel, continuation) items, next_continuation = Invidious::Channels::About.fetch_related_channels(channel, continuation)
rescue ex rescue ex
return error_json(500, ex) return Errors.error_json(500, ex)
end end
JSON.build do |json| JSON.build do |json|
@ -520,7 +520,7 @@ module Invidious::Routes::API::V1::Channels
begin begin
search_results = query.process search_results = query.process
rescue ex rescue ex
return error_json(400, ex) return Errors.error_json(400, ex)
end end
JSON.build do |json| JSON.build do |json|

4
src/invidious/routes/api/v1/feeds.cr
View File

@ -8,9 +8,9 @@ module Invidious::Routes::API::V1::Feeds
trending_type = env.params.query["type"]? trending_type = env.params.query["type"]?
begin begin
trending, _ = fetch_trending(trending_type, region, locale) trending, _ = Invidious::Feeds::Trending.fetch(trending_type, region, locale)
rescue ex rescue ex
return error_json(500, ex) return Errors.error_json(500, ex)
end end
videos = JSON.build do |json| videos = JSON.build do |json|

18
src/invidious/routes/api/v1/misc.cr
View File

@ -50,16 +50,16 @@ module Invidious::Routes::API::V1::Misc
end end
begin begin
playlist = get_playlist(plid) playlist = Invidious::Playlists::Playlists.get_playlist(plid)
rescue ex : InfoException rescue ex : InfoException
return error_json(404, ex) return Errors.error_json(404, ex)
rescue ex rescue ex
return error_json(404, "Playlist does not exist.") return Errors.error_json(404, "Playlist does not exist.")
end end
user = env.get?("user").try &.as(User) user = env.get?("user").try &.as(User)
if !playlist || playlist.privacy.private? && playlist.author != user.try &.email if !playlist || playlist.privacy.private? && playlist.author != user.try &.email
return error_json(404, "Playlist does not exist.") return Errors.error_json(404, "Playlist does not exist.")
end end
# includes into the playlist a maximum of 20 videos, before the offset # includes into the playlist a maximum of 20 videos, before the offset
@ -88,7 +88,7 @@ module Invidious::Routes::API::V1::Misc
end end
if format == "html" if format == "html"
playlist_html = template_playlist(json_response, listen) playlist_html = Invidious::Playlists::Playlists.template_playlist(json_response, listen)
index, next_video = json_response["videos"].as_a.skip(1 + lookback).select { |video| !video["author"].as_s.empty? }[0]?.try { |v| {v["index"], v["videoId"]} } || {nil, nil} index, next_video = json_response["videos"].as_a.skip(1 + lookback).select { |video| !video["author"].as_s.empty? }[0]?.try { |v| {v["index"], v["videoId"]} } || {nil, nil}
response = { response = {
@ -127,7 +127,7 @@ module Invidious::Routes::API::V1::Misc
mix.videos = mix.videos[index..-1] mix.videos = mix.videos[index..-1]
rescue ex rescue ex
return error_json(500, ex) return Errors.error_json(500, ex)
end end
response = JSON.build do |json| response = JSON.build do |json|
@ -178,14 +178,14 @@ module Invidious::Routes::API::V1::Misc
env.response.content_type = "application/json" env.response.content_type = "application/json"
url = env.params.query["url"]? url = env.params.query["url"]?
return error_json(400, "Missing URL to resolve") if !url return Errors.error_json(400, "Missing URL to resolve") if !url
begin begin
resolved_url = YoutubeAPI.resolve_url(url.as(String)) resolved_url = YoutubeAPI.resolve_url(url.as(String))
endpoint = resolved_url["endpoint"] endpoint = resolved_url["endpoint"]
page_type = endpoint.dig?("commandMetadata", "webCommandMetadata", "webPageType").try &.as_s || "" page_type = endpoint.dig?("commandMetadata", "webCommandMetadata", "webPageType").try &.as_s || ""
if page_type == "WEB_PAGE_TYPE_UNKNOWN" if page_type == "WEB_PAGE_TYPE_UNKNOWN"
return error_json(400, "Unknown url") return Errors.error_json(400, "Unknown url")
end end
sub_endpoint = endpoint["watchEndpoint"]? || endpoint["browseEndpoint"]? || endpoint sub_endpoint = endpoint["watchEndpoint"]? || endpoint["browseEndpoint"]? || endpoint
@ -204,7 +204,7 @@ module Invidious::Routes::API::V1::Misc
post_id = nil post_id = nil
end end
rescue ex rescue ex
return error_json(500, ex) return Errors.error_json(500, ex)
end end
JSON.build do |json| JSON.build do |json|
json.object do json.object do

8
src/invidious/routes/api/v1/search.cr
View File

@ -10,7 +10,7 @@ module Invidious::Routes::API::V1::Search
begin begin
search_results = query.process search_results = query.process
rescue ex rescue ex
return error_json(400, ex) return Errors.error_json(400, ex)
end end
JSON.build do |json| JSON.build do |json|
@ -53,7 +53,7 @@ module Invidious::Routes::API::V1::Search
end end
end end
rescue ex rescue ex
return error_json(500, ex) return Errors.error_json(500, ex)
end end
end end
@ -67,9 +67,9 @@ module Invidious::Routes::API::V1::Search
env.response.content_type = "application/json" env.response.content_type = "application/json"
begin begin
results = Invidious::Hashtag.fetch(hashtag, page, region) results = Invidious::Search::Hashtag.fetch_hashtag(hashtag, page, region)
rescue ex rescue ex
return error_json(400, ex) return Errors.error_json(400, ex)
end end
JSON.build do |json| JSON.build do |json|

36
src/invidious/routes/api/v1/videos.cr
View File

@ -16,9 +16,9 @@ module Invidious::Routes::API::V1::Videos
begin begin
video = get_video(id, region: region) video = get_video(id, region: region)
rescue ex : NotFoundException rescue ex : NotFoundException
return error_json(404, ex) return Errors.error_json(404, ex)
rescue ex rescue ex
return error_json(500, ex) return Errors.error_json(500, ex)
end end
return JSON.build do |json| return JSON.build do |json|
@ -34,7 +34,7 @@ module Invidious::Routes::API::V1::Videos
# Sanity checks # Sanity checks
unless validate_video_id(id) unless validate_video_id(id)
return error_json(400, InvalidVideoID.new(id)) return Errors.error_json(400, InvalidVideoID.new(id))
end end
# See https://github.com/ytdl-org/youtube-dl/blob/6ab30ff50bf6bd0585927cb73c7421bef184f87a/youtube_dl/extractor/youtube.py#L1354 # See https://github.com/ytdl-org/youtube-dl/blob/6ab30ff50bf6bd0585927cb73c7421bef184f87a/youtube_dl/extractor/youtube.py#L1354
@ -301,7 +301,7 @@ module Invidious::Routes::API::V1::Videos
annotations = response.body annotations = response.body
Helpers.cache_annotation(video_id, annotations) Invidious::Helpers.cache_annotation(video_id, annotations)
end end
else # "youtube" else # "youtube"
response = YT_POOL.client &.get("/annotations_invideo?video_id=#{video_id}") response = YT_POOL.client &.get("/annotations_invideo?video_id=#{video_id}")
@ -354,9 +354,9 @@ module Invidious::Routes::API::V1::Videos
begin begin
comments = Comments.fetch_youtube(id, continuation, format, locale, thin_mode, region, sort_by: sort_by) comments = Comments.fetch_youtube(id, continuation, format, locale, thin_mode, region, sort_by: sort_by)
rescue ex : NotFoundException rescue ex : NotFoundException
return error_json(404, ex) return Errors.error_json(404, ex)
rescue ex rescue ex
return error_json(500, ex) return Errors.error_json(500, ex)
end end
return comments return comments
@ -371,7 +371,7 @@ module Invidious::Routes::API::V1::Videos
end end
if !reddit_thread || !comments if !reddit_thread || !comments
return error_json(404, "No reddit threads found") return Errors.error_json(404, "No reddit threads found")
end end
if format == "json" if format == "json"
@ -404,10 +404,10 @@ module Invidious::Routes::API::V1::Videos
proxy = {"1", "true"}.any? &.== env.params.query["local"]? proxy = {"1", "true"}.any? &.== env.params.query["local"]?
response = YoutubeAPI.resolve_url("https://www.youtube.com/clip/#{clip_id}") response = YoutubeAPI.resolve_url("https://www.youtube.com/clip/#{clip_id}")
return error_json(400, "Invalid clip ID") if response["error"]? return Errors.error_json(400, "Invalid clip ID") if response["error"]?
video_id = response.dig?("endpoint", "watchEndpoint", "videoId").try &.as_s video_id = response.dig?("endpoint", "watchEndpoint", "videoId").try &.as_s
return error_json(400, "Invalid clip ID") if video_id.nil? return Errors.error_json(400, "Invalid clip ID") if video_id.nil?
start_time = nil start_time = nil
end_time = nil end_time = nil
@ -420,9 +420,9 @@ module Invidious::Routes::API::V1::Videos
begin begin
video = get_video(video_id, region: region) video = get_video(video_id, region: region)
rescue ex : NotFoundException rescue ex : NotFoundException
return error_json(404, ex) return Errors.error_json(404, ex)
rescue ex rescue ex
return error_json(500, ex) return Errors.error_json(500, ex)
end end
return JSON.build do |json| return JSON.build do |json|
@ -454,9 +454,9 @@ module Invidious::Routes::API::V1::Videos
begin begin
video = get_video(id) video = get_video(id)
rescue ex : NotFoundException rescue ex : NotFoundException
return error_json(404, ex) return Errors.error_json(404, ex)
rescue ex rescue ex
return error_json(500, ex) return Errors.error_json(500, ex)
end end
response = JSON.build do |json| response = JSON.build do |json|
@ -494,14 +494,14 @@ module Invidious::Routes::API::V1::Videos
begin begin
video = get_video(id) video = get_video(id)
rescue ex : NotFoundException rescue ex : NotFoundException
return error_json(404, ex) return Errors.error_json(404, ex)
rescue ex rescue ex
return error_json(500, ex) return Errors.error_json(500, ex)
end end
target_transcript = video.captions.select(&.name.== label) target_transcript = video.captions.select(&.name.== label)
if target_transcript.empty? if target_transcript.empty?
return error_json(404, NotFoundException.new("Requested transcript does not exist")) return Errors.error_json(404, NotFoundException.new("Requested transcript does not exist"))
else else
target_transcript = target_transcript[0] target_transcript = target_transcript[0]
lang, auto_generated = target_transcript.language_code, target_transcript.auto_generated lang, auto_generated = target_transcript.language_code, target_transcript.auto_generated
@ -515,9 +515,9 @@ module Invidious::Routes::API::V1::Videos
YoutubeAPI.get_transcript(params), lang, auto_generated YoutubeAPI.get_transcript(params), lang, auto_generated
) )
rescue ex : NotFoundException rescue ex : NotFoundException
return error_json(404, ex) return Errors.error_json(404, ex)
rescue ex rescue ex
return error_json(500, ex) return Errors.error_json(500, ex)
end end
return transcript.to_json return transcript.to_json

6
src/invidious/routes/before_all.cr
View File

@ -1,4 +1,6 @@
module Invidious::Routes::BeforeAll module Invidious::Routes::BeforeAll
extend self
struct CompanionCSP struct CompanionCSP
property companion_urls : String = "" property companion_urls : String = ""
@ -89,7 +91,7 @@ module Invidious::Routes::BeforeAll
if email = Database::SessionIDs.select_email(sid) if email = Database::SessionIDs.select_email(sid)
user = Database::Users.select!(email: email) user = Database::Users.select!(email: email)
csrf_token = generate_response(sid, { csrf_token = Invidious::Helpers::Tokens.generate_response(sid, {
":authorize_token", ":authorize_token",
":playlist_ajax", ":playlist_ajax",
":signout", ":signout",
@ -107,7 +109,7 @@ module Invidious::Routes::BeforeAll
end end
end end
dark_mode = convert_theme(env.params.query["dark_mode"]?) || preferences.dark_mode.to_s dark_mode = Invidious::User::Converters.convert_theme(env.params.query["dark_mode"]?) || preferences.dark_mode.to_s
thin_mode = env.params.query["thin_mode"]? thin_mode = env.params.query["thin_mode"]?
thin_mode = (thin_mode == "true") || preferences.thin_mode thin_mode = (thin_mode == "true") || preferences.thin_mode
locale = env.params.query["hl"]? || preferences.locale locale = env.params.query["hl"]? || preferences.locale

46
src/invidious/routes/channels.cr
View File

@ -23,7 +23,7 @@ module Invidious::Routes::Channels
sort_by ||= "last" sort_by ||= "last"
sort_options = {"last", "oldest", "newest"} sort_options = {"last", "oldest", "newest"}
items, next_continuation = fetch_channel_playlists( items, next_continuation = Invidious::Channels::Playlists.fetch_channel_playlists(
channel.ucid, channel.author, continuation, sort_by channel.ucid, channel.author, continuation, sort_by
) )
@ -42,8 +42,8 @@ module Invidious::Routes::Channels
sort_by = "" sort_by = ""
sort_options = [] of String sort_options = [] of String
begin begin
playlist = get_playlist(channel.ucid.sub("UC", "UULF")) playlist = Invidious::Playlists::Playlists.get_playlist(channel.ucid.sub("UC", "UULF"))
items = get_playlist_videos(playlist, offset: 0) items = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: 0)
rescue ex : InfoException rescue ex : InfoException
# playlist doesnt exist. # playlist doesnt exist.
items = [] of PlaylistVideo items = [] of PlaylistVideo
@ -77,8 +77,8 @@ module Invidious::Routes::Channels
sort_by = "" sort_by = ""
sort_options = [] of String sort_options = [] of String
begin begin
playlist = get_playlist(channel.ucid.sub("UC", "UUSH")) playlist = Invidious::Playlists::Playlists.get_playlist(channel.ucid.sub("UC", "UUSH"))
items = get_playlist_videos(playlist, offset: 0) items = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: 0)
rescue ex : InfoException rescue ex : InfoException
# playlist doesnt exist. # playlist doesnt exist.
items = [] of PlaylistVideo items = [] of PlaylistVideo
@ -112,8 +112,8 @@ module Invidious::Routes::Channels
sort_by = "" sort_by = ""
sort_options = [] of String sort_options = [] of String
begin begin
playlist = get_playlist(channel.ucid.sub("UC", "UULV")) playlist = Invidious::Playlists::Playlists.get_playlist(channel.ucid.sub("UC", "UULV"))
items = get_playlist_videos(playlist, offset: 0) items = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: 0)
rescue ex : InfoException rescue ex : InfoException
# playlist doesnt exist. # playlist doesnt exist.
items = [] of PlaylistVideo items = [] of PlaylistVideo
@ -146,7 +146,7 @@ module Invidious::Routes::Channels
return env.redirect "/channel/#{channel.ucid}" return env.redirect "/channel/#{channel.ucid}"
end end
items, next_continuation = fetch_channel_playlists( items, next_continuation = Invidious::Channels::Playlists.fetch_channel_playlists(
channel.ucid, channel.author, continuation, (sort_by || "last") channel.ucid, channel.author, continuation, (sort_by || "last")
) )
@ -166,7 +166,7 @@ module Invidious::Routes::Channels
sort_by = "" sort_by = ""
sort_options = [] of String sort_options = [] of String
items, next_continuation = fetch_channel_podcasts( items, next_continuation = Invidious::Channels::Playlists.fetch_channel_podcasts(
channel.ucid, channel.author, continuation channel.ucid, channel.author, continuation
) )
@ -186,7 +186,7 @@ module Invidious::Routes::Channels
sort_by = "" sort_by = ""
sort_options = [] of String sort_options = [] of String
items, next_continuation = fetch_channel_releases( items, next_continuation = Invidious::Channels::Playlists.fetch_channel_releases(
channel.ucid, channel.author, continuation channel.ucid, channel.author, continuation
) )
@ -206,7 +206,7 @@ module Invidious::Routes::Channels
sort_by = "" sort_by = ""
sort_options = [] of String sort_options = [] of String
items, next_continuation = fetch_channel_courses( items, next_continuation = Invidious::Channels::Playlists.fetch_channel_courses(
channel.ucid, channel.author, continuation channel.ucid, channel.author, continuation
) )
@ -247,7 +247,7 @@ module Invidious::Routes::Channels
sort_options = [] of String sort_options = [] of String
begin begin
items = JSON.parse(fetch_channel_community(ucid, continuation, locale, "json", thin_mode)) items = JSON.parse(Invidious::Channels::Community.fetch_channel_community(ucid, continuation, locale, "json", thin_mode))
rescue ex : InfoException rescue ex : InfoException
env.response.status_code = 500 env.response.status_code = 500
error_message = ex.message error_message = ex.message
@ -255,7 +255,7 @@ module Invidious::Routes::Channels
env.response.status_code = 404 env.response.status_code = 404
error_message = ex.message error_message = ex.message
rescue ex rescue ex
return error_template(500, ex) return Errors.error_template(500, ex)
end end
templated "community" templated "community"
@ -280,14 +280,14 @@ module Invidious::Routes::Channels
if !ucid.nil? if !ucid.nil?
ucid = ucid.to_s ucid = ucid.to_s
post_response = fetch_channel_community_post(ucid, id, locale, "json", thin_mode) post_response = Invidious::Channels::Community.fetch_channel_community_post(ucid, id, locale, "json", thin_mode)
else else
# resolve the url to get the author's UCID # resolve the url to get the author's UCID
response = YoutubeAPI.resolve_url("https://www.youtube.com/post/#{id}") response = YoutubeAPI.resolve_url("https://www.youtube.com/post/#{id}")
return error_template(400, "Invalid post ID") if response["error"]? return Errors.error_template(400, "Invalid post ID") if response["error"]?
ucid = decode_ucid_from_post_protobuf(response.dig("endpoint", "browseEndpoint", "params").as_s) ucid = Invidious::Channels::Community.decode_ucid_from_post_protobuf(response.dig("endpoint", "browseEndpoint", "params").as_s)
post_response = fetch_channel_community_post(ucid, id, locale, "json", thin_mode) post_response = Invidious::Channels::Community.fetch_channel_community_post(ucid, id, locale, "json", thin_mode)
end end
post_response = JSON.parse(post_response) post_response = JSON.parse(post_response)
@ -309,7 +309,7 @@ module Invidious::Routes::Channels
return env.redirect "/channel/#{channel.ucid}" return env.redirect "/channel/#{channel.ucid}"
end end
items, next_continuation = fetch_related_channels(channel, continuation) items, next_continuation = Invidious::Channels::About.fetch_related_channels(channel, continuation)
# Featured/related channels can't be sorted # Featured/related channels can't be sorted
sort_options = [] of String sort_options = [] of String
@ -354,7 +354,7 @@ module Invidious::Routes::Channels
resolved_url = YoutubeAPI.resolve_url("https://www.youtube.com#{env.request.path}#{yt_url_params.size > 0 ? "?#{yt_url_params}" : ""}") resolved_url = YoutubeAPI.resolve_url("https://www.youtube.com#{env.request.path}#{yt_url_params.size > 0 ? "?#{yt_url_params}" : ""}")
ucid = resolved_url["endpoint"]["browseEndpoint"]["browseId"] ucid = resolved_url["endpoint"]["browseEndpoint"]["browseId"]
rescue ex : InfoException | KeyError rescue ex : InfoException | KeyError
return error_template(404, I18n.translate(locale, "This channel does not exist.")) return Errors.error_template(404, I18n.translate(locale, "This channel does not exist."))
end end
selected_tab = env.params.url["tab"]? selected_tab = env.params.url["tab"]?
@ -380,7 +380,7 @@ module Invidious::Routes::Channels
user = env.params.query["user"]? user = env.params.query["user"]?
if !user if !user
return error_template(404, "This channel does not exist.") return Errors.error_template(404, "This channel does not exist.")
else else
env.redirect "/user/#{user}#{uri_params}" env.redirect "/user/#{user}#{uri_params}"
end end
@ -433,13 +433,13 @@ module Invidious::Routes::Channels
continuation = env.params.query["continuation"]? continuation = env.params.query["continuation"]?
begin begin
channel = get_about_info(ucid) channel = Invidious::Channels::About.get_about_info(ucid)
rescue ex : ChannelRedirect rescue ex : ChannelRedirect
return env.redirect env.request.resource.gsub(ucid, ex.channel_id) return env.redirect env.request.resource.gsub(ucid, ex.channel_id)
rescue ex : NotFoundException rescue ex : NotFoundException
return error_template(404, ex) return Errors.error_template(404, ex)
rescue ex rescue ex
return error_template(500, ex) return Errors.error_template(500, ex)
end end
env.set "search", "channel:#{ucid} " env.set "search", "channel:#{ucid} "

22
src/invidious/routes/embed.cr
View File

@ -5,9 +5,9 @@ module Invidious::Routes::Embed
locale = env.get("preferences").as(Preferences).locale locale = env.get("preferences").as(Preferences).locale
if plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "") if plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "")
begin begin
playlist = get_playlist(plid) playlist = Invidious::Playlists::Playlists.get_playlist(plid)
offset = env.params.query["index"]?.try &.to_i? || 0 offset = env.params.query["index"]?.try &.to_i? || 0
videos = get_playlist_videos(playlist, offset: offset) videos = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: offset)
if videos.empty? if videos.empty?
url = "/playlist?list=#{plid}" url = "/playlist?list=#{plid}"
raise NotFoundException.new(I18n.translate(locale, "error_video_not_in_playlist", url)) raise NotFoundException.new(I18n.translate(locale, "error_video_not_in_playlist", url))
@ -15,9 +15,9 @@ module Invidious::Routes::Embed
first_playlist_video = videos[0].as(PlaylistVideo) first_playlist_video = videos[0].as(PlaylistVideo)
rescue ex : NotFoundException rescue ex : NotFoundException
return error_template(404, ex) return Errors.error_template(404, ex)
rescue ex rescue ex
return error_template(500, ex) return Errors.error_template(500, ex)
end end
url = "/embed/#{first_playlist_video.id}?#{env.params.query}" url = "/embed/#{first_playlist_video.id}?#{env.params.query}"
@ -66,9 +66,9 @@ module Invidious::Routes::Embed
if plid if plid
begin begin
playlist = get_playlist(plid) playlist = Invidious::Playlists::Playlists.get_playlist(plid)
offset = env.params.query["index"]?.try &.to_i? || 0 offset = env.params.query["index"]?.try &.to_i? || 0
videos = get_playlist_videos(playlist, offset: offset) videos = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: offset)
if videos.empty? if videos.empty?
url = "/playlist?list=#{plid}" url = "/playlist?list=#{plid}"
raise NotFoundException.new(I18n.translate(locale, "error_video_not_in_playlist", url)) raise NotFoundException.new(I18n.translate(locale, "error_video_not_in_playlist", url))
@ -76,9 +76,9 @@ module Invidious::Routes::Embed
first_playlist_video = videos[0].as(PlaylistVideo) first_playlist_video = videos[0].as(PlaylistVideo)
rescue ex : NotFoundException rescue ex : NotFoundException
return error_template(404, ex) return Errors.error_template(404, ex)
rescue ex rescue ex
return error_template(500, ex) return Errors.error_template(500, ex)
end end
url = "/embed/#{first_playlist_video.id}" url = "/embed/#{first_playlist_video.id}"
@ -101,7 +101,7 @@ module Invidious::Routes::Embed
env.params.query.delete_all("channel") env.params.query.delete_all("channel")
if !video_id || video_id == "live_stream" if !video_id || video_id == "live_stream"
return error_template(500, "Video is unavailable.") return Errors.error_template(500, "Video is unavailable.")
end end
url = "/embed/#{video_id}" url = "/embed/#{video_id}"
@ -135,9 +135,9 @@ module Invidious::Routes::Embed
begin begin
video = get_video(id, region: params.region) video = get_video(id, region: params.region)
rescue ex : NotFoundException rescue ex : NotFoundException
return error_template(404, ex) return Errors.error_template(404, ex)
rescue ex rescue ex
return error_template(500, ex) return Errors.error_template(500, ex)
end end
if preferences.annotations_subscribed && if preferences.annotations_subscribed &&

18
src/invidious/routes/feeds.cr
View File

@ -53,9 +53,9 @@ module Invidious::Routes::Feeds
region ||= preferences.region region ||= preferences.region
begin begin
trending, plid = fetch_trending(trending_type, region, locale) trending, plid = Invidious::Feeds::Trending.fetch(trending_type, region, locale)
rescue ex rescue ex
return error_template(500, ex) return Errors.error_template(500, ex)
end end
templated "feeds/trending" templated "feeds/trending"
@ -91,7 +91,7 @@ module Invidious::Routes::Feeds
page = env.params.query["page"]?.try &.to_i? page = env.params.query["page"]?.try &.to_i?
page ||= 1 page ||= 1
videos, notifications = get_subscription_feed(user, max_results, page) videos, notifications = Invidious::User::Users.get_subscription_feed(user, max_results, page)
if CONFIG.enable_user_notifications if CONFIG.enable_user_notifications
# "updated" here is used for delivering new notifications, so if # "updated" here is used for delivering new notifications, so if
@ -150,7 +150,7 @@ module Invidious::Routes::Feeds
if env.params.url["ucid"].matches?(/^[\w-]+$/) if env.params.url["ucid"].matches?(/^[\w-]+$/)
ucid = env.params.url["ucid"] ucid = env.params.url["ucid"]
else else
return error_atom(400, InfoException.new("Invalid channel ucid provided.")) return Errors.error_atom(400, InfoException.new("Invalid channel ucid provided."))
end end
params = HTTP::Params.parse(env.params.query["params"]? || "") params = HTTP::Params.parse(env.params.query["params"]? || "")
@ -162,7 +162,7 @@ module Invidious::Routes::Feeds
} }
response = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}") response = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}")
return error_atom(404, NotFoundException.new("Channel does not exist.")) if response.status_code == 404 return Errors.error_atom(404, NotFoundException.new("Channel does not exist.")) if response.status_code == 404
rss = XML.parse(response.body) rss = XML.parse(response.body)
videos = rss.xpath_nodes("//default:feed/default:entry", namespaces).map do |entry| videos = rss.xpath_nodes("//default:feed/default:entry", namespaces).map do |entry|
@ -250,7 +250,7 @@ module Invidious::Routes::Feeds
params = HTTP::Params.parse(env.params.query["params"]? || "") params = HTTP::Params.parse(env.params.query["params"]? || "")
videos, notifications = get_subscription_feed(user, max_results, page) videos, notifications = Invidious::User::Users.get_subscription_feed(user, max_results, page)
XML.build(indent: " ", encoding: "UTF-8") do |xml| XML.build(indent: " ", encoding: "UTF-8") do |xml|
xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015", xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015",
@ -281,11 +281,11 @@ module Invidious::Routes::Feeds
if plid.starts_with? "IV" if plid.starts_with? "IV"
if playlist = Invidious::Database::Playlists.select(id: plid) if playlist = Invidious::Database::Playlists.select(id: plid)
videos = get_playlist_videos(playlist, offset: 0) videos = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: 0)
user = env.get?("user").try &.as(User) user = env.get?("user").try &.as(User)
if !playlist || playlist.privacy.private? && playlist.author != user.try &.email if !playlist || playlist.privacy.private? && playlist.author != user.try &.email
return error_atom(404, "Playlist does not exist.") return Errors.error_atom(404, "Playlist does not exist.")
end end
return XML.build(indent: " ", encoding: "UTF-8") do |xml| return XML.build(indent: " ", encoding: "UTF-8") do |xml|
@ -317,7 +317,7 @@ module Invidious::Routes::Feeds
end end
response = YT_POOL.client &.get("/feeds/videos.xml?playlist_id=#{plid}") response = YT_POOL.client &.get("/feeds/videos.xml?playlist_id=#{plid}")
return error_atom(404, NotFoundException.new("Playlist does not exist.")) if response.status_code == 404 return Errors.error_atom(404, NotFoundException.new("Playlist does not exist.")) if response.status_code == 404
document = XML.parse(response.body) document = XML.parse(response.body)
document.xpath_nodes(%q(//*[@href]|//*[@url])).each do |node| document.xpath_nodes(%q(//*[@href]|//*[@url])).each do |node|

4
src/invidious/routes/images.cr
View File

@ -106,7 +106,7 @@ module Invidious::Routes::Images
break break
end end
Helpers.proxy_file(response, env) Invidious::Helpers.proxy_file(response, env)
end end
rescue ex rescue ex
end end
@ -158,6 +158,6 @@ module Invidious::Routes::Images
return env.response.headers.delete("Transfer-Encoding") return env.response.headers.delete("Transfer-Encoding")
end end
return Helpers.proxy_file(response, env) return Invidious::Helpers.proxy_file(response, env)
end end
end end

30
src/invidious/routes/login.cr
View File

@ -11,7 +11,7 @@ module Invidious::Routes::Login
return env.redirect referer if user return env.redirect referer if user
if !CONFIG.login_enabled if !CONFIG.login_enabled
return error_template(400, "Login has been disabled by administrator.") return Errors.error_template(400, "Login has been disabled by administrator.")
end end
email = nil email = nil
@ -31,7 +31,7 @@ module Invidious::Routes::Login
referer = get_referer(env, "/feed/subscriptions") referer = get_referer(env, "/feed/subscriptions")
if !CONFIG.login_enabled if !CONFIG.login_enabled
return error_template(403, "Login has been disabled by administrator.") return Errors.error_template(403, "Login has been disabled by administrator.")
end end
# https://stackoverflow.com/a/574698 # https://stackoverflow.com/a/574698
@ -44,11 +44,11 @@ module Invidious::Routes::Login
case account_type case account_type
when "invidious" when "invidious"
if email.nil? || email.empty? if email.nil? || email.empty?
return error_template(401, "User ID is a required field") return Errors.error_template(401, "User ID is a required field")
end end
if password.nil? || password.empty? if password.nil? || password.empty?
return error_template(401, "Password is a required field") return Errors.error_template(401, "Password is a required field")
end end
user = Invidious::Database::Users.select(email: email) user = Invidious::Database::Users.select(email: email)
@ -64,7 +64,7 @@ module Invidious::Routes::Login
env.response.cookies["SID"] = Invidious::User::Cookies.sid(CONFIG.domain, sid) env.response.cookies["SID"] = Invidious::User::Cookies.sid(CONFIG.domain, sid)
end end
else else
return error_template(401, "Wrong username or password") return Errors.error_template(401, "Wrong username or password")
end end
# Since this user has already registered, we don't want to overwrite their preferences # Since this user has already registered, we don't want to overwrite their preferences
@ -75,16 +75,16 @@ module Invidious::Routes::Login
end end
else else
if !CONFIG.registration_enabled if !CONFIG.registration_enabled
return error_template(400, "Registration has been disabled by administrator.") return Errors.error_template(400, "Registration has been disabled by administrator.")
end end
if password.empty? if password.empty?
return error_template(401, "Password cannot be empty") return Errors.error_template(401, "Password cannot be empty")
end end
# See https://security.stackexchange.com/a/39851 # See https://security.stackexchange.com/a/39851
if password.bytesize > 55 if password.bytesize > 55
return error_template(400, "Password cannot be longer than 55 characters") return Errors.error_template(400, "Password cannot be longer than 55 characters")
end end
password = password.byte_slice(0, 55) password = password.byte_slice(0, 55)
@ -102,11 +102,11 @@ module Invidious::Routes::Login
answer = OpenSSL::HMAC.hexdigest(:sha256, HMAC_KEY, answer) answer = OpenSSL::HMAC.hexdigest(:sha256, HMAC_KEY, answer)
begin begin
validate_request(tokens[0], answer, env.request, HMAC_KEY, locale) Invidious::Helpers::Tokens.validate_request(tokens[0], answer, env.request, HMAC_KEY, locale)
rescue ex : InfoException rescue ex : InfoException
return error_template(400, InfoException.new("Erroneous CAPTCHA")) return Errors.error_template(400, InfoException.new("Erroneous CAPTCHA"))
rescue ex rescue ex
return error_template(400, ex) return Errors.error_template(400, ex)
end end
else else
return templated "user/login" return templated "user/login"
@ -114,7 +114,7 @@ module Invidious::Routes::Login
end end
sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32)) sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
user, sid = create_user(sid, email, password) user, sid = Invidious::User::Users.create_user(sid, email, password)
if language_header = env.request.headers["Accept-Language"]? if language_header = env.request.headers["Accept-Language"]?
if language = ANG.language_negotiator.best(language_header, I18n::LOCALES.keys) if language = ANG.language_negotiator.best(language_header, I18n::LOCALES.keys)
@ -126,7 +126,7 @@ module Invidious::Routes::Login
Invidious::Database::SessionIDs.insert(sid, email) Invidious::Database::SessionIDs.insert(sid, email)
view_name = "subscriptions_#{sha256(user.email)}" view_name = "subscriptions_#{sha256(user.email)}"
PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}") PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{Invidious::User::Users::MATERIALIZED_VIEW_SQL.call(user.email)}")
if alt = CONFIG.alternative_domains.index(host) if alt = CONFIG.alternative_domains.index(host)
env.response.cookies["SID"] = Invidious::User::Cookies.sid(CONFIG.alternative_domains[alt], sid) env.response.cookies["SID"] = Invidious::User::Cookies.sid(CONFIG.alternative_domains[alt], sid)
@ -166,9 +166,9 @@ module Invidious::Routes::Login
token = env.params.body["csrf_token"]? token = env.params.body["csrf_token"]?
begin begin
validate_request(token, sid, env.request, HMAC_KEY, locale) Invidious::Helpers::Tokens.validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex rescue ex
return error_template(400, ex) return Errors.error_template(400, ex)
end end
Invidious::Database::SessionIDs.delete(sid: sid) Invidious::Database::SessionIDs.delete(sid: sid)

2
src/invidious/routes/notifications.cr
View File

@ -18,7 +18,7 @@ module Invidious::Routes::Notifications
if redirect if redirect
return env.redirect referer return env.redirect referer
else else
return error_json(403, "No such user") return Errors.error_json(403, "No such user")
end end
end end

86
src/invidious/routes/playlists.cr
View File

@ -12,7 +12,7 @@ module Invidious::Routes::Playlists
user = user.as(User) user = user.as(User)
sid = sid.as(String) sid = sid.as(String)
csrf_token = generate_response(sid, {":create_playlist"}, HMAC_KEY) csrf_token = Invidious::Helpers::Tokens.generate_response(sid, {":create_playlist"}, HMAC_KEY)
templated "create_playlist" templated "create_playlist"
end end
@ -31,26 +31,26 @@ module Invidious::Routes::Playlists
token = env.params.body["csrf_token"]? token = env.params.body["csrf_token"]?
begin begin
validate_request(token, sid, env.request, HMAC_KEY, locale) Invidious::Helpers::Tokens.validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex rescue ex
return error_template(400, ex) return Errors.error_template(400, ex)
end end
title = env.params.body["title"]?.try &.as(String) title = env.params.body["title"]?.try &.as(String)
if !title || title.empty? if !title || title.empty?
return error_template(400, "Title cannot be empty.") return Errors.error_template(400, "Title cannot be empty.")
end end
privacy = PlaylistPrivacy.parse?(env.params.body["privacy"]?.try &.as(String) || "") privacy = PlaylistPrivacy.parse?(env.params.body["privacy"]?.try &.as(String) || "")
if !privacy if !privacy
return error_template(400, "Invalid privacy setting.") return Errors.error_template(400, "Invalid privacy setting.")
end end
if Invidious::Database::Playlists.count_owned_by(user.email) >= 100 if Invidious::Database::Playlists.count_owned_by(user.email) >= 100
return error_template(400, "User cannot have more than 100 playlists.") return Errors.error_template(400, "User cannot have more than 100 playlists.")
end end
playlist = create_playlist(title, privacy, user) playlist = Invidious::Playlists::Playlists.create_playlist(title, privacy, user)
env.redirect "/playlist?list=#{playlist.id}" env.redirect "/playlist?list=#{playlist.id}"
end end
@ -67,13 +67,13 @@ module Invidious::Routes::Playlists
playlist_id = env.params.query["list"] playlist_id = env.params.query["list"]
begin begin
playlist = get_playlist(playlist_id) playlist = Invidious::Playlists::Playlists.get_playlist(playlist_id)
rescue ex : NotFoundException rescue ex : NotFoundException
return error_template(404, ex) return Errors.error_template(404, ex)
rescue ex rescue ex
return error_template(500, ex) return Errors.error_template(500, ex)
end end
subscribe_playlist(user, playlist) Invidious::Playlists::Playlists.subscribe_playlist(user, playlist)
env.redirect "/playlist?list=#{playlist.id}" env.redirect "/playlist?list=#{playlist.id}"
end end
@ -92,7 +92,7 @@ module Invidious::Routes::Playlists
plid = env.params.query["list"]? plid = env.params.query["list"]?
if !plid || plid.empty? if !plid || plid.empty?
return error_template(400, "A playlist ID is required") return Errors.error_template(400, "A playlist ID is required")
end end
playlist = Invidious::Database::Playlists.select(id: plid) playlist = Invidious::Database::Playlists.select(id: plid)
@ -100,7 +100,7 @@ module Invidious::Routes::Playlists
return env.redirect referer return env.redirect referer
end end
csrf_token = generate_response(sid, {":delete_playlist"}, HMAC_KEY) csrf_token = Invidious::Helpers::Tokens.generate_response(sid, {":delete_playlist"}, HMAC_KEY)
templated "delete_playlist" templated "delete_playlist"
end end
@ -122,9 +122,9 @@ module Invidious::Routes::Playlists
token = env.params.body["csrf_token"]? token = env.params.body["csrf_token"]?
begin begin
validate_request(token, sid, env.request, HMAC_KEY, locale) Invidious::Helpers::Tokens.validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex rescue ex
return error_template(400, ex) return Errors.error_template(400, ex)
end end
playlist = Invidious::Database::Playlists.select(id: plid) playlist = Invidious::Database::Playlists.select(id: plid)
@ -163,12 +163,12 @@ module Invidious::Routes::Playlists
end end
begin begin
items = get_playlist_videos(playlist, offset: (page - 1) * 100) items = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: (page - 1) * 100)
rescue ex rescue ex
items = [] of PlaylistVideo items = [] of PlaylistVideo
end end
csrf_token = generate_response(sid, {":edit_playlist"}, HMAC_KEY) csrf_token = Invidious::Helpers::Tokens.generate_response(sid, {":edit_playlist"}, HMAC_KEY)
# Pagination # Pagination
page_nav_html = Frontend::Pagination.nav_numeric(locale, page_nav_html = Frontend::Pagination.nav_numeric(locale,
@ -197,9 +197,9 @@ module Invidious::Routes::Playlists
token = env.params.body["csrf_token"]? token = env.params.body["csrf_token"]?
begin begin
validate_request(token, sid, env.request, HMAC_KEY, locale) Invidious::Helpers::Tokens.validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex rescue ex
return error_template(400, ex) return Errors.error_template(400, ex)
end end
playlist = Invidious::Database::Playlists.select(id: plid) playlist = Invidious::Database::Playlists.select(id: plid)
@ -286,7 +286,7 @@ module Invidious::Routes::Playlists
if redirect if redirect
return env.redirect referer return env.redirect referer
else else
return error_json(403, "No such user") return Errors.error_json(403, "No such user")
end end
end end
@ -295,26 +295,26 @@ module Invidious::Routes::Playlists
token = env.params.body["csrf_token"]? token = env.params.body["csrf_token"]?
begin begin
validate_request(token, sid, env.request, HMAC_KEY, locale) Invidious::Helpers::Tokens.validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex rescue ex
if redirect if redirect
return error_template(400, ex) return Errors.error_template(400, ex)
else else
return error_json(400, ex) return Errors.error_json(400, ex)
end end
end end
begin begin
playlist_id = env.params.query["playlist_id"] playlist_id = env.params.query["playlist_id"]
playlist = get_playlist(playlist_id).as(InvidiousPlaylist) playlist = Invidious::Playlists::Playlists.get_playlist(playlist_id).as(InvidiousPlaylist)
raise "Invalid user" if playlist.author != user.email raise "Invalid user" if playlist.author != user.email
rescue ex : NotFoundException rescue ex : NotFoundException
return error_json(404, ex) return Errors.error_json(404, ex)
rescue ex rescue ex
if redirect if redirect
return error_template(400, ex) return Errors.error_template(400, ex)
else else
return error_json(400, ex) return Errors.error_json(400, ex)
end end
end end
@ -322,9 +322,9 @@ module Invidious::Routes::Playlists
when "add_video" when "add_video"
if playlist.index.size >= CONFIG.playlist_length_limit if playlist.index.size >= CONFIG.playlist_length_limit
if redirect if redirect
return error_template(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos") return Errors.error_template(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos")
else else
return error_json(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos") return Errors.error_json(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos")
end end
end end
@ -333,12 +333,12 @@ module Invidious::Routes::Playlists
begin begin
video = get_video(video_id) video = get_video(video_id)
rescue ex : NotFoundException rescue ex : NotFoundException
return error_json(404, ex) return Errors.error_json(404, ex)
rescue ex rescue ex
if redirect if redirect
return error_template(500, ex) return Errors.error_template(500, ex)
else else
return error_json(500, ex) return Errors.error_json(500, ex)
end end
end end
@ -359,7 +359,7 @@ module Invidious::Routes::Playlists
when "remove_video" when "remove_video"
index = env.params.query["set_video_id"].to_i64? index = env.params.query["set_video_id"].to_i64?
if index.nil? || !playlist.index.includes? index if index.nil? || !playlist.index.includes? index
return error_json(404, "Playlist does not contain index") return Errors.error_json(404, "Playlist does not contain index")
end end
Invidious::Database::PlaylistVideos.delete(index, playlist_id) Invidious::Database::PlaylistVideos.delete(index, playlist_id)
@ -367,9 +367,9 @@ module Invidious::Routes::Playlists
when "move_video_before" when "move_video_before"
# TODO: Playlist stub # TODO: Playlist stub
when nil when nil
return error_json(400, "Missing action") return Errors.error_json(400, "Missing action")
else else
return error_json(400, "Unsupported action #{action}") return Errors.error_json(400, "Unsupported action #{action}")
end end
if redirect if redirect
@ -399,11 +399,11 @@ module Invidious::Routes::Playlists
end end
begin begin
playlist = get_playlist(plid) playlist = Invidious::Playlists::Playlists.get_playlist(plid)
rescue ex : NotFoundException rescue ex : NotFoundException
return error_template(404, ex) return Errors.error_template(404, ex)
rescue ex rescue ex
return error_template(500, ex) return Errors.error_template(500, ex)
end end
if playlist.is_a? InvidiousPlaylist if playlist.is_a? InvidiousPlaylist
@ -419,17 +419,17 @@ module Invidious::Routes::Playlists
end end
if playlist.privacy == PlaylistPrivacy::Private && playlist.author != user.try &.email if playlist.privacy == PlaylistPrivacy::Private && playlist.author != user.try &.email
return error_template(403, "This playlist is private.") return Errors.error_template(403, "This playlist is private.")
end end
begin begin
if playlist.is_a? InvidiousPlaylist if playlist.is_a? InvidiousPlaylist
items = get_playlist_videos(playlist, offset: (page - 1) * 100) items = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: (page - 1) * 100)
else else
items = get_playlist_videos(playlist, offset: (page - 1) * 200) items = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: (page - 1) * 200)
end end
rescue ex rescue ex
return error_template(500, "Error encountered while retrieving playlist videos.<br>#{ex.message}") return Errors.error_template(500, "Error encountered while retrieving playlist videos.<br>#{ex.message}")
end end
if playlist.author == user.try &.email if playlist.author == user.try &.email
@ -460,7 +460,7 @@ module Invidious::Routes::Playlists
begin begin
mix = fetch_mix(rdid, continuation, locale: locale) mix = fetch_mix(rdid, continuation, locale: locale)
rescue ex rescue ex
return error_template(500, ex) return Errors.error_template(500, ex)
end end
templated "mix" templated "mix"

8
src/invidious/routes/preferences.cr
View File

@ -329,7 +329,7 @@ module Invidious::Routes::PreferencesRoute
if !success if !success
haltf(env, status_code: 415, haltf(env, status_code: 415,
response: error_template(415, "Invalid subscription file uploaded") response: Errors.error_template(415, "Invalid subscription file uploaded")
) )
end end
when "import_youtube_pl" when "import_youtube_pl"
@ -338,7 +338,7 @@ module Invidious::Routes::PreferencesRoute
if !success if !success
haltf(env, status_code: 415, haltf(env, status_code: 415,
response: error_template(415, "Invalid playlist file uploaded") response: Errors.error_template(415, "Invalid playlist file uploaded")
) )
end end
when "import_youtube_wh" when "import_youtube_wh"
@ -347,7 +347,7 @@ module Invidious::Routes::PreferencesRoute
if !success if !success
haltf(env, status_code: 415, haltf(env, status_code: 415,
response: error_template(415, "Invalid watch history file uploaded") response: Errors.error_template(415, "Invalid watch history file uploaded")
) )
end end
when "import_freetube" when "import_freetube"
@ -359,7 +359,7 @@ module Invidious::Routes::PreferencesRoute
if !success if !success
haltf(env, status_code: 415, haltf(env, status_code: 415,
response: error_template(415, "Uploaded file is too large") response: Errors.error_template(415, "Uploaded file is too large")
) )
end end
else nil # Ignore else nil # Ignore

10
src/invidious/routes/search.cr
View File

@ -71,9 +71,9 @@ module Invidious::Routes::Search
items = query.process items = query.process
end end
rescue ex : ChannelSearchException rescue ex : ChannelSearchException
return error_template(404, "Unable to find channel with id of '#{HTML.escape(ex.channel)}'. Are you sure that's an actual channel id? It should look like 'UC4QobU6STFB0P71PMvOGN5A'.") return Errors.error_template(404, "Unable to find channel with id of '#{HTML.escape(ex.channel)}'. Are you sure that's an actual channel id? It should look like 'UC4QobU6STFB0P71PMvOGN5A'.")
rescue ex rescue ex
return error_template(500, ex) return Errors.error_template(500, ex)
end end
redirect_url = Invidious::Frontend::Misc.redirect_url(env) redirect_url = Invidious::Frontend::Misc.redirect_url(env)
@ -100,7 +100,7 @@ module Invidious::Routes::Search
hashtag = env.params.url["hashtag"]? hashtag = env.params.url["hashtag"]?
if hashtag.nil? || hashtag.empty? if hashtag.nil? || hashtag.empty?
return error_template(400, "Invalid request") return Errors.error_template(400, "Invalid request")
end end
page = env.params.query["page"]? page = env.params.query["page"]?
@ -112,9 +112,9 @@ module Invidious::Routes::Search
end end
begin begin
items = Invidious::Hashtag.fetch(hashtag, page) items = Invidious::Search::Hashtag.fetch_hashtag(hashtag, page)
rescue ex rescue ex
return error_template(500, ex) return Errors.error_template(500, ex)
end end
# Pagination # Pagination

12
src/invidious/routes/subscriptions.cr
View File

@ -14,7 +14,7 @@ module Invidious::Routes::Subscriptions
if redirect if redirect
return env.redirect referer return env.redirect referer
else else
return error_json(403, "No such user") return Errors.error_json(403, "No such user")
end end
end end
@ -23,12 +23,12 @@ module Invidious::Routes::Subscriptions
token = env.params.body["csrf_token"]? token = env.params.body["csrf_token"]?
begin begin
validate_request(token, sid, env.request, HMAC_KEY, locale) Invidious::Helpers::Tokens.validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex rescue ex
if redirect if redirect
return error_template(400, ex) return Errors.error_template(400, ex)
else else
return error_json(400, ex) return Errors.error_json(400, ex)
end end
end end
@ -38,13 +38,13 @@ module Invidious::Routes::Subscriptions
case action = env.params.query["action"]? case action = env.params.query["action"]?
when "create_subscription_to_channel" when "create_subscription_to_channel"
if !user.subscriptions.includes? channel_id if !user.subscriptions.includes? channel_id
get_channel(channel_id) Invidious::Channels::Channels.get_channel(channel_id)
Invidious::Database::Users.subscribe_channel(user, channel_id) Invidious::Database::Users.subscribe_channel(user, channel_id)
end end
when "remove_subscriptions" when "remove_subscriptions"
Invidious::Database::Users.unsubscribe_channel(user, channel_id) Invidious::Database::Users.unsubscribe_channel(user, channel_id)
else else
return error_json(400, "Unsupported action #{action}") return Errors.error_json(400, "Unsupported action #{action}")
end end
if redirect if redirect

20
src/invidious/routes/video_playback.cr
View File

@ -24,7 +24,7 @@ module Invidious::Routes::VideoPlayback
# Sanity check, to avoid being used as an open proxy # Sanity check, to avoid being used as an open proxy
if !host.matches?(/[\w-]+\.(?:googlevideo|c\.youtube)\.com/) if !host.matches?(/[\w-]+\.(?:googlevideo|c\.youtube)\.com/)
return error_template(400, "Invalid \"host\" parameter.") return Errors.error_template(400, "Invalid \"host\" parameter.")
end end
host = "https://#{host}" host = "https://#{host}"
@ -83,7 +83,7 @@ module Invidious::Routes::VideoPlayback
# Remove the Range header added previously. # Remove the Range header added previously.
headers.delete("Range") if range_header.nil? headers.delete("Range") if range_header.nil?
playback_statistics = Helpers.get_playback_statistic playback_statistics = Invidious::Helpers.get_playback_statistic
playback_statistics["totalRequests"] += 1 playback_statistics["totalRequests"] += 1
if response.status_code >= 400 if response.status_code >= 400
@ -95,7 +95,7 @@ module Invidious::Routes::VideoPlayback
if url.includes? "&file=seg.ts" if url.includes? "&file=seg.ts"
if CONFIG.disabled?("livestreams") if CONFIG.disabled?("livestreams")
return error_template(403, "Administrator has disabled this endpoint.") return Errors.error_template(403, "Administrator has disabled this endpoint.")
end end
begin begin
@ -120,7 +120,7 @@ module Invidious::Routes::VideoPlayback
else else
if query_params["title"]? && CONFIG.disabled?("downloads") || if query_params["title"]? && CONFIG.disabled?("downloads") ||
CONFIG.disabled?("dash") CONFIG.disabled?("dash")
return error_template(403, "Administrator has disabled this endpoint.") return Errors.error_template(403, "Administrator has disabled this endpoint.")
end end
content_length = nil content_length = nil
@ -195,7 +195,7 @@ module Invidious::Routes::VideoPlayback
end end
end end
Helpers.proxy_file(resp, env) Invidious::Helpers.proxy_file(resp, env)
end end
rescue ex rescue ex
if ex.message != "Error reading socket: Connection reset by peer" if ex.message != "Error reading socket: Connection reset by peer"
@ -269,11 +269,11 @@ module Invidious::Routes::VideoPlayback
# Sanity checks # Sanity checks
unless id && validate_video_id(id) unless id && validate_video_id(id)
return error_template(400, InvalidVideoID.new(id)) return Errors.error_template(400, InvalidVideoID.new(id))
end end
if !itag.nil? && (itag <= 0 || itag >= 1000) if !itag.nil? && (itag <= 0 || itag >= 1000)
return error_template(400, "Invalid itag") return Errors.error_template(400, "Invalid itag")
end end
region = env.params.query["region"]? region = env.params.query["region"]?
@ -282,15 +282,15 @@ module Invidious::Routes::VideoPlayback
title = env.params.query["title"]? title = env.params.query["title"]?
if title && CONFIG.disabled?("downloads") if title && CONFIG.disabled?("downloads")
return error_template(403, "Administrator has disabled this endpoint.") return Errors.error_template(403, "Administrator has disabled this endpoint.")
end end
begin begin
video = get_video(id, region: region) video = get_video(id, region: region)
rescue ex : NotFoundException rescue ex : NotFoundException
return error_template(404, ex) return Errors.error_template(404, ex)
rescue ex rescue ex
return error_template(500, ex) return Errors.error_template(500, ex)
end end
if itag.nil? if itag.nil?

32
src/invidious/routes/watch.cr
View File

@ -15,11 +15,11 @@ module Invidious::Routes::Watch
id = env.params.query["v"] id = env.params.query["v"]
if env.params.query["v"].empty? if env.params.query["v"].empty?
return error_template(400, "Invalid parameters.") return Errors.error_template(400, "Invalid parameters.")
end end
unless validate_video_id(id) unless validate_video_id(id)
return error_template(400, InvalidVideoID.new(id)) return Errors.error_template(400, InvalidVideoID.new(id))
end end
else else
return env.redirect "/" return env.redirect "/"
@ -48,10 +48,10 @@ module Invidious::Routes::Watch
video = get_video(id, region: params.region) video = get_video(id, region: params.region)
rescue ex : NotFoundException rescue ex : NotFoundException
LOGGER.error("get_video not found: #{id} : #{ex.message}") LOGGER.error("get_video not found: #{id} : #{ex.message}")
return error_template(404, ex) return Errors.error_template(404, ex)
rescue ex rescue ex
LOGGER.error("get_video: #{id} : #{ex.message}") LOGGER.error("get_video: #{id} : #{ex.message}")
return error_template(500, ex) return Errors.error_template(500, ex)
end end
if preferences.annotations_subscribed && if preferences.annotations_subscribed &&
@ -220,7 +220,7 @@ module Invidious::Routes::Watch
if redirect if redirect
return env.redirect referer return env.redirect referer
else else
return error_json(403, "No such user") return Errors.error_json(403, "No such user")
end end
end end
@ -235,12 +235,12 @@ module Invidious::Routes::Watch
end end
begin begin
validate_request(token, sid, env.request, HMAC_KEY, locale) Invidious::Helpers::Tokens.validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex rescue ex
if redirect if redirect
return error_template(400, ex) return Errors.error_template(400, ex)
else else
return error_json(400, ex) return Errors.error_json(400, ex)
end end
end end
@ -250,7 +250,7 @@ module Invidious::Routes::Watch
when "mark_unwatched" when "mark_unwatched"
Invidious::Database::Users.mark_unwatched(user, id) Invidious::Database::Users.mark_unwatched(user, id)
else else
return error_json(400, "Unsupported action #{action}") return Errors.error_json(400, "Unsupported action #{action}")
end end
if redirect if redirect
@ -264,10 +264,10 @@ module Invidious::Routes::Watch
def self.clip(env) def self.clip(env)
clip_id = env.params.url["clip"]? clip_id = env.params.url["clip"]?
return error_template(400, "A clip ID is required") if !clip_id return Errors.error_template(400, "A clip ID is required") if !clip_id
response = YoutubeAPI.resolve_url("https://www.youtube.com/clip/#{clip_id}") response = YoutubeAPI.resolve_url("https://www.youtube.com/clip/#{clip_id}")
return error_template(400, "Invalid clip ID") if response["error"]? return Errors.error_template(400, "Invalid clip ID") if response["error"]?
if video_id = response.dig?("endpoint", "watchEndpoint", "videoId") if video_id = response.dig?("endpoint", "watchEndpoint", "videoId")
if params = response.dig?("endpoint", "watchEndpoint", "params").try &.as_s if params = response.dig?("endpoint", "watchEndpoint", "params").try &.as_s
@ -278,16 +278,16 @@ module Invidious::Routes::Watch
return env.redirect "/watch?v=#{video_id}&#{env.params.query}" return env.redirect "/watch?v=#{video_id}&#{env.params.query}"
else else
return error_template(404, "The requested clip doesn't exist") return Errors.error_template(404, "The requested clip doesn't exist")
end end
end end
def self.download(env) def self.download(env)
if CONFIG.disabled?("downloads") if CONFIG.disabled?("downloads")
return error_template(403, "Administrator has disabled this endpoint.") return Errors.error_template(403, "Administrator has disabled this endpoint.")
end end
if CONFIG.invidious_companion.present? if CONFIG.invidious_companion.present?
return error_template(403, "Downloads should be routed through Companion when present") return Errors.error_template(403, "Downloads should be routed through Companion when present")
end end
title = env.params.body["title"]? || "" title = env.params.body["title"]? || ""
@ -295,7 +295,7 @@ module Invidious::Routes::Watch
selection = env.params.body["download_widget"]? selection = env.params.body["download_widget"]?
if title.empty? || video_id.empty? || selection.nil? if title.empty? || video_id.empty? || selection.nil?
return error_template(400, "Missing form data") return Errors.error_template(400, "Missing form data")
end end
download_widget = JSON.parse(selection) download_widget = JSON.parse(selection)
@ -326,7 +326,7 @@ module Invidious::Routes::Watch
return Invidious::Routes::VideoPlayback.latest_version(env) return Invidious::Routes::VideoPlayback.latest_version(env)
else else
return error_template(400, "Invalid label or itag") return Errors.error_template(400, "Invalid label or itag")
end end
end end
end end

56
src/invidious/search/ctoken.cr
View File

@ -1,32 +1,36 @@
def produce_channel_search_continuation(ucid, query, page) module Invidious::Search::CToken
if page <= 1 extend self
idx = 0_i64
else
idx = 30_i64 * (page - 1)
end
object = { def produce_channel_search_continuation(ucid, query, page)
"80226972:embedded" => { if page <= 1
"2:string" => ucid, idx = 0_i64
"3:base64" => { else
"2:string" => "search", idx = 30_i64 * (page - 1)
"6:varint" => 1_i64, end
"7:varint" => 1_i64,
"12:varint" => 1_i64, object = {
"15:base64" => { "80226972:embedded" => {
"3:varint" => idx, "2:string" => ucid,
"3:base64" => {
"2:string" => "search",
"6:varint" => 1_i64,
"7:varint" => 1_i64,
"12:varint" => 1_i64,
"15:base64" => {
"3:varint" => idx,
},
"23:varint" => 0_i64,
}, },
"23:varint" => 0_i64, "11:string" => query,
"35:string" => "browse-feed#{ucid}search",
}, },
"11:string" => query, }
"35:string" => "browse-feed#{ucid}search",
},
}
continuation = object.try { |i| Protodec::Any.cast_json(i) } continuation = object.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) } .try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) } .try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) } .try { |i| URI.encode_www_form(i) }
return continuation return continuation
end
end end

4
src/invidious/hashtag.cr → src/invidious/search/hashtag.cr
View File

@ -1,7 +1,7 @@
module Invidious::Hashtag module Invidious::Search::Hashtag
extend self extend self
def fetch(hashtag : String, page : Int, region : String? = nil) : Array(SearchItem) def fetch_hashtag(hashtag : String, page : Int, region : String? = nil) : Array(SearchItem)
cursor = (page - 1) * 60 cursor = (page - 1) * 60
ctoken = generate_continuation(hashtag, cursor) ctoken = generate_continuation(hashtag, cursor)

74
src/invidious/search/processors.cr
View File

@ -1,45 +1,44 @@
module Invidious::Search module Invidious::Search::Processors
module Processors extend self
extend self
# Regular search (`/search` endpoint) # Regular search (`/search` endpoint)
def regular(query : Query) : Array(SearchItem) def regular(query : Query) : Array(SearchItem)
search_params = query.filters.to_yt_params(page: query.page) search_params = query.filters.to_yt_params(page: query.page)
client_config = YoutubeAPI::ClientConfig.new(region: query.region) client_config = YoutubeAPI::ClientConfig.new(region: query.region)
initial_data = YoutubeAPI.search(query.text, search_params, client_config: client_config) initial_data = YoutubeAPI.search(query.text, search_params, client_config: client_config)
items, _ = extract_items(initial_data) items, _ = extract_items(initial_data)
return items.reject!(Category) return items.reject!(Category)
end
# Search a youtube channel
# TODO: clean code, and rely more on YoutubeAPI
def channel(query : Query) : Array(SearchItem)
response = YT_POOL.client &.get("/channel/#{query.channel}")
if response.status_code == 404
response = YT_POOL.client &.get("/user/#{query.channel}")
response = YT_POOL.client &.get("/c/#{query.channel}") if response.status_code == 404
initial_data = Invidious::Helpers.extract_initial_data(response.body)
ucid = initial_data.dig?("header", "c4TabbedHeaderRenderer", "channelId").try(&.as_s?)
raise ChannelSearchException.new(query.channel) if !ucid
else
ucid = query.channel
end end
# Search a youtube channel continuation = Invidious::Search::CToken.produce_channel_search_continuation(ucid, query.text, query.page)
# TODO: clean code, and rely more on YoutubeAPI response_json = YoutubeAPI.browse(continuation)
def channel(query : Query) : Array(SearchItem)
response = YT_POOL.client &.get("/channel/#{query.channel}")
if response.status_code == 404 items, _ = extract_items(response_json, "", ucid)
response = YT_POOL.client &.get("/user/#{query.channel}") return items.reject!(Category)
response = YT_POOL.client &.get("/c/#{query.channel}") if response.status_code == 404 end
initial_data = Helpers.extract_initial_data(response.body)
ucid = initial_data.dig?("header", "c4TabbedHeaderRenderer", "channelId").try(&.as_s?)
raise ChannelSearchException.new(query.channel) if !ucid
else
ucid = query.channel
end
continuation = produce_channel_search_continuation(ucid, query.text, query.page) # Search inside of user subscriptions
response_json = YoutubeAPI.browse(continuation) def subscriptions(query : Query, user : Invidious::User) : Array(ChannelVideo)
view_name = "subscriptions_#{sha256(user.email)}"
items, _ = extract_items(response_json, "", ucid) return PG_DB.query_all("
return items.reject!(Category)
end
# Search inside of user subscriptions
def subscriptions(query : Query, user : Invidious::User) : Array(ChannelVideo)
view_name = "subscriptions_#{sha256(user.email)}"
return PG_DB.query_all("
SELECT id,title,published,updated,ucid,author,length_seconds SELECT id,title,published,updated,ucid,author,length_seconds
FROM ( FROM (
SELECT *, SELECT *,
@ -48,9 +47,8 @@ module Invidious::Search
as document as document
FROM #{view_name} FROM #{view_name}
) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;", ) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;",
query.text, (query.page - 1) * 20, query.text, (query.page - 1) * 20,
as: ChannelVideo as: ChannelVideo
) )
end
end end
end end

48
src/invidious/trending.cr
View File

@ -1,48 +0,0 @@
def fetch_trending(trending_type, region, locale)
region ||= "US"
region = region.upcase
plid = nil
browse_id = ""
case trending_type.try &.downcase
when "gaming"
browse_id = "UCOpNcN46UbXVtpKMrmU4Abg"
params = "Egh0cmVuZGluZw%3D%3D"
when "livestreams"
browse_id = "UC4R8DWoMoI7CAwX8_LjQHig"
params = "EgdsaXZldGFikgEDCKEK"
else
# Livestreams is the default one as Youtube removed
# the aggregated trending page
# https://github.com/iv-org/invidious/issues/5397#issuecomment-3218928458
browse_id = "UC4R8DWoMoI7CAwX8_LjQHig"
params = "EgdsaXZldGFikgEDCKEK"
end
client_config = YoutubeAPI::ClientConfig.new(region: region)
initial_data = YoutubeAPI.browse(browse_id, params: params, client_config: client_config)
items, _ = extract_items(initial_data)
extracted = [] of SearchItem
deduplicate = items.size > 1
items.each do |itm|
if itm.is_a?(Category)
# Ignore the smaller categories, as they generally contain a sponsored
# channel, which brings a lot of noise on the trending page.
# See: https://github.com/iv-org/invidious/issues/2989
next if (itm.contents.size < 24 && deduplicate)
extracted.concat itm.contents.select(SearchItem)
else
extracted << itm
end
end
# Deduplicate items before returning results
return extracted.select(SearchVideo | ProblematicTimelineItem).uniq!(&.id), plid
end

2
src/invidious/user/captcha.cr
View File

@ -55,7 +55,7 @@ struct Invidious::User
return { return {
question: image, question: image,
tokens: {generate_response(answer, {":login"}, key, use_nonce: true)}, tokens: {Invidious::Helpers::Tokens.generate_response(answer, {":login"}, key, use_nonce: true)},
} }
end end
end end

24
src/invidious/user/converters.cr
View File

@ -1,12 +1,16 @@
def convert_theme(theme) module Invidious::User::Converters
case theme extend self
when "true"
"dark" def convert_theme(theme)
when "false" case theme
"light" when "true"
when "", nil "dark"
nil when "false"
else "light"
theme when "", nil
nil
else
theme
end
end end
end end

2
src/invidious/user/exports.cr
View File

@ -15,7 +15,7 @@ struct Invidious::User
playlists.each do |playlist| playlists.each do |playlist|
json.object do json.object do
json.field "title", playlist.title json.field "title", playlist.title
json.field "description", Helpers.html_to_content(playlist.description_html) json.field "description", Invidious::Helpers.html_to_content(playlist.description_html)
json.field "privacy", playlist.privacy.to_s json.field "privacy", playlist.privacy.to_s
json.field "videos" do json.field "videos" do
json.array do json.array do

4
src/invidious/user/imports.cr
View File

@ -43,7 +43,7 @@ struct Invidious::User
description = "This is the default description of an imported playlist. Feel Free to change it as you see fit." description = "This is the default description of an imported playlist. Feel Free to change it as you see fit."
privacy = PlaylistPrivacy::Private privacy = PlaylistPrivacy::Private
playlist = create_playlist(title, privacy, user) playlist = Invidious::Playlists::Playlists.create_playlist(title, privacy, user)
Invidious::Database::Playlists.update_description(playlist.id, description) Invidious::Database::Playlists.update_description(playlist.id, description)
# Add each video to the playlist from the body content # Add each video to the playlist from the body content
@ -117,7 +117,7 @@ struct Invidious::User
next if !description next if !description
next if !privacy next if !privacy
playlist = create_playlist(title, privacy, user) playlist = Invidious::Playlists::Playlists.create_playlist(title, privacy, user)
Invidious::Database::Playlists.update_description(playlist.id, description) Invidious::Database::Playlists.update_description(playlist.id, description)
item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx| item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx|

109
src/invidious/user/users.cr Normal file
View File

@ -0,0 +1,109 @@
require "crypto/bcrypt/password"
module Invidious::User::Users
extend self
# Materialized views may not be defined using bound parameters (`$1` as used elsewhere)
MATERIALIZED_VIEW_SQL = ->(email : String) { "SELECT cv.* FROM channel_videos cv WHERE EXISTS (SELECT subscriptions FROM users u WHERE cv.ucid = ANY (u.subscriptions) AND u.email = E'#{email.gsub({'\'' => "\\'", '\\' => "\\\\"})}') ORDER BY published DESC" }
def create_user(sid, email, password)
password = Crypto::Bcrypt::Password.create(password, cost: 10)
token = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
user = Invidious::User.new({
updated: Time.utc,
notifications: [] of String,
subscriptions: [] of String,
email: email,
preferences: Preferences.new(CONFIG.default_user_preferences.to_tuple),
password: password.to_s,
token: token,
watched: [] of String,
feed_needs_update: true,
})
return user, sid
end
def get_subscription_feed(user, max_results = 40, page = 1)
limit = max_results.clamp(0, MAX_ITEMS_PER_PAGE)
offset = (page - 1) * limit
notifications = Invidious::Database::Users.select_notifications(user)
view_name = "subscriptions_#{sha256(user.email)}"
if user.preferences.notifications_only && !notifications.empty?
# Only show notifications
notifications = Invidious::Database::ChannelVideos.select(notifications)
videos = [] of ChannelVideo
notifications.sort_by!(&.published).reverse!
case user.preferences.sort
when "alphabetically"
notifications.sort_by!(&.title)
when "alphabetically - reverse"
notifications.sort_by!(&.title).reverse!
when "channel name"
notifications.sort_by!(&.author)
when "channel name - reverse"
notifications.sort_by!(&.author).reverse!
else nil # Ignore
end
else
if user.preferences.latest_only
if user.preferences.unseen_only
# Show latest video from a channel that a user hasn't watched
# "unseen_only" isn't really correct here, more accurate would be "unwatched_only"
if user.watched.empty?
values = "'{}'"
else
values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}"
end
videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY ucid, published DESC", as: ChannelVideo)
else
# Show latest video from each channel
videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} ORDER BY ucid, published DESC", as: ChannelVideo)
end
videos.sort_by!(&.published).reverse!
else
if user.preferences.unseen_only
# Only show unwatched
if user.watched.empty?
values = "'{}'"
else
values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}"
end
videos = PG_DB.query_all("SELECT * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo)
else
# Sort subscriptions as normal
videos = PG_DB.query_all("SELECT * FROM #{view_name} ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo)
end
end
case user.preferences.sort
when "published - reverse"
videos.sort_by!(&.published)
when "alphabetically"
videos.sort_by!(&.title)
when "alphabetically - reverse"
videos.sort_by!(&.title).reverse!
when "channel name"
videos.sort_by!(&.author)
when "channel name - reverse"
videos.sort_by!(&.author).reverse!
else nil # Ignore
end
notifications = Invidious::Database::Users.select_notifications(user)
notifications = videos.select { |v| notifications.includes? v.id }
videos = videos - notifications
end
return videos, notifications
end
end

106
src/invidious/users.cr
View File

@ -1,106 +0,0 @@
require "crypto/bcrypt/password"
# Materialized views may not be defined using bound parameters (`$1` as used elsewhere)
MATERIALIZED_VIEW_SQL = ->(email : String) { "SELECT cv.* FROM channel_videos cv WHERE EXISTS (SELECT subscriptions FROM users u WHERE cv.ucid = ANY (u.subscriptions) AND u.email = E'#{email.gsub({'\'' => "\\'", '\\' => "\\\\"})}') ORDER BY published DESC" }
def create_user(sid, email, password)
password = Crypto::Bcrypt::Password.create(password, cost: 10)
token = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
user = Invidious::User.new({
updated: Time.utc,
notifications: [] of String,
subscriptions: [] of String,
email: email,
preferences: Preferences.new(CONFIG.default_user_preferences.to_tuple),
password: password.to_s,
token: token,
watched: [] of String,
feed_needs_update: true,
})
return user, sid
end
def get_subscription_feed(user, max_results = 40, page = 1)
limit = max_results.clamp(0, MAX_ITEMS_PER_PAGE)
offset = (page - 1) * limit
notifications = Invidious::Database::Users.select_notifications(user)
view_name = "subscriptions_#{sha256(user.email)}"
if user.preferences.notifications_only && !notifications.empty?
# Only show notifications
notifications = Invidious::Database::ChannelVideos.select(notifications)
videos = [] of ChannelVideo
notifications.sort_by!(&.published).reverse!
case user.preferences.sort
when "alphabetically"
notifications.sort_by!(&.title)
when "alphabetically - reverse"
notifications.sort_by!(&.title).reverse!
when "channel name"
notifications.sort_by!(&.author)
when "channel name - reverse"
notifications.sort_by!(&.author).reverse!
else nil # Ignore
end
else
if user.preferences.latest_only
if user.preferences.unseen_only
# Show latest video from a channel that a user hasn't watched
# "unseen_only" isn't really correct here, more accurate would be "unwatched_only"
if user.watched.empty?
values = "'{}'"
else
values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}"
end
videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY ucid, published DESC", as: ChannelVideo)
else
# Show latest video from each channel
videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} ORDER BY ucid, published DESC", as: ChannelVideo)
end
videos.sort_by!(&.published).reverse!
else
if user.preferences.unseen_only
# Only show unwatched
if user.watched.empty?
values = "'{}'"
else
values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}"
end
videos = PG_DB.query_all("SELECT * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo)
else
# Sort subscriptions as normal
videos = PG_DB.query_all("SELECT * FROM #{view_name} ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo)
end
end
case user.preferences.sort
when "published - reverse"
videos.sort_by!(&.published)
when "alphabetically"
videos.sort_by!(&.title)
when "alphabetically - reverse"
videos.sort_by!(&.title).reverse!
when "channel name"
videos.sort_by!(&.author)
when "channel name - reverse"
videos.sort_by!(&.author).reverse!
else nil # Ignore
end
notifications = Invidious::Database::Users.select_notifications(user)
notifications = videos.select { |v| notifications.includes? v.id }
videos = videos - notifications
end
return videos, notifications
end

2
src/invidious/videos/parser.cr
View File

@ -87,7 +87,7 @@ module Invidious::Videos::Parser
# Although technically not a call to /videoplayback the fact that YouTube is returning the # Although technically not a call to /videoplayback the fact that YouTube is returning the
# wrong video means that we should count it as a failure. # wrong video means that we should count it as a failure.
Helpers.get_playback_statistic["totalRequests"] += 1 Invidious::Helpers.get_playback_statistic["totalRequests"] += 1
return { return {
"version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64), "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),

0
src/invidious/videos.cr → src/invidious/videos/videos.cr
View File

2
src/invidious/views/components/item.ecr
View File

@ -106,7 +106,7 @@
</div> </div>
<details> <details>
<summary class="pure-button pure-button-secondary"><%=I18n.translate(locale, "timeline_parse_error_show_technical_details")%></summary> <summary class="pure-button pure-button-secondary"><%=I18n.translate(locale, "timeline_parse_error_show_technical_details")%></summary>
<pre class="error-issue-template"><%=get_issue_template(env, item.parse_exception)[1]%></pre> <pre class="error-issue-template"><%=Errors.get_issue_template(env, item.parse_exception)[1]%></pre>
</details> </details>
</div> </div>
<% else %> <% else %>