VeritableValor/invidious
1
0
Fork 0
mirror of https://github.com/iv-org/invidious.git synced 2026-09-26 11:35:27 -05:00
Code Issues Packages Projects Releases Wiki Activity

chore: Encapsulate more functions into their own modules and improve specs

Browse Source
This commit is contained in:
Fijxu 2026-09-15 17:49:12 -03:00
parent e4ad826f1f
commit 7e106030e6
No known key found for this signature in database
GPG Key ID: 32C1DDF333EDA6A4
58 changed files with 2269 additions and 2006 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
spec/invidious/channel/community_spec.cr Normal file
View File

File diff suppressed because one or more lines are too long

26
spec/invidious/helpers/tokens_spec.cr Normal file
View File

@ -0,0 +1,26 @@
require "../../spec_helper"
Spectator.describe Invidious::Helpers::Tokens do
describe ".sign_token" do
it "correctly signs a given hash" do
token = {
"session" => "v1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
"expires" => 1554680038,
"scopes" => [
":notifications",
":subscriptions/*",
"GET:tokens*",
],
"signature" => "f__2hS20th8pALF305PJFK-D2aVtvefNnQheILHD2vU=",
}
expect(Invidious::Helpers::Tokens.sign_token("SECRET_KEY", token)).to eq(token["signature"])
token = {
"session" => "v1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
"scopes" => [":notifications", "POST:subscriptions/*"],
"signature" => "fNvXoT0MRAL9eE6lTE33CEg8HitYJDOL9a22rSN2Ihg=",
}
expect(Invidious::Helpers::Tokens.sign_token("SECRET_KEY", token)).to eq(token["signature"])
end
end
end

56
spec/invidious/helpers_spec.cr
View File

File diff suppressed because one or more lines are too long

11
spec/invidious/search/ctoken_spec.cr Normal file
View File

@ -0,0 +1,11 @@
require "../../spec_helper"
Spectator.describe Invidious::Search::CToken do
describe ".produce_channel_search_continuation" do
it "correctly produces token for searching a specific channel" do
expect(Invidious::Search::CToken.produce_channel_search_continuation("UCXuqSBlHAE6Xw-yeJA0Tunw", "", 100)).to eq("4qmFsgJqEhhVQ1h1cVNCbEhBRTZYdy15ZUpBMFR1bncaIEVnWnpaV0Z5WTJnd0FUZ0JZQUY2QkVkS2IxaTRBUUE9WgCaAilicm93c2UtZmVlZFVDWHVxU0JsSEFFNlh3LXllSkEwVHVud3NlYXJjaA%3D%3D")
expect(Invidious::Search::CToken.produce_channel_search_continuation("UCXuqSBlHAE6Xw-yeJA0Tunw", "По ожиशुपतिरपि子而時ஸ்றீனி", 0)).to eq("4qmFsgKoARIYVUNYdXFTQmxIQUU2WHcteWVKQTBUdW53GiBFZ1p6WldGeVkyZ3dBVGdCWUFGNkJFZEJRVDI0QVFBPVo-0J_QviDQvtC20LjgpLbgpYHgpKrgpKTgpL_gpLDgpKrgpL_lrZDogIzmmYLgrrjgr43grrHgr4Dgrqngrr-aAilicm93c2UtZmVlZFVDWHVxU0JsSEFFNlh3LXllSkEwVHVud3NlYXJjaA%3D%3D")
end
end
end

0
spec/invidious/hashtag_spec.cr → spec/invidious/search/hashtag_spec.cr
View File

1
spec/spec_helper.cr
View File

@ -9,7 +9,6 @@ require "../src/invidious/videos/caption"
require "../src/invidious/videos"
require "../src/invidious/playlists"
require "../src/invidious/search/ctoken"
require "../src/invidious/trending"
require "spectator"
Spectator.configure do |config|

4
src/invidious.cr
View File

@ -44,6 +44,8 @@ require "./invidious/jsonify/**"
require "./invidious/*"
require "./invidious/comments/*"
require "./invidious/channels/*"
require "./invidious/playlists/*"
require "./invidious/feeds/*"
require "./invidious/user/*"
require "./invidious/search/*"
require "./invidious/routes/**"
@ -230,7 +232,7 @@ error 404 do |env|
end
error 500 do |env, exception|
error_template(500, exception)
Errors.error_template(500, exception)
end
# Init Kemal

340
src/invidious/channels/about.cr
View File

@ -19,202 +19,206 @@ record AboutChannel,
verified : Bool,
is_age_gated : Bool
def get_about_info(ucid) : AboutChannel
begin
# Fetch channel information from channel home page
initdata = YoutubeAPI.browse(browse_id: ucid, params: "")
rescue
raise InfoException.new("Could not get channel info.")
end
module Invidious::Channels::About
extend self
if initdata.dig?("alerts", 0, "alertRenderer", "type") == "ERROR"
error_message = initdata["alerts"][0]["alertRenderer"]["text"]["simpleText"].as_s
if error_message == "This channel does not exist."
raise NotFoundException.new(error_message)
else
raise InfoException.new(error_message)
def get_about_info(ucid) : AboutChannel
begin
# Fetch channel information from channel home page
initdata = YoutubeAPI.browse(browse_id: ucid, params: "")
rescue
raise InfoException.new("Could not get channel info.")
end
end
if browse_endpoint = initdata["onResponseReceivedActions"]?.try &.[0]?.try &.["navigateAction"]?.try &.["endpoint"]?.try &.["browseEndpoint"]?
raise ChannelRedirect.new(channel_id: browse_endpoint["browseId"].to_s)
end
if initdata.dig?("alerts", 0, "alertRenderer", "type") == "ERROR"
error_message = initdata["alerts"][0]["alertRenderer"]["text"]["simpleText"].as_s
if error_message == "This channel does not exist."
raise NotFoundException.new(error_message)
else
raise InfoException.new(error_message)
end
end
auto_generated = false
# Check for special auto generated gaming channels
if !initdata.has_key?("metadata")
auto_generated = true
end
if browse_endpoint = initdata["onResponseReceivedActions"]?.try &.[0]?.try &.["navigateAction"]?.try &.["endpoint"]?.try &.["browseEndpoint"]?
raise ChannelRedirect.new(channel_id: browse_endpoint["browseId"].to_s)
end
tags = [] of String
tab_names = [] of String
total_views = 0_i64
joined = Time.unix(0)
if age_gate_renderer = initdata.dig?("contents", "twoColumnBrowseResultsRenderer", "tabs", 0, "tabRenderer", "content", "sectionListRenderer", "contents", 0, "channelAgeGateRenderer")
description_node = nil
author = age_gate_renderer["channelTitle"].as_s
ucid = initdata.dig("responseContext", "serviceTrackingParams", 0, "params", 0, "value").as_s
author_url = "https://www.youtube.com/channel/#{ucid}"
author_thumbnail = age_gate_renderer.dig("avatar", "thumbnails", 0, "url").as_s
banner = nil
is_family_friendly = false
is_age_gated = true
tab_names = ["videos", "shorts", "streams"]
auto_generated = false
else
if auto_generated
author = initdata["header"]["interactiveTabbedHeaderRenderer"]["title"]["simpleText"].as_s
author_url = initdata["microformat"]["microformatDataRenderer"]["urlCanonical"].as_s
author_thumbnail = initdata["header"]["interactiveTabbedHeaderRenderer"]["boxArt"]["thumbnails"][0]["url"].as_s
# Check for special auto generated gaming channels
if !initdata.has_key?("metadata")
auto_generated = true
end
# Raises a KeyError on failure.
banners = initdata["header"]["interactiveTabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]?
banner = banners.try &.[-1]?.try &.["url"].as_s?
tags = [] of String
tab_names = [] of String
total_views = 0_i64
joined = Time.unix(0)
description_base_node = initdata["header"]["interactiveTabbedHeaderRenderer"]["description"]
# some channels have the description in a simpleText
# ex: https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/
description_node = description_base_node.dig?("simpleText") || description_base_node
tags = initdata.dig?("header", "interactiveTabbedHeaderRenderer", "badges")
.try &.as_a.map(&.["metadataBadgeRenderer"]["label"].as_s) || [] of String
if age_gate_renderer = initdata.dig?("contents", "twoColumnBrowseResultsRenderer", "tabs", 0, "tabRenderer", "content", "sectionListRenderer", "contents", 0, "channelAgeGateRenderer")
description_node = nil
author = age_gate_renderer["channelTitle"].as_s
ucid = initdata.dig("responseContext", "serviceTrackingParams", 0, "params", 0, "value").as_s
author_url = "https://www.youtube.com/channel/#{ucid}"
author_thumbnail = age_gate_renderer.dig("avatar", "thumbnails", 0, "url").as_s
banner = nil
is_family_friendly = false
is_age_gated = true
tab_names = ["videos", "shorts", "streams"]
auto_generated = false
else
author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s
author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s
author_thumbnail = initdata["metadata"]["channelMetadataRenderer"]["avatar"]["thumbnails"][0]["url"].as_s
author_badge = initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "title", "dynamicTextViewModel", "text", "attachmentRuns", 0, "element", "type", "imageType", "image", "sources", 0, "clientResource", "imageName")
.try &.as_s
# CHECK_CIRCLE_FILLED is used for normal channels and AUDIO_BADGE if used For
# music/artist channels
# TODO: Maybe separate verified author from verified artist?
author_verified = author_badge.try { |badge| badge == "CHECK_CIRCLE_FILLED" || badge == "AUDIO_BADGE" } || false
ucid = initdata["metadata"]["channelMetadataRenderer"]["externalId"].as_s
if auto_generated
author = initdata["header"]["interactiveTabbedHeaderRenderer"]["title"]["simpleText"].as_s
author_url = initdata["microformat"]["microformatDataRenderer"]["urlCanonical"].as_s
author_thumbnail = initdata["header"]["interactiveTabbedHeaderRenderer"]["boxArt"]["thumbnails"][0]["url"].as_s
# Raises a KeyError on failure.
# TODO: Check if `c4TabbedHeaderRenderer` still exists on some channels.
banners = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]?
banners ||= initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "banner", "imageBannerViewModel", "image", "sources")
banner = banners.try &.[-1]?.try &.["url"].as_s?
# Raises a KeyError on failure.
banners = initdata["header"]["interactiveTabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]?
banner = banners.try &.[-1]?.try &.["url"].as_s?
# if banner.includes? "channels/c4/default_banner"
# banner = nil
# end
description_base_node = initdata["header"]["interactiveTabbedHeaderRenderer"]["description"]
# some channels have the description in a simpleText
# ex: https://www.youtube.com/channel/UCQvWX73GQygcwXOTSf_VDVg/
description_node = description_base_node.dig?("simpleText") || description_base_node
description_node = initdata["metadata"]["channelMetadataRenderer"]?.try &.["description"]?
tags = initdata.dig?("microformat", "microformatDataRenderer", "tags").try &.as_a.map(&.as_s) || [] of String
end
tags = initdata.dig?("header", "interactiveTabbedHeaderRenderer", "badges")
.try &.as_a.map(&.["metadataBadgeRenderer"]["label"].as_s) || [] of String
else
author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s
author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s
author_thumbnail = initdata["metadata"]["channelMetadataRenderer"]["avatar"]["thumbnails"][0]["url"].as_s
author_badge = initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "title", "dynamicTextViewModel", "text", "attachmentRuns", 0, "element", "type", "imageType", "image", "sources", 0, "clientResource", "imageName")
.try &.as_s
# CHECK_CIRCLE_FILLED is used for normal channels and AUDIO_BADGE if used For
# music/artist channels
# TODO: Maybe separate verified author from verified artist?
author_verified = author_badge.try { |badge| badge == "CHECK_CIRCLE_FILLED" || badge == "AUDIO_BADGE" } || false
ucid = initdata["metadata"]["channelMetadataRenderer"]["externalId"].as_s
is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool
if tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]?
# Get the name of the tabs available on this channel
tab_names = tabs_json.as_a.compact_map do |entry|
name = entry.dig?("tabRenderer", "title").try &.as_s.downcase
# Raises a KeyError on failure.
# TODO: Check if `c4TabbedHeaderRenderer` still exists on some channels.
banners = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["banner"]?.try &.["thumbnails"]?
banners ||= initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "banner", "imageBannerViewModel", "image", "sources")
banner = banners.try &.[-1]?.try &.["url"].as_s?
# This is a small fix to not add extra code on the HTML side
# I.e, the URL for the "live" tab is .../streams, so use "streams"
# everywhere for the sake of simplicity
(name == "live") ? "streams" : name
# if banner.includes? "channels/c4/default_banner"
# banner = nil
# end
description_node = initdata["metadata"]["channelMetadataRenderer"]?.try &.["description"]?
tags = initdata.dig?("microformat", "microformatDataRenderer", "tags").try &.as_a.map(&.as_s) || [] of String
end
# Get the currently active tab ("About")
about_tab = extract_selected_tab(tabs_json)
is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool
if tabs_json = initdata["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]?
# Get the name of the tabs available on this channel
tab_names = tabs_json.as_a.compact_map do |entry|
name = entry.dig?("tabRenderer", "title").try &.as_s.downcase
# Try to find the about metadata section
channel_about_meta = about_tab.dig?(
"content",
"sectionListRenderer", "contents", 0,
"itemSectionRenderer", "contents", 0,
"channelAboutFullMetadataRenderer"
)
# This is a small fix to not add extra code on the HTML side
# I.e, the URL for the "live" tab is .../streams, so use "streams"
# everywhere for the sake of simplicity
(name == "live") ? "streams" : name
end
if !channel_about_meta.nil?
total_views = channel_about_meta.dig?("viewCountText", "simpleText").try &.as_s.gsub(/\D/, "").to_i64? || 0_i64
# Get the currently active tab ("About")
about_tab = extract_selected_tab(tabs_json)
# The joined text is split to several sub strings. The reduce joins those strings before parsing the date.
joined = extract_text(channel_about_meta["joinedDateText"]?)
.try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0)
# Normal Auto-generated channels
# https://support.google.com/youtube/answer/2579942
# For auto-generated channels, channel_about_meta only has
# ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"]
auto_generated = (
(channel_about_meta["primaryLinks"]?.try &.size) == 1 && \
extract_text(channel_about_meta.dig?("primaryLinks", 0, "title")) == "Auto-generated by YouTube" ||
channel_about_meta.dig?("links", 0, "channelExternalLinkViewModel", "title", "content").try &.as_s == "Auto-generated by YouTube"
# Try to find the about metadata section
channel_about_meta = about_tab.dig?(
"content",
"sectionListRenderer", "contents", 0,
"itemSectionRenderer", "contents", 0,
"channelAboutFullMetadataRenderer"
)
if !channel_about_meta.nil?
total_views = channel_about_meta.dig?("viewCountText", "simpleText").try &.as_s.gsub(/\D/, "").to_i64? || 0_i64
# The joined text is split to several sub strings. The reduce joins those strings before parsing the date.
joined = extract_text(channel_about_meta["joinedDateText"]?)
.try { |text| Time.parse(text, "Joined %b %-d, %Y", Time::Location.local) } || Time.unix(0)
# Normal Auto-generated channels
# https://support.google.com/youtube/answer/2579942
# For auto-generated channels, channel_about_meta only has
# ["description"]["simpleText"] and ["primaryLinks"][0]["title"]["simpleText"]
auto_generated = (
(channel_about_meta["primaryLinks"]?.try &.size) == 1 && \
extract_text(channel_about_meta.dig?("primaryLinks", 0, "title")) == "Auto-generated by YouTube" ||
channel_about_meta.dig?("links", 0, "channelExternalLinkViewModel", "title", "content").try &.as_s == "Auto-generated by YouTube"
)
end
end
end
end
allowed_regions = initdata
.dig?("microformat", "microformatDataRenderer", "availableCountries")
.try &.as_a.map(&.as_s) || [] of String
allowed_regions = initdata
.dig?("microformat", "microformatDataRenderer", "availableCountries")
.try &.as_a.map(&.as_s) || [] of String
description = !description_node.nil? ? description_node.as_s : ""
description_html = HTML.escape(description)
description = !description_node.nil? ? description_node.as_s : ""
description_html = HTML.escape(description)
if !description_node.nil?
if description_node.as_h?.nil?
description_node = text_to_parsed_content(description_node.as_s)
end
description_html = parse_content(description_node)
if description_html == "" && description != ""
description_html = HTML.escape(description)
end
end
sub_count = 0
pronouns = nil
if (metadata_rows = initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "metadata", "contentMetadataViewModel", "metadataRows").try &.as_a)
metadata_rows.each do |row|
subscribe_metadata_part = row.dig?("metadataParts").try &.as_a.find { |i| i.dig?("text", "content").try &.as_s.includes?("subscribers") }
if !subscribe_metadata_part.nil?
sub_count = short_text_to_number(subscribe_metadata_part.dig("text", "content").as_s.split(" ")[0]).to_i32
if !description_node.nil?
if description_node.as_h?.nil?
description_node = text_to_parsed_content(description_node.as_s)
end
pronoun_metadata_part = row.dig?("metadataParts").try &.as_a.find { |i| i.dig?("tooltip").try &.as_s.includes?("Pronouns") }
if !pronoun_metadata_part.nil?
pronouns = pronoun_metadata_part.dig("text", "content").as_s
description_html = parse_content(description_node)
if description_html == "" && description != ""
description_html = HTML.escape(description)
end
break if sub_count != 0 && !pronouns.nil?
end
sub_count = 0
pronouns = nil
if (metadata_rows = initdata.dig?("header", "pageHeaderRenderer", "content", "pageHeaderViewModel", "metadata", "contentMetadataViewModel", "metadataRows").try &.as_a)
metadata_rows.each do |row|
subscribe_metadata_part = row.dig?("metadataParts").try &.as_a.find { |i| i.dig?("text", "content").try &.as_s.includes?("subscribers") }
if !subscribe_metadata_part.nil?
sub_count = short_text_to_number(subscribe_metadata_part.dig("text", "content").as_s.split(" ")[0]).to_i32
end
pronoun_metadata_part = row.dig?("metadataParts").try &.as_a.find { |i| i.dig?("tooltip").try &.as_s.includes?("Pronouns") }
if !pronoun_metadata_part.nil?
pronouns = pronoun_metadata_part.dig("text", "content").as_s
end
break if sub_count != 0 && !pronouns.nil?
end
end
AboutChannel.new(
ucid: ucid,
author: author,
auto_generated: auto_generated,
author_url: author_url,
author_thumbnail: author_thumbnail,
banner: banner,
description: description,
description_html: description_html,
total_views: total_views,
sub_count: sub_count,
joined: joined,
is_family_friendly: is_family_friendly,
pronouns: pronouns,
allowed_regions: allowed_regions,
tabs: tab_names,
tags: tags,
verified: author_verified || false,
is_age_gated: is_age_gated || false,
)
end
AboutChannel.new(
ucid: ucid,
author: author,
auto_generated: auto_generated,
author_url: author_url,
author_thumbnail: author_thumbnail,
banner: banner,
description: description,
description_html: description_html,
total_views: total_views,
sub_count: sub_count,
joined: joined,
is_family_friendly: is_family_friendly,
pronouns: pronouns,
allowed_regions: allowed_regions,
tabs: tab_names,
tags: tags,
verified: author_verified || false,
is_age_gated: is_age_gated || false,
)
end
def fetch_related_channels(about_channel : AboutChannel, continuation : String? = nil) : {Array(SearchChannel), String?}
if continuation.nil?
# params is {"2:string":"channels"} encoded
initial_data = YoutubeAPI.browse(browse_id: about_channel.ucid, params: "EghjaGFubmVscw%3D%3D")
else
initial_data = YoutubeAPI.browse(continuation)
end
items, continuation = extract_items(initial_data)
return items.select(SearchChannel), continuation
def fetch_related_channels(about_channel : AboutChannel, continuation : String? = nil) : {Array(SearchChannel), String?}
if continuation.nil?
# params is {"2:string":"channels"} encoded
initial_data = YoutubeAPI.browse(browse_id: about_channel.ucid, params: "EghjaGFubmVscw%3D%3D")
else
initial_data = YoutubeAPI.browse(continuation)
end
items, continuation = extract_items(initial_data)
return items.select(SearchChannel), continuation
end
end

258
src/invidious/channels/channels.cr
View File

@ -123,7 +123,7 @@ def get_batch_channels(channels)
active_threads += 1
spawn do
begin
get_channel(ucid)
Invidious::Channels::Channels.get_channel(ucid)
finished_channel.send(ucid)
rescue ex
finished_channel.send(nil)
@ -144,153 +144,157 @@ def get_batch_channels(channels)
return final
end
def get_channel(id) : InvidiousChannel
channel = Invidious::Database::Channels.select(id)
module Invidious::Channels::Channels
extend self
if channel.nil? || (Time.utc - channel.updated) > 2.days
channel = fetch_channel(id, pull_all_videos: false)
Invidious::Database::Channels.insert(channel, update_on_conflict: true)
def get_channel(id) : InvidiousChannel
channel = Invidious::Database::Channels.select(id)
if channel.nil? || (Time.utc - channel.updated) > 2.days
channel = self.fetch_channel(id, pull_all_videos: false)
Invidious::Database::Channels.insert(channel, update_on_conflict: true)
end
return channel
end
return channel
end
def fetch_channel(ucid, pull_all_videos : Bool)
LOGGER.debug("fetch_channel: #{ucid}")
LOGGER.trace("fetch_channel: #{ucid} : pull_all_videos = #{pull_all_videos}")
def fetch_channel(ucid, pull_all_videos : Bool)
LOGGER.debug("fetch_channel: #{ucid}")
LOGGER.trace("fetch_channel: #{ucid} : pull_all_videos = #{pull_all_videos}")
namespaces = {
"yt" => "http://www.youtube.com/xml/schemas/2015",
"media" => "http://search.yahoo.com/mrss/",
"default" => "http://www.w3.org/2005/Atom",
}
namespaces = {
"yt" => "http://www.youtube.com/xml/schemas/2015",
"media" => "http://search.yahoo.com/mrss/",
"default" => "http://www.w3.org/2005/Atom",
}
LOGGER.trace("fetch_channel: #{ucid} : Downloading RSS feed")
rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}").body
LOGGER.trace("fetch_channel: #{ucid} : Parsing RSS feed")
rss = XML.parse(rss)
LOGGER.trace("fetch_channel: #{ucid} : Downloading RSS feed")
rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}").body
LOGGER.trace("fetch_channel: #{ucid} : Parsing RSS feed")
rss = XML.parse(rss)
author = rss.xpath_node("//default:feed/default:title", namespaces)
if !author
raise InfoException.new("Deleted or invalid channel")
end
author = rss.xpath_node("//default:feed/default:title", namespaces)
if !author
raise InfoException.new("Deleted or invalid channel")
end
author = author.content
author = author.content
# Auto-generated channels
# https://support.google.com/youtube/answer/2579942
if author.ends_with?(" - Topic") ||
{"Popular on YouTube", "Music", "Sports", "Gaming"}.includes? author
auto_generated = true
end
# Auto-generated channels
# https://support.google.com/youtube/answer/2579942
if author.ends_with?(" - Topic") ||
{"Popular on YouTube", "Music", "Sports", "Gaming"}.includes? author
auto_generated = true
end
LOGGER.trace("fetch_channel: #{ucid} : author = #{author}, auto_generated = #{auto_generated}")
LOGGER.trace("fetch_channel: #{ucid} : author = #{author}, auto_generated = #{auto_generated}")
channel = InvidiousChannel.new({
id: ucid,
author: author,
updated: Time.utc,
deleted: false,
subscribed: nil,
})
LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos page")
videos, continuation = IV::Channel::Tabs.get_videos(channel)
LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel RSS feed")
rss.xpath_nodes("//default:feed/default:entry", namespaces).each do |entry|
video_id = entry.xpath_node("yt:videoId", namespaces).not_nil!.content
title = entry.xpath_node("default:title", namespaces).not_nil!.content
published = Time.parse_rfc3339(
entry.xpath_node("default:published", namespaces).not_nil!.content
)
updated = Time.parse_rfc3339(
entry.xpath_node("default:updated", namespaces).not_nil!.content
)
author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content
ucid = entry.xpath_node("yt:channelId", namespaces).not_nil!.content
views = entry
.xpath_node("media:group/media:community/media:statistics", namespaces)
.try &.["views"]?.try &.to_i64? || 0_i64
channel_video = videos
.select(SearchVideo)
.select(&.id.== video_id)[0]?
length_seconds = channel_video.try &.length_seconds
length_seconds ||= 0
live_now = channel_video.try &.badges.live_now?
live_now ||= false
premiere_timestamp = channel_video.try &.premiere_timestamp
video = ChannelVideo.new({
id: video_id,
title: title,
published: published,
updated: updated,
ucid: ucid,
author: author,
length_seconds: length_seconds,
live_now: live_now,
premiere_timestamp: premiere_timestamp,
views: views,
channel = InvidiousChannel.new({
id: ucid,
author: author,
updated: Time.utc,
deleted: false,
subscribed: nil,
})
LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updating or inserting video")
LOGGER.trace("fetch_channel: #{ucid} : Downloading channel videos page")
videos, continuation = IV::Channel::Tabs.get_videos(channel)
# We don't include the 'premiere_timestamp' here because channel pages don't include them,
# meaning the above timestamp is always null
was_insert = Invidious::Database::ChannelVideos.insert(video)
LOGGER.trace("fetch_channel: #{ucid} : Extracting videos from channel RSS feed")
rss.xpath_nodes("//default:feed/default:entry", namespaces).each do |entry|
video_id = entry.xpath_node("yt:videoId", namespaces).not_nil!.content
title = entry.xpath_node("default:title", namespaces).not_nil!.content
if was_insert
LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Inserted, updating subscriptions")
NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video))
else
LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updated")
published = Time.parse_rfc3339(
entry.xpath_node("default:published", namespaces).not_nil!.content
)
updated = Time.parse_rfc3339(
entry.xpath_node("default:updated", namespaces).not_nil!.content
)
author = entry.xpath_node("default:author/default:name", namespaces).not_nil!.content
ucid = entry.xpath_node("yt:channelId", namespaces).not_nil!.content
views = entry
.xpath_node("media:group/media:community/media:statistics", namespaces)
.try &.["views"]?.try &.to_i64? || 0_i64
channel_video = videos
.select(SearchVideo)
.select(&.id.== video_id)[0]?
length_seconds = channel_video.try &.length_seconds
length_seconds ||= 0
live_now = channel_video.try &.badges.live_now?
live_now ||= false
premiere_timestamp = channel_video.try &.premiere_timestamp
video = ChannelVideo.new({
id: video_id,
title: title,
published: published,
updated: updated,
ucid: ucid,
author: author,
length_seconds: length_seconds,
live_now: live_now,
premiere_timestamp: premiere_timestamp,
views: views,
})
LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updating or inserting video")
# We don't include the 'premiere_timestamp' here because channel pages don't include them,
# meaning the above timestamp is always null
was_insert = Invidious::Database::ChannelVideos.insert(video)
if was_insert
LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Inserted, updating subscriptions")
NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video))
else
LOGGER.trace("fetch_channel: #{ucid} : video #{video_id} : Updated")
end
end
end
if pull_all_videos
loop do
# Keep fetching videos using the continuation token retrieved earlier
videos, continuation = IV::Channel::Tabs.get_videos(channel, continuation: continuation)
if pull_all_videos
loop do
# Keep fetching videos using the continuation token retrieved earlier
videos, continuation = IV::Channel::Tabs.get_videos(channel, continuation: continuation)
count = 0
videos.select(SearchVideo).each do |video|
count += 1
video = ChannelVideo.new({
id: video.id,
title: video.title,
published: video.published,
updated: Time.utc,
ucid: video.ucid,
author: video.author,
length_seconds: video.length_seconds,
live_now: video.badges.live_now?,
premiere_timestamp: video.premiere_timestamp,
views: video.views,
})
count = 0
videos.select(SearchVideo).each do |video|
count += 1
video = ChannelVideo.new({
id: video.id,
title: video.title,
published: video.published,
updated: Time.utc,
ucid: video.ucid,
author: video.author,
length_seconds: video.length_seconds,
live_now: video.badges.live_now?,
premiere_timestamp: video.premiere_timestamp,
views: video.views,
})
# We are notified of Red videos elsewhere (PubSub), which includes a correct published date,
# so since they don't provide a published date here we can safely ignore them.
if Time.utc - video.published > 1.minute
was_insert = Invidious::Database::ChannelVideos.insert(video)
if was_insert
NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video))
# We are notified of Red videos elsewhere (PubSub), which includes a correct published date,
# so since they don't provide a published date here we can safely ignore them.
if Time.utc - video.published > 1.minute
was_insert = Invidious::Database::ChannelVideos.insert(video)
if was_insert
NOTIFICATION_CHANNEL.send(VideoNotification.from_video(video))
end
end
end
break if count < 25
sleep 500.milliseconds
end
break if count < 25
sleep 500.milliseconds
end
end
channel.updated = Time.utc
return channel
channel.updated = Time.utc
return channel
end
end

519
src/invidious/channels/community.cr
View File

@ -1,198 +1,201 @@
private IMAGE_QUALITIES = {320, 560, 640, 1280, 2000}
module Invidious::Channels::Community
extend self
private IMAGE_QUALITIES = {320, 560, 640, 1280, 2000}
# TODO: Add "sort_by"
def fetch_channel_community(ucid, cursor, locale, format, thin_mode)
if cursor.nil?
# EgVwb3N0c_IGBAoCSgA%3D is the protobuf object to load "posts"
initial_data = YoutubeAPI.browse(ucid, params: "EgVwb3N0c_IGBAoCSgA%3D")
# TODO: Add "sort_by"
def fetch_channel_community(ucid, cursor, locale, format, thin_mode)
if cursor.nil?
# EgVwb3N0c_IGBAoCSgA%3D is the protobuf object to load "posts"
initial_data = YoutubeAPI.browse(ucid, params: "EgVwb3N0c_IGBAoCSgA%3D")
items = [] of JSON::Any
extract_items(initial_data) do |item|
items << item
end
else
continuation = self.produce_channel_community_continuation(ucid, cursor)
initial_data = YoutubeAPI.browse(continuation: continuation)
container = initial_data.dig?("continuationContents", "itemSectionContinuation", "contents")
raise InfoException.new("Can't extract community data") if container.nil?
items = container.as_a
end
return extract_channel_community(items, ucid: ucid, locale: locale, format: format, thin_mode: thin_mode)
end
def decode_ucid_from_post_protobuf(params)
decoded_protobuf = params.try { |i| URI.decode_www_form(i) }
.try { |i| Base64.decode(i) }
.try { |i| IO::Memory.new(i) }
.try { |i| Protodec::Any.parse(i) }
return decoded_protobuf.try(&.["56:0:embedded"]["2:0:string"].as_s)
end
def fetch_channel_community_post(ucid, post_id, locale, format, thin_mode)
object = {
"56:embedded" => {
"2:string" => ucid,
"3:string" => post_id.to_s,
"11:string" => ucid,
},
}
params = object.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
initial_data = YoutubeAPI.browse("FEpost_detail", params: params)
items = [] of JSON::Any
extract_items(initial_data) do |item|
items << item
end
else
continuation = produce_channel_community_continuation(ucid, cursor)
initial_data = YoutubeAPI.browse(continuation: continuation)
container = initial_data.dig?("continuationContents", "itemSectionContinuation", "contents")
raise InfoException.new("Can't extract community data") if container.nil?
items = container.as_a
return extract_channel_community(items, ucid: ucid, locale: locale, format: format, thin_mode: thin_mode, is_single_post: true)
end
return extract_channel_community(items, ucid: ucid, locale: locale, format: format, thin_mode: thin_mode)
end
def decode_ucid_from_post_protobuf(params)
decoded_protobuf = params.try { |i| URI.decode_www_form(i) }
.try { |i| Base64.decode(i) }
.try { |i| IO::Memory.new(i) }
.try { |i| Protodec::Any.parse(i) }
return decoded_protobuf.try(&.["56:0:embedded"]["2:0:string"].as_s)
end
def fetch_channel_community_post(ucid, post_id, locale, format, thin_mode)
object = {
"56:embedded" => {
"2:string" => ucid,
"3:string" => post_id.to_s,
"11:string" => ucid,
},
}
params = object.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
initial_data = YoutubeAPI.browse("FEpost_detail", params: params)
items = [] of JSON::Any
extract_items(initial_data) do |item|
items << item
end
return extract_channel_community(items, ucid: ucid, locale: locale, format: format, thin_mode: thin_mode, is_single_post: true)
end
def extract_channel_community(items, *, ucid, locale, format, thin_mode, is_single_post : Bool = false)
if message = items[0]["messageRenderer"]?
error_message = (message["text"]["simpleText"]? ||
message["text"]["runs"]?.try &.[0]?.try &.["text"]?)
.try &.as_s || ""
if error_message == "This channel does not exist."
raise NotFoundException.new(error_message)
else
raise InfoException.new(error_message)
end
end
response = JSON.build do |json|
json.object do
json.field "authorId", ucid
if is_single_post
json.field "singlePost", true
def extract_channel_community(items, *, ucid, locale, format, thin_mode, is_single_post : Bool = false)
if message = items[0]["messageRenderer"]?
error_message = (message["text"]["simpleText"]? ||
message["text"]["runs"]?.try &.[0]?.try &.["text"]?)
.try &.as_s || ""
if error_message == "This channel does not exist."
raise NotFoundException.new(error_message)
else
raise InfoException.new(error_message)
end
json.field "comments" do
json.array do
items.each do |post|
comments = post["backstagePostThreadRenderer"]?.try &.["comments"]? ||
post["backstageCommentsContinuation"]?
end
post = post["backstagePostThreadRenderer"]?.try &.["post"]["backstagePostRenderer"]? ||
post["commentThreadRenderer"]?.try &.["comment"]["commentRenderer"]?
response = JSON.build do |json|
json.object do
json.field "authorId", ucid
if is_single_post
json.field "singlePost", true
end
json.field "comments" do
json.array do
items.each do |post|
comments = post["backstagePostThreadRenderer"]?.try &.["comments"]? ||
post["backstageCommentsContinuation"]?
next if !post
post = post["backstagePostThreadRenderer"]?.try &.["post"]["backstagePostRenderer"]? ||
post["commentThreadRenderer"]?.try &.["comment"]["commentRenderer"]?
content_html = post["contentText"]?.try { |t| parse_content(t) } || ""
author = post["authorText"]["runs"]?.try &.[0]?.try &.["text"]? || ""
next if !post
json.object do
json.field "author", author
json.field "authorThumbnails" do
json.array do
qualities = {32, 48, 76, 100, 176, 512}
author_thumbnail = post["authorThumbnail"]["thumbnails"].as_a[0]["url"].as_s
content_html = post["contentText"]?.try { |t| parse_content(t) } || ""
author = post["authorText"]["runs"]?.try &.[0]?.try &.["text"]? || ""
qualities.each do |quality|
json.object do
json.field "url", author_thumbnail.gsub(/s\d+-/, "s#{quality}-")
json.field "width", quality
json.field "height", quality
json.object do
json.field "author", author
json.field "authorThumbnails" do
json.array do
qualities = {32, 48, 76, 100, 176, 512}
author_thumbnail = post["authorThumbnail"]["thumbnails"].as_a[0]["url"].as_s
qualities.each do |quality|
json.object do
json.field "url", author_thumbnail.gsub(/s\d+-/, "s#{quality}-")
json.field "width", quality
json.field "height", quality
end
end
end
end
end
if post["authorEndpoint"]?
json.field "authorId", post["authorEndpoint"]["browseEndpoint"]["browseId"]
json.field "authorUrl", post["authorEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"].as_s
else
json.field "authorId", ""
json.field "authorUrl", ""
end
if post["authorEndpoint"]?
json.field "authorId", post["authorEndpoint"]["browseEndpoint"]["browseId"]
json.field "authorUrl", post["authorEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"].as_s
else
json.field "authorId", ""
json.field "authorUrl", ""
end
published_text = post["publishedTimeText"]["runs"][0]["text"].as_s
published = decode_date(published_text.rchop(" (edited)"))
published_text = post["publishedTimeText"]["runs"][0]["text"].as_s
published = decode_date(published_text.rchop(" (edited)"))
if published_text.includes?(" (edited)")
json.field "isEdited", true
else
json.field "isEdited", false
end
if published_text.includes?(" (edited)")
json.field "isEdited", true
else
json.field "isEdited", false
end
like_count = post["actionButtons"]["commentActionButtonsRenderer"]["likeButton"]["toggleButtonRenderer"]["accessibilityData"]["accessibilityData"]["label"]
.try &.as_s.gsub(/\D/, "").to_i? || 0
like_count = post["actionButtons"]["commentActionButtonsRenderer"]["likeButton"]["toggleButtonRenderer"]["accessibilityData"]["accessibilityData"]["label"]
.try &.as_s.gsub(/\D/, "").to_i? || 0
reply_count = short_text_to_number(post.dig?("actionButtons", "commentActionButtonsRenderer", "replyButton", "buttonRenderer", "text", "simpleText").try &.as_s || "0")
reply_count = short_text_to_number(post.dig?("actionButtons", "commentActionButtonsRenderer", "replyButton", "buttonRenderer", "text", "simpleText").try &.as_s || "0")
json.field "content", Helpers.html_to_content(content_html)
json.field "contentHtml", content_html
json.field "content", Invidious::Helpers.html_to_content(content_html)
json.field "contentHtml", content_html
json.field "published", published.to_unix
json.field "publishedText", I18n.translate(locale, "`x` ago", recode_date(published, locale))
json.field "published", published.to_unix
json.field "publishedText", I18n.translate(locale, "`x` ago", recode_date(published, locale))
json.field "likeCount", like_count
json.field "replyCount", reply_count
json.field "commentId", post["postId"]? || post["commentId"]? || ""
json.field "authorIsChannelOwner", post["authorEndpoint"]["browseEndpoint"]["browseId"] == ucid
json.field "likeCount", like_count
json.field "replyCount", reply_count
json.field "commentId", post["postId"]? || post["commentId"]? || ""
json.field "authorIsChannelOwner", post["authorEndpoint"]["browseEndpoint"]["browseId"] == ucid
if attachment = post["backstageAttachment"]?
json.field "attachment" do
case attachment.as_h
when .has_key?("videoRenderer")
parse_item(attachment)
.as(SearchVideo | ProblematicTimelineItem)
.to_json(locale, json)
when .has_key?("backstageImageRenderer")
json.object do
attachment = attachment["backstageImageRenderer"]
json.field "type", "image"
if attachment = post["backstageAttachment"]?
json.field "attachment" do
case attachment.as_h
when .has_key?("videoRenderer")
parse_item(attachment)
.as(SearchVideo | ProblematicTimelineItem)
.to_json(locale, json)
when .has_key?("backstageImageRenderer")
json.object do
attachment = attachment["backstageImageRenderer"]
json.field "type", "image"
json.field "imageThumbnails" do
json.array do
thumbnail = attachment["image"]["thumbnails"][0].as_h
width = thumbnail["width"].as_i
height = thumbnail["height"].as_i
aspect_ratio = (width.to_f / height.to_f)
url = thumbnail["url"].as_s.gsub(/=w\d+-h\d+(-p)?(-nd)?(-df)?(-rwa)?/, "=s640")
json.field "imageThumbnails" do
json.array do
thumbnail = attachment["image"]["thumbnails"][0].as_h
width = thumbnail["width"].as_i
height = thumbnail["height"].as_i
aspect_ratio = (width.to_f / height.to_f)
url = thumbnail["url"].as_s.gsub(/=w\d+-h\d+(-p)?(-nd)?(-df)?(-rwa)?/, "=s640")
IMAGE_QUALITIES.each do |quality|
json.object do
json.field "url", url.gsub(/=s\d+/, "=s#{quality}")
json.field "width", quality
json.field "height", (quality / aspect_ratio).ceil.to_i
IMAGE_QUALITIES.each do |quality|
json.object do
json.field "url", url.gsub(/=s\d+/, "=s#{quality}")
json.field "width", quality
json.field "height", (quality / aspect_ratio).ceil.to_i
end
end
end
end
end
end
when .has_key?("pollRenderer")
json.object do
attachment = attachment["pollRenderer"]
json.field "type", "poll"
json.field "totalVotes", short_text_to_number(attachment["totalVotes"]["simpleText"].as_s.split(" ")[0])
json.field "choices" do
json.array do
attachment["choices"].as_a.each do |choice|
json.object do
json.field "text", choice.dig("text", "runs", 0, "text").as_s
# A choice can have an image associated with it.
# Ex post: https://www.youtube.com/post/UgkxD4XavXUD4NQiddJXXdohbwOwcVqrH9Re
if choice["image"]?
thumbnail = choice["image"]["thumbnails"][0].as_h
width = thumbnail["width"].as_i
height = thumbnail["height"].as_i
aspect_ratio = (width.to_f / height.to_f)
url = thumbnail["url"].as_s.gsub(/=w\d+-h\d+(-p)?(-nd)?(-df)?(-rwa)?/, "=s640")
json.field "image" do
json.array do
IMAGE_QUALITIES.each do |quality|
json.object do
json.field "url", url.gsub(/=s\d+/, "=s#{quality}")
json.field "width", quality
json.field "height", (quality / aspect_ratio).ceil.to_i
when .has_key?("pollRenderer")
json.object do
attachment = attachment["pollRenderer"]
json.field "type", "poll"
json.field "totalVotes", short_text_to_number(attachment["totalVotes"]["simpleText"].as_s.split(" ")[0])
json.field "choices" do
json.array do
attachment["choices"].as_a.each do |choice|
json.object do
json.field "text", choice.dig("text", "runs", 0, "text").as_s
# A choice can have an image associated with it.
# Ex post: https://www.youtube.com/post/UgkxD4XavXUD4NQiddJXXdohbwOwcVqrH9Re
if choice["image"]?
thumbnail = choice["image"]["thumbnails"][0].as_h
width = thumbnail["width"].as_i
height = thumbnail["height"].as_i
aspect_ratio = (width.to_f / height.to_f)
url = thumbnail["url"].as_s.gsub(/=w\d+-h\d+(-p)?(-nd)?(-df)?(-rwa)?/, "=s640")
json.field "image" do
json.array do
IMAGE_QUALITIES.each do |quality|
json.object do
json.field "url", url.gsub(/=s\d+/, "=s#{quality}")
json.field "width", quality
json.field "height", (quality / aspect_ratio).ceil.to_i
end
end
end
end
@ -202,137 +205,137 @@ def extract_channel_community(items, *, ucid, locale, format, thin_mode, is_sing
end
end
end
end
when .has_key?("postMultiImageRenderer")
json.object do
attachment = attachment["postMultiImageRenderer"]
json.field "type", "multiImage"
json.field "images" do
json.array do
attachment["images"].as_a.each do |image|
json.array do
thumbnail = image["backstageImageRenderer"]["image"]["thumbnails"][0].as_h
width = thumbnail["width"].as_i
height = thumbnail["height"].as_i
aspect_ratio = (width.to_f / height.to_f)
url = thumbnail["url"].as_s.gsub(/=w\d+-h\d+(-p)?(-nd)?(-df)?(-rwa)?/, "=s640")
when .has_key?("postMultiImageRenderer")
json.object do
attachment = attachment["postMultiImageRenderer"]
json.field "type", "multiImage"
json.field "images" do
json.array do
attachment["images"].as_a.each do |image|
json.array do
thumbnail = image["backstageImageRenderer"]["image"]["thumbnails"][0].as_h
width = thumbnail["width"].as_i
height = thumbnail["height"].as_i
aspect_ratio = (width.to_f / height.to_f)
url = thumbnail["url"].as_s.gsub(/=w\d+-h\d+(-p)?(-nd)?(-df)?(-rwa)?/, "=s640")
IMAGE_QUALITIES.each do |quality|
json.object do
json.field "url", url.gsub(/=s\d+/, "=s#{quality}")
json.field "width", quality
json.field "height", (quality / aspect_ratio).ceil.to_i
IMAGE_QUALITIES.each do |quality|
json.object do
json.field "url", url.gsub(/=s\d+/, "=s#{quality}")
json.field "width", quality
json.field "height", (quality / aspect_ratio).ceil.to_i
end
end
end
end
end
end
end
end
when .has_key?("playlistRenderer")
parse_item(attachment)
.as(SearchPlaylist)
.to_json(locale, json)
when .has_key?("quizRenderer")
json.object do
attachment = attachment["quizRenderer"]
json.field "type", "quiz"
json.field "totalVotes", short_text_to_number(attachment["totalVotes"]["simpleText"].as_s.split(" ")[0])
json.field "choices" do
json.array do
attachment["choices"].as_a.each do |choice|
json.object do
json.field "text", choice.dig("text", "runs", 0, "text").as_s
json.field "isCorrect", choice["isCorrect"].as_bool
when .has_key?("playlistRenderer")
parse_item(attachment)
.as(SearchPlaylist)
.to_json(locale, json)
when .has_key?("quizRenderer")
json.object do
attachment = attachment["quizRenderer"]
json.field "type", "quiz"
json.field "totalVotes", short_text_to_number(attachment["totalVotes"]["simpleText"].as_s.split(" ")[0])
json.field "choices" do
json.array do
attachment["choices"].as_a.each do |choice|
json.object do
json.field "text", choice.dig("text", "runs", 0, "text").as_s
json.field "isCorrect", choice["isCorrect"].as_bool
end
end
end
end
end
end
else
json.object do
json.field "type", "unknown"
json.field "error", "Unrecognized attachment type."
else
json.object do
json.field "type", "unknown"
json.field "error", "Unrecognized attachment type."
end
end
end
end
end
if comments && (reply_count = (comments["backstageCommentsRenderer"]["moreText"]["simpleText"]? ||
comments["backstageCommentsRenderer"]["moreText"]["runs"]?.try &.[0]?.try &.["text"]?)
.try &.as_s.gsub(/\D/, "").to_i?)
continuation = comments["backstageCommentsRenderer"]["continuations"]?.try &.as_a[0]["nextContinuationData"]["continuation"].as_s
continuation ||= ""
if comments && (reply_count = (comments["backstageCommentsRenderer"]["moreText"]["simpleText"]? ||
comments["backstageCommentsRenderer"]["moreText"]["runs"]?.try &.[0]?.try &.["text"]?)
.try &.as_s.gsub(/\D/, "").to_i?)
continuation = comments["backstageCommentsRenderer"]["continuations"]?.try &.as_a[0]["nextContinuationData"]["continuation"].as_s
continuation ||= ""
json.field "replies" do
json.object do
json.field "replyCount", reply_count
json.field "continuation", extract_channel_community_cursor(continuation)
json.field "replies" do
json.object do
json.field "replyCount", reply_count
json.field "continuation", extract_channel_community_cursor(continuation)
end
end
end
end
end
end
end
end
if !is_single_post
if cont = items.dig?(-1, "continuationItemRenderer", "continuationEndpoint", "continuationCommand", "token")
json.field "continuation", extract_channel_community_cursor(cont.as_s)
if !is_single_post
if cont = items.dig?(-1, "continuationItemRenderer", "continuationEndpoint", "continuationCommand", "token")
json.field "continuation", extract_channel_community_cursor(cont.as_s)
end
end
end
end
end
if format == "html"
response = JSON.parse(response)
content_html = IV::Frontend::Comments.template_youtube(response, locale, thin_mode)
if format == "html"
response = JSON.parse(response)
content_html = IV::Frontend::Comments.template_youtube(response, locale, thin_mode)
response = JSON.build do |json|
json.object do
json.field "contentHtml", content_html
response = JSON.build do |json|
json.object do
json.field "contentHtml", content_html
end
end
end
return response
end
return response
end
def produce_channel_community_continuation(ucid, cursor)
object = {
"80226972:embedded" => {
"2:string" => ucid,
"3:string" => cursor || "",
},
}
def produce_channel_community_continuation(ucid, cursor)
object = {
"80226972:embedded" => {
"2:string" => ucid,
"3:string" => cursor || "",
},
}
continuation = object.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
return continuation
end
def extract_channel_community_cursor(continuation)
object = URI.decode_www_form(continuation)
.try { |i| Base64.decode(i) }
.try { |i| IO::Memory.new(i) }
.try { |i| Protodec::Any.parse(i) }
.try(&.["80226972:0:embedded"]["3:1:base64"].as_h)
if object["53:2:embedded"]?.try &.["3:0:embedded"]?
object["53:2:embedded"]["3:0:embedded"]["2:0:string"] = object["53:2:embedded"]["3:0:embedded"]
.try(&.["2:0:base64"].as_h)
.try { |i| Protodec::Any.cast_json(i) }
continuation = object.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i, padding: false) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
object["53:2:embedded"]["3:0:embedded"].as_h.delete("2:0:base64")
return continuation
end
cursor = Protodec::Any.cast_json(object)
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
def extract_channel_community_cursor(continuation)
object = URI.decode_www_form(continuation)
.try { |i| Base64.decode(i) }
.try { |i| IO::Memory.new(i) }
.try { |i| Protodec::Any.parse(i) }
.try(&.["80226972:0:embedded"]["3:1:base64"].as_h)
cursor
if object["53:2:embedded"]?.try &.["3:0:embedded"]?
object["53:2:embedded"]["3:0:embedded"]["2:0:string"] = object["53:2:embedded"]["3:0:embedded"]
.try(&.["2:0:base64"].as_h)
.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i, padding: false) }
object["53:2:embedded"]["3:0:embedded"].as_h.delete("2:0:base64")
end
cursor = Protodec::Any.cast_json(object)
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
cursor
end
end

96
src/invidious/channels/playlists.cr
View File

@ -1,55 +1,59 @@
def fetch_channel_playlists(ucid, author, continuation, sort_by)
if continuation
initial_data = YoutubeAPI.browse(continuation)
else
params =
case sort_by
when "last", "last_added"
# Equivalent to "&sort=lad"
# {"2:string": "playlists", "3:varint": 4, "4:varint": 1, "6:varint": 1, "110:embedded": {"1:embedded": {"8:string": ""}}}
"EglwbGF5bGlzdHMYBCABMAHyBgQKAkIA"
when "oldest", "oldest_created"
# formerly "&sort=da"
# Not available anymore :c or maybe ??
# {"2:string": "playlists", "3:varint": 2, "4:varint": 1, "6:varint": 1, "110:embedded": {"1:embedded": {"8:string": ""}}}
"EglwbGF5bGlzdHMYAiABMAHyBgQKAkIA"
# {"2:string": "playlists", "3:varint": 1, "4:varint": 1, "6:varint": 1}
# "EglwbGF5bGlzdHMYASABMAE%3D"
when "newest", "newest_created"
# Formerly "&sort=dd"
# {"2:string": "playlists", "3:varint": 3, "4:varint": 1, "6:varint": 1, "110:embedded": {"1:embedded": {"8:string": ""}}}
"EglwbGF5bGlzdHMYAyABMAHyBgQKAkIA"
end
module Invidious::Channels::Playlists
extend self
initial_data = YoutubeAPI.browse(ucid, params: params || "")
def fetch_channel_playlists(ucid, author, continuation, sort_by)
if continuation
initial_data = YoutubeAPI.browse(continuation)
else
params =
case sort_by
when "last", "last_added"
# Equivalent to "&sort=lad"
# {"2:string": "playlists", "3:varint": 4, "4:varint": 1, "6:varint": 1, "110:embedded": {"1:embedded": {"8:string": ""}}}
"EglwbGF5bGlzdHMYBCABMAHyBgQKAkIA"
when "oldest", "oldest_created"
# formerly "&sort=da"
# Not available anymore :c or maybe ??
# {"2:string": "playlists", "3:varint": 2, "4:varint": 1, "6:varint": 1, "110:embedded": {"1:embedded": {"8:string": ""}}}
"EglwbGF5bGlzdHMYAiABMAHyBgQKAkIA"
# {"2:string": "playlists", "3:varint": 1, "4:varint": 1, "6:varint": 1}
# "EglwbGF5bGlzdHMYASABMAE%3D"
when "newest", "newest_created"
# Formerly "&sort=dd"
# {"2:string": "playlists", "3:varint": 3, "4:varint": 1, "6:varint": 1, "110:embedded": {"1:embedded": {"8:string": ""}}}
"EglwbGF5bGlzdHMYAyABMAHyBgQKAkIA"
end
initial_data = YoutubeAPI.browse(ucid, params: params || "")
end
return extract_items(initial_data, author, ucid)
end
return extract_items(initial_data, author, ucid)
end
def fetch_channel_podcasts(ucid, author, continuation)
if continuation
initial_data = YoutubeAPI.browse(continuation)
else
initial_data = YoutubeAPI.browse(ucid, params: "Eghwb2RjYXN0c_IGBQoDugEA")
def fetch_channel_podcasts(ucid, author, continuation)
if continuation
initial_data = YoutubeAPI.browse(continuation)
else
initial_data = YoutubeAPI.browse(ucid, params: "Eghwb2RjYXN0c_IGBQoDugEA")
end
return extract_items(initial_data, author, ucid)
end
return extract_items(initial_data, author, ucid)
end
def fetch_channel_releases(ucid, author, continuation)
if continuation
initial_data = YoutubeAPI.browse(continuation)
else
initial_data = YoutubeAPI.browse(ucid, params: "EghyZWxlYXNlc_IGBQoDsgEA")
def fetch_channel_releases(ucid, author, continuation)
if continuation
initial_data = YoutubeAPI.browse(continuation)
else
initial_data = YoutubeAPI.browse(ucid, params: "EghyZWxlYXNlc_IGBQoDsgEA")
end
return extract_items(initial_data, author, ucid)
end
return extract_items(initial_data, author, ucid)
end
def fetch_channel_courses(ucid, author, continuation)
if continuation
initial_data = YoutubeAPI.browse(continuation)
else
initial_data = YoutubeAPI.browse(ucid, params: "Egdjb3Vyc2Vz8gYFCgPCAQA%3D")
def fetch_channel_courses(ucid, author, continuation)
if continuation
initial_data = YoutubeAPI.browse(continuation)
else
initial_data = YoutubeAPI.browse(ucid, params: "Egdjb3Vyc2Vz8gYFCgPCAQA%3D")
end
return extract_items(initial_data, author, ucid)
end
return extract_items(initial_data, author, ucid)
end

0
src/invidious/channels/videos.cr → src/invidious/channels/tabs.cr
View File

2
src/invidious/comments/youtube.cr
View File

@ -246,7 +246,7 @@ module Invidious::Comments
end
content_html = html_content || ""
json.field "content", Helpers.html_to_content(content_html)
json.field "content", Invidious::Helpers.html_to_content(content_html)
json.field "contentHtml", content_html
if published_text != nil

216
src/invidious/ext/kemal_static_file_handler.cr Normal file
View File

@ -0,0 +1,216 @@
{% if compare_versions(Crystal::VERSION, "1.17.0-dev") >= 0 %}
# Strip StaticFileHandler from the binary
#
# This allows us to compile on 1.17.0 as the compiler won't try to
# semantically check the outdated upstream code.
class Kemal::Config
private def setup_static_file_handler
end
end
# Nullify `Kemal::StaticFileHandler`
#
# Needed until the next release of Kemal after 1.7
class Kemal::StaticFileHandler < HTTP::StaticFileHandler
def call(context : HTTP::Server::Context)
end
end
{% skip_file %}
{% end %}
# Since systems have a limit on number of open files (`ulimit -a`),
# we serve them from memory to avoid 'Too many open files' without needing
# to modify ulimit.
#
# Very heavily re-used:
# https://github.com/kemalcr/kemal/blob/master/src/kemal/helpers/helpers.cr
# https://github.com/kemalcr/kemal/blob/master/src/kemal/static_file_handler.cr
#
# Changes:
# - A `send_file` overload is added which supports sending a Slice, file_path, filestat
# - `StaticFileHandler` is patched to cache to and serve from @cached_files
private def multipart(file, env : HTTP::Server::Context)
# See http://httpwg.org/specs/rfc7233.html
fileb = file.size
startb = endb = 0
if match = env.request.headers["Range"].match /bytes=(\d{1,})-(\d{0,})/
startb = match[1].to_i { 0 } if match.size >= 2
endb = match[2].to_i { 0 } if match.size >= 3
end
endb = fileb - 1 if endb == 0
if startb < endb < fileb
content_length = 1 + endb - startb
env.response.status_code = 206
env.response.content_length = content_length
env.response.headers["Accept-Ranges"] = "bytes"
env.response.headers["Content-Range"] = "bytes #{startb}-#{endb}/#{fileb}" # MUST
if startb > 1024
skipped = 0
# file.skip only accepts values less or equal to 1024 (buffer size, undocumented)
until (increase_skipped = skipped + 1024) > startb
file.skip(1024)
skipped = increase_skipped
end
if (skipped_minus_startb = skipped - startb) > 0
file.skip skipped_minus_startb
end
else
file.skip(startb)
end
IO.copy(file, env.response, content_length)
else
env.response.content_length = fileb
env.response.status_code = 200 # Range not satisfable, see 4.4 Note
IO.copy(file, env.response)
end
end
# Set the Content-Disposition to "attachment" with the specified filename,
# instructing the user agents to prompt to save.
private def attachment(env : HTTP::Server::Context, filename : String? = nil, disposition : String? = nil)
disposition = "attachment" if disposition.nil? && filename
if disposition && filename
env.response.headers["Content-Disposition"] = "#{disposition}; filename=\"#{File.basename(filename)}\""
end
end
def send_file(env : HTTP::Server::Context, file_path : String, data : Slice(UInt8), filestat : File::Info, filename : String? = nil, disposition : String? = nil)
config = Kemal.config.serve_static
mime_type = MIME.from_filename(file_path, "application/octet-stream")
env.response.content_type = mime_type
env.response.headers["Accept-Ranges"] = "bytes"
env.response.headers["X-Content-Type-Options"] = "nosniff"
minsize = 860 # http://webmasters.stackexchange.com/questions/31750/what-is-recommended-minimum-object-size-for-gzip-performance-benefits ??
request_headers = env.request.headers
filesize = data.bytesize
attachment(env, filename, disposition)
Kemal.config.static_headers.try(&.call(env, file_path, filestat))
file = IO::Memory.new(data)
if env.request.method == "GET" && env.request.headers.has_key?("Range")
return multipart(file, env)
end
condition = config.is_a?(Hash) && config["gzip"]? == true && filesize > minsize && Kemal::Utils.zip_types(file_path)
if condition && request_headers.includes_word?("Accept-Encoding", "gzip")
env.response.headers["Content-Encoding"] = "gzip"
Compress::Gzip::Writer.open(env.response) do |deflate|
IO.copy(file, deflate)
end
elsif condition && request_headers.includes_word?("Accept-Encoding", "deflate")
env.response.headers["Content-Encoding"] = "deflate"
Compress::Deflate::Writer.open(env.response) do |deflate|
IO.copy(file, deflate)
end
else
env.response.content_length = filesize
IO.copy(file, env.response)
end
return
end
module Kemal
class StaticFileHandler < HTTP::StaticFileHandler
CACHE_LIMIT = 5_000_000 # 5MB
@cached_files = {} of String => {data: Bytes, filestat: File::Info}
def call(context : HTTP::Server::Context)
return call_next(context) if context.request.path.not_nil! == "/"
case context.request.method
when "GET", "HEAD"
else
if @fallthrough
call_next(context)
else
context.response.status = HTTP::Status::METHOD_NOT_ALLOWED
context.response.headers.add("Allow", "GET, HEAD")
end
return
end
config = Kemal.config.serve_static
original_path = context.request.path.not_nil!
request_path = URI.decode_www_form(original_path)
# File path cannot contains '\0' (NUL) because all filesystem I know
# don't accept '\0' character as file name.
if request_path.includes? '\0'
context.response.status = HTTP::Status::BAD_REQUEST
return
end
expanded_path = File.expand_path(request_path, "/")
is_dir_path = if original_path.ends_with?('/') && !expanded_path.ends_with? '/'
expanded_path = expanded_path + '/'
true
else
expanded_path.ends_with? '/'
end
file_path = File.join(@public_dir, expanded_path)
if file = @cached_files[file_path]?
last_modified = file[:filestat].modification_time
add_cache_headers(context.response.headers, last_modified)
if cache_request?(context, last_modified)
context.response.status = HTTP::Status::NOT_MODIFIED
return
end
send_file(context, file_path, file[:data], file[:filestat])
else
file_info = File.info?(file_path)
is_dir = file_info.try &.directory? || false
is_file = file_info.try &.file? || false
if request_path != expanded_path
redirect_to context, expanded_path
elsif is_dir && !is_dir_path
redirect_to context, expanded_path + '/'
end
return call_next(context) if file_info.nil?
if is_dir
if config.is_a?(Hash) && config["dir_listing"] == true
context.response.content_type = "text/html"
directory_listing(context.response, request_path, file_path)
else
call_next(context)
end
elsif is_file
last_modified = file_info.modification_time
add_cache_headers(context.response.headers, last_modified)
if cache_request?(context, last_modified)
context.response.status = HTTP::Status::NOT_MODIFIED
return
end
if @cached_files.sum(&.[1][:data].bytesize) + (size = File.size(file_path)) < CACHE_LIMIT
data = Bytes.new(size)
File.open(file_path, &.read(data))
@cached_files[file_path] = {data: data, filestat: file_info}
send_file(context, file_path, data, file_info)
else
send_file(context, file_path)
end
else # Not a normal file (FIFO/device/socket)
call_next(context)
end
end
end
end
end

52
src/invidious/feeds/trending.cr Normal file
View File

@ -0,0 +1,52 @@
module Invidious::Feeds::Trending
extend self
def fetch(trending_type, region, locale)
region ||= "US"
region = region.upcase
plid = nil
browse_id = ""
case trending_type.try &.downcase
when "gaming"
browse_id = "UCOpNcN46UbXVtpKMrmU4Abg"
params = "Egh0cmVuZGluZw%3D%3D"
when "livestreams"
browse_id = "UC4R8DWoMoI7CAwX8_LjQHig"
params = "EgdsaXZldGFikgEDCKEK"
else
# Livestreams is the default one as Youtube removed
# the aggregated trending page
# https://github.com/iv-org/invidious/issues/5397#issuecomment-3218928458
browse_id = "UC4R8DWoMoI7CAwX8_LjQHig"
params = "EgdsaXZldGFikgEDCKEK"
end
client_config = YoutubeAPI::ClientConfig.new(region: region)
initial_data = YoutubeAPI.browse(browse_id, params: params, client_config: client_config)
items, _ = extract_items(initial_data)
extracted = [] of SearchItem
deduplicate = items.size > 1
items.each do |itm|
if itm.is_a?(Category)
# Ignore the smaller categories, as they generally contain a sponsored
# channel, which brings a lot of noise on the trending page.
# See: https://github.com/iv-org/invidious/issues/2989
next if (itm.contents.size < 24 && deduplicate)
extracted.concat itm.contents.select(SearchItem)
else
extracted << itm
end
end
# Deduplicate items before returning results
return extracted.select(SearchVideo | ProblematicTimelineItem).uniq!(&.id), plid
end
end

270
src/invidious/helpers/errors.cr
View File

@ -2,26 +2,29 @@
# Issue template
# -------------------
macro error_template(*args)
error_template_helper(env, {{args.splat}})
end
module Errors
extend self
def github_details(summary : String, content : String)
details = %(\n<details>)
details += %(\n<summary>#{summary}</summary>)
details += %(\n<p>)
details += %(\n \n```\n)
details += content.strip
details += %(\n```)
details += %(\n</p>)
details += %(\n</details>)
return HTML.escape(details)
end
macro error_template(*args)
Errors.error_template_helper(env, {{args.splat}})
end
def get_issue_template(env : HTTP::Server::Context, exception : Exception) : Tuple(String, String)
issue_title = "#{exception.message} (#{exception.class})"
def github_details(summary : String, content : String)
details = %(\n<details>)
details += %(\n<summary>#{summary}</summary>)
details += %(\n<p>)
details += %(\n \n```\n)
details += content.strip
details += %(\n```)
details += %(\n</p>)
details += %(\n</details>)
return HTML.escape(details)
end
issue_template = <<-TEXT
def get_issue_template(env : HTTP::Server::Context, exception : Exception) : Tuple(String, String)
issue_title = "#{exception.message} (#{exception.class})"
issue_template = <<-TEXT
Title: `#{HTML.escape(issue_title)}`
Date: `#{Time::Format::ISO_8601_DATE_TIME.format(Time.utc)}`
Route: `#{HTML.escape(env.request.resource)}`
@ -29,39 +32,39 @@ def get_issue_template(env : HTTP::Server::Context, exception : Exception) : Tup
TEXT
issue_template += github_details("Backtrace", exception.inspect_with_backtrace)
issue_template += github_details("Backtrace", exception.inspect_with_backtrace)
return issue_title, issue_template
end
def error_template_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception)
if exception.is_a?(InfoException)
return error_template_helper(env, status_code, exception.message || "")
return issue_title, issue_template
end
locale = env.get("preferences").as(Preferences).locale
def error_template_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception)
if exception.is_a?(InfoException)
return error_template_helper(env, status_code, exception.message || "")
end
env.response.content_type = "text/html"
env.response.status_code = status_code
locale = env.get("preferences").as(Preferences).locale
# Unpacking into issue_title, issue_template directly causes a compiler error
# I have no idea why.
issue_template_components = get_issue_template(env, exception)
issue_title, issue_template = issue_template_components
env.response.content_type = "text/html"
env.response.status_code = status_code
# URLs for the error message below
url_faq = "https://github.com/iv-org/documentation/blob/master/docs/faq.md"
url_search_issues = "https://github.com/iv-org/invidious/issues"
url_search_issues += "?q=is:issue+is:open+"
url_search_issues += URI.encode_www_form("[Bug] #{issue_title}")
# Unpacking into issue_title, issue_template directly causes a compiler error
# I have no idea why.
issue_template_components = self.get_issue_template(env, exception)
issue_title, issue_template = issue_template_components
url_switch = "https://redirect.invidious.io" + env.request.resource
# URLs for the error message below
url_faq = "https://github.com/iv-org/documentation/blob/master/docs/faq.md"
url_search_issues = "https://github.com/iv-org/invidious/issues"
url_search_issues += "?q=is:issue+is:open+"
url_search_issues += URI.encode_www_form("[Bug] #{issue_title}")
url_new_issue = "https://github.com/iv-org/invidious/issues/new"
url_new_issue += "?labels=bug&template=bug_report.md&title="
url_new_issue += URI.encode_www_form("[Bug] " + issue_title)
url_switch = "https://redirect.invidious.io" + env.request.resource
error_message = <<-END_HTML
url_new_issue = "https://github.com/iv-org/invidious/issues/new"
url_new_issue += "?labels=bug&template=bug_report.md&title="
url_new_issue += URI.encode_www_form("[Bug] " + issue_title)
error_message = <<-END_HTML
<div class="error_message">
<h2>#{I18n.translate(locale, "crash_page_you_found_a_bug")}</h2>
<br/><br/>
@ -82,116 +85,116 @@ def error_template_helper(env : HTTP::Server::Context, status_code : Int32, exce
</div>
END_HTML
# Don't show the usual "next steps" widget. The same options are
# proposed above the error message, just worded differently.
next_steps = ""
# Don't show the usual "next steps" widget. The same options are
# proposed above the error message, just worded differently.
next_steps = ""
return templated "error"
end
def error_template_helper(env : HTTP::Server::Context, status_code : Int32, message : String)
env.response.content_type = "text/html"
env.response.status_code = status_code
locale = env.get("preferences").as(Preferences).locale
error_message = I18n.translate(locale, message)
next_steps = error_redirect_helper(env)
return templated "error"
end
# -------------------
# Atom feeds
# -------------------
macro error_atom(*args)
error_atom_helper(env, {{args.splat}})
end
def error_atom_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception)
if exception.is_a?(InfoException)
return error_atom_helper(env, status_code, exception.message || "")
return templated "error"
end
env.response.content_type = "application/atom+xml"
env.response.status_code = status_code
def error_template_helper(env : HTTP::Server::Context, status_code : Int32, message : String)
env.response.content_type = "text/html"
env.response.status_code = status_code
return "<error>#{exception.inspect_with_backtrace}</error>"
end
locale = env.get("preferences").as(Preferences).locale
def error_atom_helper(env : HTTP::Server::Context, status_code : Int32, message : String)
env.response.content_type = "application/atom+xml"
env.response.status_code = status_code
error_message = I18n.translate(locale, message)
next_steps = error_redirect_helper(env)
return "<error>#{message}</error>"
end
# -------------------
# JSON
# -------------------
macro error_json(*args)
error_json_helper(env, {{args.splat}})
end
def error_json_helper(
env : HTTP::Server::Context,
status_code : Int32,
exception : Exception,
additional_fields : Hash(String, Object) | Nil = nil,
)
if exception.is_a?(InfoException)
return error_json_helper(env, status_code, exception.message || "", additional_fields)
return templated "error"
end
env.response.content_type = "application/json"
env.response.status_code = status_code
# -------------------
# Atom feeds
# -------------------
error_message = {"error" => exception.message, "errorBacktrace" => exception.inspect_with_backtrace}
if additional_fields
error_message = error_message.merge(additional_fields)
macro error_atom(*args)
Errors.error_atom_helper(env, {{args.splat}})
end
return error_message.to_json
end
def error_atom_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception)
if exception.is_a?(InfoException)
return error_atom_helper(env, status_code, exception.message || "")
end
def error_json_helper(
env : HTTP::Server::Context,
status_code : Int32,
message : String,
additional_fields : Hash(String, Object) | Nil = nil,
)
env.response.content_type = "application/json"
env.response.status_code = status_code
env.response.content_type = "application/atom+xml"
env.response.status_code = status_code
error_message = {"error" => message}
if additional_fields
error_message = error_message.merge(additional_fields)
return "<error>#{exception.inspect_with_backtrace}</error>"
end
return error_message.to_json
end
def error_atom_helper(env : HTTP::Server::Context, status_code : Int32, message : String)
env.response.content_type = "application/atom+xml"
env.response.status_code = status_code
# -------------------
# Redirect
# -------------------
return "<error>#{message}</error>"
end
def error_redirect_helper(env : HTTP::Server::Context)
request_path = env.request.path
# -------------------
# JSON
# -------------------
locale = env.get("preferences").as(Preferences).locale
macro error_json(*args)
Errors.error_json_helper(env, {{args.splat}})
end
if request_path.starts_with?("/search") || request_path.starts_with?("/watch") ||
request_path.starts_with?("/channel") || request_path.starts_with?("/playlist?list=PL")
next_steps_text = I18n.translate(locale, "next_steps_error_message")
refresh = I18n.translate(locale, "next_steps_error_message_refresh")
go_to_youtube = I18n.translate(locale, "next_steps_error_message_go_to_youtube")
switch_instance = I18n.translate(locale, "Switch Invidious Instance")
def error_json_helper(
env : HTTP::Server::Context,
status_code : Int32,
exception : Exception,
additional_fields : Hash(String, Object) | Nil = nil,
)
if exception.is_a?(InfoException)
return error_json_helper(env, status_code, exception.message || "", additional_fields)
end
return <<-END_HTML
env.response.content_type = "application/json"
env.response.status_code = status_code
error_message = {"error" => exception.message, "errorBacktrace" => exception.inspect_with_backtrace}
if additional_fields
error_message = error_message.merge(additional_fields)
end
return error_message.to_json
end
def error_json_helper(
env : HTTP::Server::Context,
status_code : Int32,
message : String,
additional_fields : Hash(String, Object) | Nil = nil,
)
env.response.content_type = "application/json"
env.response.status_code = status_code
error_message = {"error" => message}
if additional_fields
error_message = error_message.merge(additional_fields)
end
return error_message.to_json
end
# -------------------
# Redirect
# -------------------
def error_redirect_helper(env : HTTP::Server::Context)
request_path = env.request.path
locale = env.get("preferences").as(Preferences).locale
if request_path.starts_with?("/search") || request_path.starts_with?("/watch") ||
request_path.starts_with?("/channel") || request_path.starts_with?("/playlist?list=PL")
next_steps_text = I18n.translate(locale, "next_steps_error_message")
refresh = I18n.translate(locale, "next_steps_error_message_refresh")
go_to_youtube = I18n.translate(locale, "next_steps_error_message_go_to_youtube")
switch_instance = I18n.translate(locale, "Switch Invidious Instance")
return <<-END_HTML
<p style="margin-bottom: 4px;">#{next_steps_text}</p>
<ul>
<li>
@ -205,7 +208,8 @@ def error_redirect_helper(env : HTTP::Server::Context)
</li>
</ul>
END_HTML
else
return ""
else
return ""
end
end
end

2
src/invidious/helpers/handlers.cr
View File

@ -83,7 +83,7 @@ class AuthHandler < Kemal::Handler
if token = env.request.headers["Authorization"]?
token = JSON.parse(URI.decode_www_form(token.lchop("Bearer ")))
session = URI.decode_www_form(token["session"].as_s)
scopes, _, _ = validate_request(token, session, env.request, HMAC_KEY, nil)
scopes, _, _ = Invidious::Helpers::Tokens.validate_request(token, session, env.request, HMAC_KEY, nil)
if email = Invidious::Database::SessionIDs.select_email(session)
user = Invidious::Database::Users.select!(email: email)

4
src/invidious/helpers/helpers.cr
View File

@ -22,7 +22,7 @@ struct Annotation
property annotations : String
end
module Helpers
module Invidious::Helpers
extend self
private TEST_IDS = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"}
@ -62,7 +62,7 @@ module Helpers
end
def create_notification_stream(env, topics, connection_channel)
connection = Channel(PQ::Notification).new(8)
connection = ::Channel(PQ::Notification).new(8)
connection_channel.send({true, connection})
locale = env.get("preferences").as(Preferences).locale

10
src/invidious/helpers/serialized_yt_data.cr
View File

@ -53,7 +53,7 @@ struct SearchVideo
xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg")
end
xml.element("p", style: "word-break:break-word;white-space:pre-wrap") { xml.text Helpers.html_to_content(self.description_html) }
xml.element("p", style: "word-break:break-word;white-space:pre-wrap") { xml.text Invidious::Helpers.html_to_content(self.description_html) }
end
end
@ -63,7 +63,7 @@ struct SearchVideo
xml.element("media:title") { xml.text self.title }
xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg",
width: "320", height: "180")
xml.element("media:description") { xml.text Helpers.html_to_content(self.description_html) }
xml.element("media:description") { xml.text Invidious::Helpers.html_to_content(self.description_html) }
end
xml.element("media:community") do
@ -111,7 +111,7 @@ struct SearchVideo
Invidious::JSONify::APIv1.thumbnails(json, self.id)
end
json.field "description", Helpers.html_to_content(self.description_html)
json.field "description", Invidious::Helpers.html_to_content(self.description_html)
json.field "descriptionHtml", self.description_html
json.field "viewCount", self.views
@ -255,7 +255,7 @@ struct SearchChannel
json.field "videoCount", self.video_count
json.field "channelHandle", self.channel_handle
json.field "description", Helpers.html_to_content(self.description_html)
json.field "description", Invidious::Helpers.html_to_content(self.description_html)
json.field "descriptionHtml", self.description_html
end
end
@ -332,7 +332,7 @@ struct ProblematicTimelineItem
end
xml.element("pre") do
get_issue_template(env, @parse_exception)
Errors.get_issue_template(env, @parse_exception)
end
end
end

250
src/invidious/helpers/tokens.cr
View File

@ -1,145 +1,149 @@
require "crypto/subtle"
def generate_token(email, scopes, expire, key)
session = "v1:#{Base64.urlsafe_encode(Random::Secure.random_bytes(32))}"
Invidious::Database::SessionIDs.insert(session, email)
module Invidious::Helpers::Tokens
extend self
token = {
"session" => session,
"scopes" => scopes,
"expire" => expire,
}
def generate_token(email, scopes, expire, key)
session = "v1:#{Base64.urlsafe_encode(Random::Secure.random_bytes(32))}"
Invidious::Database::SessionIDs.insert(session, email)
if !expire
token.delete("expire")
end
token = {
"session" => session,
"scopes" => scopes,
"expire" => expire,
}
token["signature"] = sign_token(key, token)
return token.to_json
end
def generate_response(session, scopes, key, expire = 6.hours, use_nonce = false)
expire = Time.utc + expire
token = {
"session" => session,
"expire" => expire.to_unix,
"scopes" => scopes,
}
if use_nonce
nonce = Random::Secure.hex(16)
Invidious::Database::Nonces.insert(nonce, expire)
token["nonce"] = nonce
end
token["signature"] = sign_token(key, token)
return token.to_json
end
def sign_token(key, hash)
string_to_sign = [] of String
# TODO: figure out which "key" variable is used
# Ameba reports a warning for "Lint/ShadowingOuterLocalVar" on this
# variable, but it's preferable to not touch that (works fine atm).
hash.each do |key, value|
next if key == "signature"
if value.is_a?(JSON::Any) && value.as_a?
value = value.as_a.map(&.as_s)
if !expire
token.delete("expire")
end
case value
when Array
string_to_sign << "#{key}=#{value.sort.join(",")}"
when Tuple
string_to_sign << "#{key}=#{value.to_a.sort.join(",")}"
else
string_to_sign << "#{key}=#{value}"
token["signature"] = sign_token(key, token)
return token.to_json
end
def generate_response(session, scopes, key, expire = 6.hours, use_nonce = false)
expire = Time.utc + expire
token = {
"session" => session,
"expire" => expire.to_unix,
"scopes" => scopes,
}
if use_nonce
nonce = Random::Secure.hex(16)
Invidious::Database::Nonces.insert(nonce, expire)
token["nonce"] = nonce
end
token["signature"] = sign_token(key, token)
return token.to_json
end
string_to_sign = string_to_sign.sort.join("\n")
return Base64.urlsafe_encode(OpenSSL::HMAC.digest(:sha256, key, string_to_sign)).strip
end
def sign_token(key, hash)
string_to_sign = [] of String
def validate_request(token, session, request, key, locale = nil)
case token
when String
token = JSON.parse(URI.decode_www_form(token)).as_h
when JSON::Any
token = token.as_h
when Nil
raise InfoException.new("Hidden field \"token\" is a required field")
# TODO: figure out which "key" variable is used
# Ameba reports a warning for "Lint/ShadowingOuterLocalVar" on this
# variable, but it's preferable to not touch that (works fine atm).
hash.each do |key, value|
next if key == "signature"
if value.is_a?(JSON::Any) && value.as_a?
value = value.as_a.map(&.as_s)
end
case value
when Array
string_to_sign << "#{key}=#{value.sort.join(",")}"
when Tuple
string_to_sign << "#{key}=#{value.to_a.sort.join(",")}"
else
string_to_sign << "#{key}=#{value}"
end
end
string_to_sign = string_to_sign.sort.join("\n")
return Base64.urlsafe_encode(OpenSSL::HMAC.digest(:sha256, key, string_to_sign)).strip
end
expire = token["expire"]?.try &.as_i
if expire.try &.< Time.utc.to_unix
raise InfoException.new("Token is expired, please try again")
end
def validate_request(token, session, request, key, locale = nil)
case token
when String
token = JSON.parse(URI.decode_www_form(token)).as_h
when JSON::Any
token = token.as_h
when Nil
raise InfoException.new("Hidden field \"token\" is a required field")
end
if token["session"] != session
raise InfoException.new("Erroneous token")
end
expire = token["expire"]?.try &.as_i
if expire.try &.< Time.utc.to_unix
raise InfoException.new("Token is expired, please try again")
end
scopes = token["scopes"].as_a.map(&.as_s)
scope = "#{request.method}:#{request.path.lchop("/api/v1/auth/").lstrip("/")}"
if !scopes_include_scope(scopes, scope)
raise InfoException.new("Invalid scope")
end
if !Crypto::Subtle.constant_time_compare(token["signature"].to_s, sign_token(key, token))
raise InfoException.new("Invalid signature")
end
if token["nonce"]? && (nonce = Invidious::Database::Nonces.select(token["nonce"].as_s))
if nonce[1] > Time.utc
Invidious::Database::Nonces.update_set_expired(nonce[0])
else
if token["session"] != session
raise InfoException.new("Erroneous token")
end
end
return {scopes, expire, token["signature"].as_s}
end
def scope_includes_scope(scope, subset)
methods, endpoint = scope.split(":")
methods = methods.split(";").map(&.upcase).reject(&.empty?).sort!
endpoint = endpoint.downcase
subset_methods, subset_endpoint = subset.split(":")
subset_methods = subset_methods.split(";").map(&.upcase).sort!
subset_endpoint = subset_endpoint.downcase
if methods.empty?
methods = %w(GET POST PUT HEAD DELETE PATCH OPTIONS)
end
if methods & subset_methods != subset_methods
return false
end
if endpoint.ends_with?("*") && !subset_endpoint.starts_with? endpoint.rchop("*")
return false
end
if !endpoint.ends_with?("*") && subset_endpoint != endpoint
return false
end
return true
end
def scopes_include_scope(scopes, subset)
scopes.each do |scope|
if scope_includes_scope(scope, subset)
return true
scopes = token["scopes"].as_a.map(&.as_s)
scope = "#{request.method}:#{request.path.lchop("/api/v1/auth/").lstrip("/")}"
if !scopes_include_scope(scopes, scope)
raise InfoException.new("Invalid scope")
end
if !Crypto::Subtle.constant_time_compare(token["signature"].to_s, sign_token(key, token))
raise InfoException.new("Invalid signature")
end
if token["nonce"]? && (nonce = Invidious::Database::Nonces.select(token["nonce"].as_s))
if nonce[1] > Time.utc
Invidious::Database::Nonces.update_set_expired(nonce[0])
else
raise InfoException.new("Erroneous token")
end
end
return {scopes, expire, token["signature"].as_s}
end
return false
def scope_includes_scope(scope, subset)
methods, endpoint = scope.split(":")
methods = methods.split(";").map(&.upcase).reject(&.empty?).sort!
endpoint = endpoint.downcase
subset_methods, subset_endpoint = subset.split(":")
subset_methods = subset_methods.split(";").map(&.upcase).sort!
subset_endpoint = subset_endpoint.downcase
if methods.empty?
methods = %w(GET POST PUT HEAD DELETE PATCH OPTIONS)
end
if methods & subset_methods != subset_methods
return false
end
if endpoint.ends_with?("*") && !subset_endpoint.starts_with? endpoint.rchop("*")
return false
end
if !endpoint.ends_with?("*") && subset_endpoint != endpoint
return false
end
return true
end
def scopes_include_scope(scopes, subset)
scopes.each do |scope|
if self.scope_includes_scope(scope, subset)
return true
end
end
return false
end
end

2
src/invidious/jobs/refresh_channels_job.cr
View File

@ -30,7 +30,7 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob
spawn do
begin
LOGGER.trace("RefreshChannelsJob: #{id} fiber : Fetching channel")
channel = fetch_channel(id, pull_all_videos: CONFIG.full_refresh)
channel = Invidious::Channels::Channels.fetch_channel(id, pull_all_videos: CONFIG.full_refresh)
lim_fibers = max_fibers

2
src/invidious/jobs/refresh_feeds_job.cr
View File

@ -54,7 +54,7 @@ class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob
# While iterating through, we may have an email stored from a deleted account
if db.query_one?("SELECT true FROM users WHERE email = $1", email, as: Bool)
LOGGER.info("RefreshFeedsJob: CREATE #{view_name}")
db.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(email)}")
db.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{Invidious::User::Users::MATERIALIZED_VIEW_SQL.call(email)}")
db.exec("UPDATE users SET feed_needs_update = false WHERE email = $1", email)
end
rescue ex

579
src/invidious/playlists.cr
View File

@ -1,579 +0,0 @@
struct PlaylistVideo
include DB::Serializable
property title : String
property id : String
property author : String
property ucid : String
property length_seconds : Int32
property published : Time
property plid : String
property index : Int64
property live_now : Bool
def to_xml(xml : XML::Builder)
xml.element("entry") do
xml.element("id") { xml.text "yt:video:#{self.id}" }
xml.element("yt:videoId") { xml.text self.id }
xml.element("yt:channelId") { xml.text self.ucid }
xml.element("title") { xml.text self.title }
xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?v=#{self.id}")
xml.element("author") do
xml.element("name") { xml.text self.author }
xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" }
end
xml.element("content", type: "xhtml") do
xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do
xml.element("a", href: "#{HOST_URL}/watch?v=#{self.id}") do
xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg")
end
end
end
xml.element("published") { xml.text self.published.to_s("%Y-%m-%dT%H:%M:%S%:z") }
xml.element("media:group") do
xml.element("media:title") { xml.text self.title }
xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg",
width: "320", height: "180")
end
end
end
def to_xml(_xml : Nil = nil)
XML.build { |xml| to_xml(xml) }
end
def to_json(locale : String?, json : JSON::Builder)
to_json(json)
end
def to_json(json : JSON::Builder, index : Int32? = nil)
json.object do
json.field "type", "video"
json.field "title", self.title
json.field "videoId", self.id
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "videoThumbnails" do
Invidious::JSONify::APIv1.thumbnails(json, self.id)
end
if index
json.field "index", index
json.field "indexId", self.index.to_u64.to_s(16).upcase
else
json.field "index", self.index
end
json.field "lengthSeconds", self.length_seconds
json.field "liveNow", self.live_now
end
end
def to_json(_json : Nil, index : Int32? = nil)
JSON.build { |json| to_json(json, index: index) }
end
end
struct Playlist
include DB::Serializable
property title : String
property id : String
property author : String
property author_thumbnail : String
property ucid : String
property description : String
property description_html : String
property video_count : Int32
property views : Int64
property updated : Time
property thumbnail : String?
property subtitle : String?
def to_json(offset, json : JSON::Builder, video_id : String? = nil)
json.object do
json.field "type", "playlist"
json.field "title", self.title
json.field "playlistId", self.id
json.field "playlistThumbnail", self.thumbnail
json.field "author", self.author
json.field "authorId", self.ucid
if !self.ucid.empty?
json.field "authorUrl", "/channel/#{self.ucid}"
else
json.field "authorUrl", ""
end
json.field "subtitle", self.subtitle
json.field "authorThumbnails" do
json.array do
qualities = {32, 48, 76, 100, 176, 512}
qualities.each do |quality|
json.object do
json.field "url", self.author_thumbnail.not_nil!.gsub(/=\d+/, "=s#{quality}")
json.field "width", quality
json.field "height", quality
end
end
end
end
json.field "description", self.description
json.field "descriptionHtml", self.description_html
json.field "videoCount", self.video_count
json.field "viewCount", self.views
json.field "updated", self.updated.to_unix
json.field "isListed", self.privacy.public?
json.field "videos" do
json.array do
videos = get_playlist_videos(self, offset: offset, video_id: video_id)
videos.each do |video|
video.to_json(json)
end
end
end
end
end
def to_json(offset, _json : Nil = nil, video_id : String? = nil)
JSON.build do |json|
to_json(offset, json, video_id: video_id)
end
end
def privacy
PlaylistPrivacy::Public
end
end
enum PlaylistPrivacy
Public = 0
Unlisted = 1
Private = 2
end
struct InvidiousPlaylist
include DB::Serializable
property title : String
property id : String
property author : String
property description : String = ""
property video_count : Int32
property created : Time
property updated : Time
@[DB::Field(converter: InvidiousPlaylist::PlaylistPrivacyConverter)]
property privacy : PlaylistPrivacy = PlaylistPrivacy::Private
property index : Array(Int64)
@[DB::Field(ignore: true)]
property thumbnail_id : String?
module PlaylistPrivacyConverter
def self.from_rs(rs)
return PlaylistPrivacy.parse(String.new(rs.read(Slice(UInt8))))
end
end
def to_json(offset, json : JSON::Builder, video_id : String? = nil)
json.object do
json.field "type", "invidiousPlaylist"
json.field "title", self.title
json.field "playlistId", self.id
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", nil
json.field "authorThumbnails", [] of String
json.field "description", Helpers.html_to_content(self.description_html)
json.field "descriptionHtml", self.description_html
json.field "videoCount", self.video_count
json.field "viewCount", self.views
json.field "updated", self.updated.to_unix
json.field "isListed", self.privacy.public?
json.field "videos" do
json.array do
if (!offset || offset == 0) && !video_id.nil?
index = Invidious::Database::PlaylistVideos.select_index(self.id, video_id)
offset = self.index.index(index) || 0
end
videos = get_playlist_videos(self, offset: offset, video_id: video_id)
videos.each_with_index do |video, idx|
video.to_json(json, offset + idx)
end
end
end
end
end
def to_json(offset, _json : Nil = nil, video_id : String? = nil)
JSON.build do |json|
to_json(offset, json, video_id: video_id)
end
end
def thumbnail
# TODO: Get playlist thumbnail from playlist data rather than first video
@thumbnail_id ||= Invidious::Database::PlaylistVideos.select_one_id(self.id, self.index) || "-----------"
"/vi/#{@thumbnail_id}/mqdefault.jpg"
end
def author_thumbnail
nil
end
def ucid
nil
end
def views
0_i64
end
def description_html
HTML.escape(self.description)
end
end
def create_playlist(title, privacy, user)
plid = "IVPL#{Random::Secure.urlsafe_base64(24)[0, 31]}"
playlist = InvidiousPlaylist.new({
title: title.byte_slice(0, 150),
id: plid,
author: user.email,
description: "", # Max 5000 characters
video_count: 0,
created: Time.utc,
updated: Time.utc,
privacy: privacy,
index: [] of Int64,
})
Invidious::Database::Playlists.insert(playlist)
return playlist
end
def subscribe_playlist(user, playlist)
playlist = InvidiousPlaylist.new({
title: playlist.title[..150],
id: playlist.id,
author: user.email,
description: "", # Max 5000 characters
video_count: playlist.video_count,
created: Time.utc,
updated: playlist.updated,
privacy: PlaylistPrivacy::Private,
index: [] of Int64,
})
Invidious::Database::Playlists.insert(playlist)
return playlist
end
def produce_playlist_continuation(id, index)
if id.starts_with? "UC"
id = "UU" + id.lchop("UC")
end
plid = "VL" + id
# Emulate a "request counter" increment, to make perfectly valid
# ctokens, even if at the time of writing, it's ignored by youtube.
request_count = (index / 100).to_i64 || 1_i64
data = {"1:varint" => index.to_i64}
.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i, padding: false) }
object = {
"80226972:embedded" => {
"2:string" => plid,
"3:base64" => {
"1:varint" => request_count,
"15:string" => "PT:#{data}",
"104:embedded" => {"1:0:varint" => 0_i64},
},
"35:string" => id,
},
}
continuation = object.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
return continuation
end
def get_playlist(plid : String)
if plid.starts_with? "IV"
if playlist = Invidious::Database::Playlists.select(id: plid)
return playlist
else
raise NotFoundException.new("Playlist does not exist.")
end
else
return fetch_playlist(plid)
end
end
def fetch_playlist(plid : String)
if plid.starts_with? "UC"
plid = "UU#{plid.lchop("UC")}"
end
initial_data = YoutubeAPI.browse("VL" + plid, params: "")
playlist_sidebar_renderer = initial_data.dig?("sidebar", "playlistSidebarRenderer", "items")
raise InfoException.new("Could not extract playlistSidebarRenderer.") if !playlist_sidebar_renderer
playlist_info = playlist_sidebar_renderer.dig?(0, "playlistSidebarPrimaryInfoRenderer")
raise InfoException.new("Could not extract playlist info") if !playlist_info
title = playlist_info.dig?("title", "runs", 0, "text").try &.as_s || ""
desc_item = playlist_info["description"]?
description_txt = desc_item.try &.["runs"]?.try &.as_a
.map(&.["text"].as_s).join("") || desc_item.try &.["simpleText"]?.try &.as_s || ""
description_html = desc_item.try &.["runs"]?.try &.as_a
.try { |run| content_to_comment_html(run).try &.to_s } || "<p></p>"
thumbnail = playlist_info.dig?(
"thumbnailRenderer", "playlistVideoThumbnailRenderer",
"thumbnail", "thumbnails", 0, "url"
).try &.as_s || playlist_info.dig?(
"thumbnailRenderer", "playlistCustomThumbnailRenderer",
"thumbnail", "thumbnails", 0, "url"
).try &.as_s
views = 0_i64
updated = Time.utc
video_count = 0
subtitle = extract_text(initial_data.dig?("header", "playlistHeaderRenderer", "subtitle"))
playlist_info["stats"]?.try &.as_a.each do |stat|
text = stat["runs"]?.try &.as_a.map(&.["text"].as_s).join("") || stat["simpleText"]?.try &.as_s
next if !text
if text.includes? "video"
video_count = text.gsub(/\D/, "").to_i? || 0
elsif text.includes? "episode"
video_count = text.gsub(/\D/, "").to_i? || 0
elsif text.includes? "view"
views = text.gsub(/\D/, "").to_i64? || 0_i64
elsif !text.includes? "Pay to watch"
updated = decode_date(text.lchop("Last updated on ").lchop("Updated "))
end
end
if playlist_sidebar_renderer.size < 2
author = ""
author_thumbnail = ""
ucid = ""
else
author_info = playlist_sidebar_renderer[1].dig?(
"playlistSidebarSecondaryInfoRenderer", "videoOwner", "videoOwnerRenderer"
)
raise InfoException.new("Could not extract author info") if !author_info
author = author_info.dig?("title", "runs", 0, "text").try &.as_s || ""
author_thumbnail = author_info.dig?("thumbnail", "thumbnails", 0, "url").try &.as_s || ""
ucid = author_info.dig?("title", "runs", 0, "navigationEndpoint", "browseEndpoint", "browseId").try &.as_s || ""
end
return Playlist.new({
title: title,
id: plid,
author: author,
author_thumbnail: author_thumbnail,
ucid: ucid,
description: description_txt,
description_html: description_html,
video_count: video_count,
views: views,
updated: updated,
thumbnail: thumbnail,
subtitle: subtitle,
})
end
def get_playlist_videos(playlist : InvidiousPlaylist | Playlist, offset : Int32, video_id = nil)
# Show empty playlist if requested page is out of range
# (e.g, when a new playlist has been created, offset will be negative)
if offset >= playlist.video_count || offset < 0
return [] of PlaylistVideo
end
if playlist.is_a? InvidiousPlaylist
Invidious::Database::PlaylistVideos.select(playlist.id, playlist.index, offset, limit: 100)
else
if video_id
initial_data = YoutubeAPI.next({
"videoId" => video_id,
"playlistId" => playlist.id,
})
offset = initial_data.dig?("contents", "twoColumnWatchNextResults", "playlist", "playlist", "currentIndex").try &.as_i || offset
end
videos = [] of PlaylistVideo | ProblematicTimelineItem
until videos.size >= 200 || videos.size == playlist.video_count || offset >= playlist.video_count
# 100 videos per request
ctoken = produce_playlist_continuation(playlist.id, offset)
initial_data = YoutubeAPI.browse(ctoken)
videos += extract_playlist_videos(playlist.id, initial_data)
offset += 100
end
return videos
end
end
# TODO (2026-06-24): Migrate this function to use parsers instead, as it uses,
# the same LockupViewModel used in Channel videos and Youtube playlists that
# appears on searches (Invidious /search endpoint).
# Related to https://github.com/iv-org/invidious/pull/5736
def extract_playlist_videos(playlist_id : String, initial_data : Hash(String, JSON::Any))
videos = [] of PlaylistVideo | ProblematicTimelineItem
if initial_data["contents"]?
tabs = initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]
tabs_renderer = tabs.as_a.select(&.["tabRenderer"]["selected"]?.try &.as_bool)[0]["tabRenderer"]
# Watch out the two versions, with and without "s"
if tabs_renderer["contents"]? || tabs_renderer["content"]?
# Initial playlist data
tabs_contents = tabs_renderer.["contents"]? || tabs_renderer.["content"]
list_renderer = tabs_contents.["sectionListRenderer"]["contents"][0]
contents = list_renderer.["itemSectionRenderer"]["contents"].as_a
else
# Continuation data
contents = initial_data["onResponseReceivedActions"][0]?
.try &.["appendContinuationItemsAction"]["continuationItems"].as_a
end
else
contents = initial_data["response"]?.try &.["continuationContents"]["playlistVideoListContinuation"]["contents"].as_a
end
contents.try &.each do |item|
if i = item["lockupViewModel"]?
thumbnail_view_model = i.dig?(
"contentImage", "thumbnailViewModel"
)
watch_endpoint = i.dig?("rendererContext", "commandContext", "onTap", "innertubeCommand", "watchEndpoint")
video_id = watch_endpoint.try &.["videoId"]?.try &.as_s
plid = watch_endpoint.try &.["playlistId"]?.try &.as_s || playlist_id
index = watch_endpoint.try &.["index"]?.try &.as_i64
metadata = i["metadata"]?
lockup_metadata_view_model = metadata.try &.dig?("lockupMetadataViewModel")
title = lockup_metadata_view_model.try &.dig?("title", "content").try &.as_s
lockup_metadata = lockup_metadata_view_model.try &.dig?("metadata")
metadata_rows = lockup_metadata.try &.dig?("contentMetadataViewModel", "metadataRows").try &.as_a
# Find the metadataParts with commandRuns inside, which contains author
# information.
metadata_parts = metadata_rows.try &.find { |row|
parts = row["metadataParts"]?.try &.as_a
parts && parts.any? { |item2| item2.dig?("text", "commandRuns").try &.as_a }
}.try &.["metadataParts"].as_a
if author_info = metadata_parts.try &.find(&.dig?("text", "commandRuns"))
.try &.["text"]
author = author_info["content"].as_s
ucid = author_info.dig?("commandRuns", 0, "onTap", "innertubeCommand", "browseEndpoint", "browseId")
.try &.as_s
end
length = thumbnail_view_model.try &.dig?("overlays", 0, "thumbnailBottomOverlayViewModel", "badges", 0, "thumbnailBadgeViewModel", "text").try &.as_s
length_seconds = decode_length_seconds(length) if length
live = false
if !length_seconds
live = true
length_seconds = 0
end
videos << PlaylistVideo.new({
title: title || "",
id: video_id || "",
author: author || "",
ucid: ucid || "",
length_seconds: length_seconds,
published: Time.utc,
plid: plid,
live_now: live,
index: index || -1_i64,
})
end
rescue ex
videos << ProblematicTimelineItem.new(parse_exception: ex)
end
return videos
end
def template_playlist(playlist, listen)
html = <<-END_HTML
<h3>
<a href="/playlist?list=#{playlist["playlistId"]}">
#{playlist["title"]}
</a>
</h3>
<div class="pure-menu pure-menu-scrollable playlist-restricted">
<ol class="pure-menu-list">
END_HTML
playlist["videos"].as_a.each do |video|
html += <<-END_HTML
<li class="pure-menu-item" id="#{video["videoId"]}">
<a href="/watch?v=#{video["videoId"]}&list=#{playlist["playlistId"]}&index=#{video["index"]}#{listen ? "&listen=1" : ""}">
<div class="thumbnail">
<img loading="lazy" class="thumbnail" src="/vi/#{video["videoId"]}/mqdefault.jpg" alt="" />
<p class="length">#{recode_length_seconds(video["lengthSeconds"].as_i)}</p>
</div>
<p style="width:100%">#{video["title"]}</p>
<p>
<b style="width:100%">#{video["author"]}</b>
</p>
</a>
</li>
END_HTML
end
html += <<-END_HTML
</ol>
</div>
<hr>
END_HTML
html
end

2
src/invidious/mixes.cr → src/invidious/playlists/mixes.cr
View File

@ -27,7 +27,7 @@ def fetch_mix(rdid, video_id, cookies = nil, locale = nil)
video_id = "CvFH_6DNRCY" if rdid.starts_with? "OLAK5uy_"
response = YT_POOL.client &.get("/watch?v=#{video_id}&list=#{rdid}&gl=US&hl=en", headers)
initial_data = Helpers.extract_initial_data(response.body)
initial_data = Invidious::Helpers.extract_initial_data(response.body)
if !initial_data["contents"]["twoColumnWatchNextResults"]["playlist"]?
raise InfoException.new("Could not create mix.")

583
src/invidious/playlists/playlists.cr Normal file
View File

@ -0,0 +1,583 @@
struct PlaylistVideo
include DB::Serializable
property title : String
property id : String
property author : String
property ucid : String
property length_seconds : Int32
property published : Time
property plid : String
property index : Int64
property live_now : Bool
def to_xml(xml : XML::Builder)
xml.element("entry") do
xml.element("id") { xml.text "yt:video:#{self.id}" }
xml.element("yt:videoId") { xml.text self.id }
xml.element("yt:channelId") { xml.text self.ucid }
xml.element("title") { xml.text self.title }
xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?v=#{self.id}")
xml.element("author") do
xml.element("name") { xml.text self.author }
xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" }
end
xml.element("content", type: "xhtml") do
xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do
xml.element("a", href: "#{HOST_URL}/watch?v=#{self.id}") do
xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg")
end
end
end
xml.element("published") { xml.text self.published.to_s("%Y-%m-%dT%H:%M:%S%:z") }
xml.element("media:group") do
xml.element("media:title") { xml.text self.title }
xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg",
width: "320", height: "180")
end
end
end
def to_xml(_xml : Nil = nil)
XML.build { |xml| to_xml(xml) }
end
def to_json(locale : String?, json : JSON::Builder)
to_json(json)
end
def to_json(json : JSON::Builder, index : Int32? = nil)
json.object do
json.field "type", "video"
json.field "title", self.title
json.field "videoId", self.id
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "videoThumbnails" do
Invidious::JSONify::APIv1.thumbnails(json, self.id)
end
if index
json.field "index", index
json.field "indexId", self.index.to_u64.to_s(16).upcase
else
json.field "index", self.index
end
json.field "lengthSeconds", self.length_seconds
json.field "liveNow", self.live_now
end
end
def to_json(_json : Nil, index : Int32? = nil)
JSON.build { |json| to_json(json, index: index) }
end
end
struct Playlist
include DB::Serializable
property title : String
property id : String
property author : String
property author_thumbnail : String
property ucid : String
property description : String
property description_html : String
property video_count : Int32
property views : Int64
property updated : Time
property thumbnail : String?
property subtitle : String?
def to_json(offset, json : JSON::Builder, video_id : String? = nil)
json.object do
json.field "type", "playlist"
json.field "title", self.title
json.field "playlistId", self.id
json.field "playlistThumbnail", self.thumbnail
json.field "author", self.author
json.field "authorId", self.ucid
if !self.ucid.empty?
json.field "authorUrl", "/channel/#{self.ucid}"
else
json.field "authorUrl", ""
end
json.field "subtitle", self.subtitle
json.field "authorThumbnails" do
json.array do
qualities = {32, 48, 76, 100, 176, 512}
qualities.each do |quality|
json.object do
json.field "url", self.author_thumbnail.not_nil!.gsub(/=\d+/, "=s#{quality}")
json.field "width", quality
json.field "height", quality
end
end
end
end
json.field "description", self.description
json.field "descriptionHtml", self.description_html
json.field "videoCount", self.video_count
json.field "viewCount", self.views
json.field "updated", self.updated.to_unix
json.field "isListed", self.privacy.public?
json.field "videos" do
json.array do
videos = Invidious::Playlists::Playlists.get_playlist_videos(self, offset: offset, video_id: video_id)
videos.each do |video|
video.to_json(json)
end
end
end
end
end
def to_json(offset, _json : Nil = nil, video_id : String? = nil)
JSON.build do |json|
to_json(offset, json, video_id: video_id)
end
end
def privacy
PlaylistPrivacy::Public
end
end
enum PlaylistPrivacy
Public = 0
Unlisted = 1
Private = 2
end
struct InvidiousPlaylist
include DB::Serializable
property title : String
property id : String
property author : String
property description : String = ""
property video_count : Int32
property created : Time
property updated : Time
@[DB::Field(converter: InvidiousPlaylist::PlaylistPrivacyConverter)]
property privacy : PlaylistPrivacy = PlaylistPrivacy::Private
property index : Array(Int64)
@[DB::Field(ignore: true)]
property thumbnail_id : String?
module PlaylistPrivacyConverter
def self.from_rs(rs)
return PlaylistPrivacy.parse(String.new(rs.read(Slice(UInt8))))
end
end
def to_json(offset, json : JSON::Builder, video_id : String? = nil)
json.object do
json.field "type", "invidiousPlaylist"
json.field "title", self.title
json.field "playlistId", self.id
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", nil
json.field "authorThumbnails", [] of String
json.field "description", Invidious::Helpers.html_to_content(self.description_html)
json.field "descriptionHtml", self.description_html
json.field "videoCount", self.video_count
json.field "viewCount", self.views
json.field "updated", self.updated.to_unix
json.field "isListed", self.privacy.public?
json.field "videos" do
json.array do
if (!offset || offset == 0) && !video_id.nil?
index = Invidious::Database::PlaylistVideos.select_index(self.id, video_id)
offset = self.index.index(index) || 0
end
videos = Invidious::Playlists::Playlists.get_playlist_videos(self, offset: offset, video_id: video_id)
videos.each_with_index do |video, idx|
video.to_json(json, offset + idx)
end
end
end
end
end
def to_json(offset, _json : Nil = nil, video_id : String? = nil)
JSON.build do |json|
to_json(offset, json, video_id: video_id)
end
end
def thumbnail
# TODO: Get playlist thumbnail from playlist data rather than first video
@thumbnail_id ||= Invidious::Database::PlaylistVideos.select_one_id(self.id, self.index) || "-----------"
"/vi/#{@thumbnail_id}/mqdefault.jpg"
end
def author_thumbnail
nil
end
def ucid
nil
end
def views
0_i64
end
def description_html
HTML.escape(self.description)
end
end
module Invidious::Playlists::Playlists
extend self
def create_playlist(title, privacy, user)
plid = "IVPL#{Random::Secure.urlsafe_base64(24)[0, 31]}"
playlist = InvidiousPlaylist.new({
title: title.byte_slice(0, 150),
id: plid,
author: user.email,
description: "", # Max 5000 characters
video_count: 0,
created: Time.utc,
updated: Time.utc,
privacy: privacy,
index: [] of Int64,
})
Invidious::Database::Playlists.insert(playlist)
return playlist
end
def subscribe_playlist(user, playlist)
playlist = InvidiousPlaylist.new({
title: playlist.title[..150],
id: playlist.id,
author: user.email,
description: "", # Max 5000 characters
video_count: playlist.video_count,
created: Time.utc,
updated: playlist.updated,
privacy: PlaylistPrivacy::Private,
index: [] of Int64,
})
Invidious::Database::Playlists.insert(playlist)
return playlist
end
def produce_playlist_continuation(id, index)
if id.starts_with? "UC"
id = "UU" + id.lchop("UC")
end
plid = "VL" + id
# Emulate a "request counter" increment, to make perfectly valid
# ctokens, even if at the time of writing, it's ignored by youtube.
request_count = (index / 100).to_i64 || 1_i64
data = {"1:varint" => index.to_i64}
.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i, padding: false) }
object = {
"80226972:embedded" => {
"2:string" => plid,
"3:base64" => {
"1:varint" => request_count,
"15:string" => "PT:#{data}",
"104:embedded" => {"1:0:varint" => 0_i64},
},
"35:string" => id,
},
}
continuation = object.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
return continuation
end
def get_playlist(plid : String)
if plid.starts_with? "IV"
if playlist = Invidious::Database::Playlists.select(id: plid)
return playlist
else
raise NotFoundException.new("Playlist does not exist.")
end
else
return fetch_playlist(plid)
end
end
def fetch_playlist(plid : String)
if plid.starts_with? "UC"
plid = "UU#{plid.lchop("UC")}"
end
initial_data = YoutubeAPI.browse("VL" + plid, params: "")
playlist_sidebar_renderer = initial_data.dig?("sidebar", "playlistSidebarRenderer", "items")
raise InfoException.new("Could not extract playlistSidebarRenderer.") if !playlist_sidebar_renderer
playlist_info = playlist_sidebar_renderer.dig?(0, "playlistSidebarPrimaryInfoRenderer")
raise InfoException.new("Could not extract playlist info") if !playlist_info
title = playlist_info.dig?("title", "runs", 0, "text").try &.as_s || ""
desc_item = playlist_info["description"]?
description_txt = desc_item.try &.["runs"]?.try &.as_a
.map(&.["text"].as_s).join("") || desc_item.try &.["simpleText"]?.try &.as_s || ""
description_html = desc_item.try &.["runs"]?.try &.as_a
.try { |run| content_to_comment_html(run).try &.to_s } || "<p></p>"
thumbnail = playlist_info.dig?(
"thumbnailRenderer", "playlistVideoThumbnailRenderer",
"thumbnail", "thumbnails", 0, "url"
).try &.as_s || playlist_info.dig?(
"thumbnailRenderer", "playlistCustomThumbnailRenderer",
"thumbnail", "thumbnails", 0, "url"
).try &.as_s
views = 0_i64
updated = Time.utc
video_count = 0
subtitle = extract_text(initial_data.dig?("header", "playlistHeaderRenderer", "subtitle"))
playlist_info["stats"]?.try &.as_a.each do |stat|
text = stat["runs"]?.try &.as_a.map(&.["text"].as_s).join("") || stat["simpleText"]?.try &.as_s
next if !text
if text.includes? "video"
video_count = text.gsub(/\D/, "").to_i? || 0
elsif text.includes? "episode"
video_count = text.gsub(/\D/, "").to_i? || 0
elsif text.includes? "view"
views = text.gsub(/\D/, "").to_i64? || 0_i64
elsif !text.includes? "Pay to watch"
updated = decode_date(text.lchop("Last updated on ").lchop("Updated "))
end
end
if playlist_sidebar_renderer.size < 2
author = ""
author_thumbnail = ""
ucid = ""
else
author_info = playlist_sidebar_renderer[1].dig?(
"playlistSidebarSecondaryInfoRenderer", "videoOwner", "videoOwnerRenderer"
)
raise InfoException.new("Could not extract author info") if !author_info
author = author_info.dig?("title", "runs", 0, "text").try &.as_s || ""
author_thumbnail = author_info.dig?("thumbnail", "thumbnails", 0, "url").try &.as_s || ""
ucid = author_info.dig?("title", "runs", 0, "navigationEndpoint", "browseEndpoint", "browseId").try &.as_s || ""
end
return Playlist.new({
title: title,
id: plid,
author: author,
author_thumbnail: author_thumbnail,
ucid: ucid,
description: description_txt,
description_html: description_html,
video_count: video_count,
views: views,
updated: updated,
thumbnail: thumbnail,
subtitle: subtitle,
})
end
def get_playlist_videos(playlist : InvidiousPlaylist | Playlist, offset : Int32, video_id = nil)
# Show empty playlist if requested page is out of range
# (e.g, when a new playlist has been created, offset will be negative)
if offset >= playlist.video_count || offset < 0
return [] of PlaylistVideo
end
if playlist.is_a? InvidiousPlaylist
Invidious::Database::PlaylistVideos.select(playlist.id, playlist.index, offset, limit: 100)
else
if video_id
initial_data = YoutubeAPI.next({
"videoId" => video_id,
"playlistId" => playlist.id,
})
offset = initial_data.dig?("contents", "twoColumnWatchNextResults", "playlist", "playlist", "currentIndex").try &.as_i || offset
end
videos = [] of PlaylistVideo | ProblematicTimelineItem
until videos.size >= 200 || videos.size == playlist.video_count || offset >= playlist.video_count
# 100 videos per request
ctoken = produce_playlist_continuation(playlist.id, offset)
initial_data = YoutubeAPI.browse(ctoken)
videos += extract_playlist_videos(playlist.id, initial_data)
offset += 100
end
return videos
end
end
# TODO (2026-06-24): Migrate this function to use parsers instead, as it uses,
# the same LockupViewModel used in Channel videos and Youtube playlists that
# appears on searches (Invidious /search endpoint).
# Related to https://github.com/iv-org/invidious/pull/5736
def extract_playlist_videos(playlist_id : String, initial_data : Hash(String, JSON::Any))
videos = [] of PlaylistVideo | ProblematicTimelineItem
if initial_data["contents"]?
tabs = initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]
tabs_renderer = tabs.as_a.select(&.["tabRenderer"]["selected"]?.try &.as_bool)[0]["tabRenderer"]
# Watch out the two versions, with and without "s"
if tabs_renderer["contents"]? || tabs_renderer["content"]?
# Initial playlist data
tabs_contents = tabs_renderer.["contents"]? || tabs_renderer.["content"]
list_renderer = tabs_contents.["sectionListRenderer"]["contents"][0]
contents = list_renderer.["itemSectionRenderer"]["contents"].as_a
else
# Continuation data
contents = initial_data["onResponseReceivedActions"][0]?
.try &.["appendContinuationItemsAction"]["continuationItems"].as_a
end
else
contents = initial_data["response"]?.try &.["continuationContents"]["playlistVideoListContinuation"]["contents"].as_a
end
contents.try &.each do |item|
if i = item["lockupViewModel"]?
thumbnail_view_model = i.dig?(
"contentImage", "thumbnailViewModel"
)
watch_endpoint = i.dig?("rendererContext", "commandContext", "onTap", "innertubeCommand", "watchEndpoint")
video_id = watch_endpoint.try &.["videoId"]?.try &.as_s
plid = watch_endpoint.try &.["playlistId"]?.try &.as_s || playlist_id
index = watch_endpoint.try &.["index"]?.try &.as_i64
metadata = i["metadata"]?
lockup_metadata_view_model = metadata.try &.dig?("lockupMetadataViewModel")
title = lockup_metadata_view_model.try &.dig?("title", "content").try &.as_s
lockup_metadata = lockup_metadata_view_model.try &.dig?("metadata")
metadata_rows = lockup_metadata.try &.dig?("contentMetadataViewModel", "metadataRows").try &.as_a
# Find the metadataParts with commandRuns inside, which contains author
# information.
metadata_parts = metadata_rows.try &.find { |row|
parts = row["metadataParts"]?.try &.as_a
parts && parts.any? { |item2| item2.dig?("text", "commandRuns").try &.as_a }
}.try &.["metadataParts"].as_a
if author_info = metadata_parts.try &.find(&.dig?("text", "commandRuns"))
.try &.["text"]
author = author_info["content"].as_s
ucid = author_info.dig?("commandRuns", 0, "onTap", "innertubeCommand", "browseEndpoint", "browseId")
.try &.as_s
end
length = thumbnail_view_model.try &.dig?("overlays", 0, "thumbnailBottomOverlayViewModel", "badges", 0, "thumbnailBadgeViewModel", "text").try &.as_s
length_seconds = decode_length_seconds(length) if length
live = false
if !length_seconds
live = true
length_seconds = 0
end
videos << PlaylistVideo.new({
title: title || "",
id: video_id || "",
author: author || "",
ucid: ucid || "",
length_seconds: length_seconds,
published: Time.utc,
plid: plid,
live_now: live,
index: index || -1_i64,
})
end
rescue ex
videos << ProblematicTimelineItem.new(parse_exception: ex)
end
return videos
end
def template_playlist(playlist, listen)
html = <<-END_HTML
<h3>
<a href="/playlist?list=#{playlist["playlistId"]}">
#{playlist["title"]}
</a>
</h3>
<div class="pure-menu pure-menu-scrollable playlist-restricted">
<ol class="pure-menu-list">
END_HTML
playlist["videos"].as_a.each do |video|
html += <<-END_HTML
<li class="pure-menu-item" id="#{video["videoId"]}">
<a href="/watch?v=#{video["videoId"]}&list=#{playlist["playlistId"]}&index=#{video["index"]}#{listen ? "&listen=1" : ""}">
<div class="thumbnail">
<img loading="lazy" class="thumbnail" src="/vi/#{video["videoId"]}/mqdefault.jpg" alt="" />
<p class="length">#{recode_length_seconds(video["lengthSeconds"].as_i)}</p>
</div>
<p style="width:100%">#{video["title"]}</p>
<p>
<b style="width:100%">#{video["author"]}</b>
</p>
</a>
</li>
END_HTML
end
html += <<-END_HTML
</ol>
</div>
<hr>
END_HTML
html
end
end

46
src/invidious/routes/account.cr
View File

@ -21,7 +21,7 @@ module Invidious::Routes::Account
user = user.as(User)
sid = sid.as(String)
csrf_token = generate_response(sid, {":change_password"}, HMAC_KEY)
csrf_token = Invidious::Helpers::Tokens.generate_response(sid, {":change_password"}, HMAC_KEY)
templated "user/change_password"
end
@ -43,33 +43,33 @@ module Invidious::Routes::Account
token = env.params.body["csrf_token"]?
begin
validate_request(token, sid, env.request, HMAC_KEY, locale)
Invidious::Helpers::Tokens.validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
return error_template(400, ex)
return Errors.error_template(400, ex)
end
password = env.params.body["password"]?
if password.nil? || password.empty?
return error_template(401, "Password is a required field")
return Errors.error_template(401, "Password is a required field")
end
new_passwords = env.params.body.select { |k, _| k.match(/^new_password\[\d+\]$/) }.map { |_, v| v }
if new_passwords.size <= 1 || new_passwords.uniq.size != 1
return error_template(400, "New passwords must match")
return Errors.error_template(400, "New passwords must match")
end
new_password = new_passwords.uniq[0]
if new_password.empty?
return error_template(401, "Password cannot be empty")
return Errors.error_template(401, "Password cannot be empty")
end
if new_password.bytesize > 55
return error_template(400, "Password cannot be longer than 55 characters")
return Errors.error_template(400, "Password cannot be longer than 55 characters")
end
if !Crypto::Bcrypt::Password.new(user.password.not_nil!).verify(password.byte_slice(0, 55))
return error_template(401, "Incorrect password")
return Errors.error_template(401, "Incorrect password")
end
new_password = Crypto::Bcrypt::Password.create(new_password, cost: 10)
@ -96,7 +96,7 @@ module Invidious::Routes::Account
user = user.as(User)
sid = sid.as(String)
csrf_token = generate_response(sid, {":delete_account"}, HMAC_KEY)
csrf_token = Invidious::Helpers::Tokens.generate_response(sid, {":delete_account"}, HMAC_KEY)
templated "user/delete_account"
end
@ -118,9 +118,9 @@ module Invidious::Routes::Account
token = env.params.body["csrf_token"]?
begin
validate_request(token, sid, env.request, HMAC_KEY, locale)
Invidious::Helpers::Tokens.validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
return error_template(400, ex)
return Errors.error_template(400, ex)
end
view_name = "subscriptions_#{sha256(user.email)}"
@ -154,7 +154,7 @@ module Invidious::Routes::Account
user = user.as(User)
sid = sid.as(String)
csrf_token = generate_response(sid, {":clear_watch_history"}, HMAC_KEY)
csrf_token = Invidious::Helpers::Tokens.generate_response(sid, {":clear_watch_history"}, HMAC_KEY)
templated "user/clear_watch_history"
end
@ -176,9 +176,9 @@ module Invidious::Routes::Account
token = env.params.body["csrf_token"]?
begin
validate_request(token, sid, env.request, HMAC_KEY, locale)
Invidious::Helpers::Tokens.validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
return error_template(400, ex)
return Errors.error_template(400, ex)
end
Invidious::Database::Users.clear_watch_history(user)
@ -203,7 +203,7 @@ module Invidious::Routes::Account
user = user.as(User)
sid = sid.as(String)
csrf_token = generate_response(sid, {":authorize_token"}, HMAC_KEY)
csrf_token = Invidious::Helpers::Tokens.generate_response(sid, {":authorize_token"}, HMAC_KEY)
scopes = env.params.query["scopes"]?.try &.split(",")
scopes ||= [] of String
@ -235,16 +235,16 @@ module Invidious::Routes::Account
token = env.params.body["csrf_token"]?
begin
validate_request(token, sid, env.request, HMAC_KEY, locale)
Invidious::Helpers::Tokens.validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
return error_template(400, ex)
return Errors.error_template(400, ex)
end
scopes = env.params.body.select { |k, _| k.match(/^scopes\[\d+\]$/) }.map { |_, v| v }
callback_url = env.params.body["callbackUrl"]?
expire = env.params.body["expire"]?.try &.to_i?
access_token = generate_token(user.email, scopes, expire, HMAC_KEY)
access_token = Invidious::Helpers::Tokens.generate_token(user.email, scopes, expire, HMAC_KEY)
if callback_url
access_token = URI.encode_www_form(access_token)
@ -310,7 +310,7 @@ module Invidious::Routes::Account
if redirect
return env.redirect referer
else
return error_json(403, "No such user")
return Errors.error_json(403, "No such user")
end
end
@ -319,12 +319,12 @@ module Invidious::Routes::Account
token = env.params.body["csrf_token"]?
begin
validate_request(token, sid, env.request, HMAC_KEY, locale)
Invidious::Helpers::Tokens.validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
if redirect
return error_template(400, ex)
return Errors.error_template(400, ex)
else
return error_json(400, ex)
return Errors.error_json(400, ex)
end
end
@ -333,7 +333,7 @@ module Invidious::Routes::Account
session = env.params.query["session"]
Invidious::Database::SessionIDs.delete(sid: session, email: user.email)
else
return error_json(400, "Unsupported action #{action}")
return Errors.error_json(400, "Unsupported action #{action}")
end
if redirect

64
src/invidious/routes/api/v1/authenticated.cr
View File

@ -8,7 +8,7 @@ module Invidious::Routes::API::V1::Authenticated
# topics = env.params.body["topics"]?.try &.split(",").uniq.first(1000)
# topics ||= [] of String
# Helpers.create_notification_stream(env, topics, connection_channel)
# Invidious::Helpers.create_notification_stream(env, topics, connection_channel)
# end
def self.get_preferences(env)
@ -78,13 +78,13 @@ module Invidious::Routes::API::V1::Authenticated
user = env.get("user").as(User)
if !user.preferences.watch_history
return error_json(409, "Watch history is disabled in preferences.")
return Errors.error_json(409, "Watch history is disabled in preferences.")
end
# Sanity checks
id = env.params.url["id"]
unless validate_video_id(id)
return error_json(400, InvalidVideoID.new(id))
return Errors.error_json(400, InvalidVideoID.new(id))
end
Invidious::Database::Users.mark_watched(user, id)
@ -95,12 +95,12 @@ module Invidious::Routes::API::V1::Authenticated
user = env.get("user").as(User)
if !user.preferences.watch_history
return error_json(409, "Watch history is disabled in preferences.")
return Errors.error_json(409, "Watch history is disabled in preferences.")
end
video_id = env.params.url["id"]
unless video_id && validate_video_id(video_id)
return error_json(400, InvalidVideoID.new(video_id))
return Errors.error_json(400, InvalidVideoID.new(video_id))
end
Invidious::Database::Users.mark_unwatched(user, video_id)
@ -127,7 +127,7 @@ module Invidious::Routes::API::V1::Authenticated
page = env.params.query["page"]?.try &.to_i?
page ||= 1
videos, notifications = get_subscription_feed(user, max_results, page)
videos, notifications = Invidious::User::Users.get_subscription_feed(user, max_results, page)
JSON.build do |json|
json.object do
@ -175,7 +175,7 @@ module Invidious::Routes::API::V1::Authenticated
ucid = env.params.url["ucid"]
if !user.subscriptions.includes? ucid
get_channel(ucid)
Invidious::Channels::Channels.get_channel(ucid)
Invidious::Database::Users.subscribe_channel(user, ucid)
end
@ -214,19 +214,19 @@ module Invidious::Routes::API::V1::Authenticated
title = env.params.json["title"]?.try &.as(String).delete("<>").byte_slice(0, 150)
if !title
return error_json(400, "Invalid title.")
return Errors.error_json(400, "Invalid title.")
end
privacy = env.params.json["privacy"]?.try { |p| PlaylistPrivacy.parse(p.as(String).downcase) }
if !privacy
return error_json(400, "Invalid privacy setting.")
return Errors.error_json(400, "Invalid privacy setting.")
end
if Invidious::Database::Playlists.count_owned_by(user.email) >= 100
return error_json(400, "User cannot have more than 100 playlists.")
return Errors.error_json(400, "User cannot have more than 100 playlists.")
end
playlist = create_playlist(title, privacy, user)
playlist = Invidious::Playlists::Playlists.create_playlist(title, privacy, user)
env.response.headers["Location"] = "#{HOST_URL}/api/v1/auth/playlists/#{playlist.id}"
env.response.status_code = 201
{
@ -241,16 +241,16 @@ module Invidious::Routes::API::V1::Authenticated
plid = env.params.url["plid"]?
if !plid || plid.empty?
return error_json(400, "A playlist ID is required")
return Errors.error_json(400, "A playlist ID is required")
end
playlist = Invidious::Database::Playlists.select(id: plid)
if !playlist || playlist.author != user.email && playlist.privacy.private?
return error_json(404, "Playlist does not exist.")
return Errors.error_json(404, "Playlist does not exist.")
end
if playlist.author != user.email
return error_json(403, "Invalid user")
return Errors.error_json(403, "Invalid user")
end
title = env.params.json["title"].try &.as(String).delete("<>").byte_slice(0, 150) || playlist.title
@ -278,11 +278,11 @@ module Invidious::Routes::API::V1::Authenticated
playlist = Invidious::Database::Playlists.select(id: plid)
if !playlist || playlist.author != user.email && playlist.privacy.private?
return error_json(404, "Playlist does not exist.")
return Errors.error_json(404, "Playlist does not exist.")
end
if playlist.author != user.email
return error_json(403, "Invalid user")
return Errors.error_json(403, "Invalid user")
end
Invidious::Database::Playlists.delete(plid)
@ -298,29 +298,29 @@ module Invidious::Routes::API::V1::Authenticated
playlist = Invidious::Database::Playlists.select(id: plid)
if !playlist || playlist.author != user.email && playlist.privacy.private?
return error_json(404, "Playlist does not exist.")
return Errors.error_json(404, "Playlist does not exist.")
end
if playlist.author != user.email
return error_json(403, "Invalid user")
return Errors.error_json(403, "Invalid user")
end
if playlist.index.size >= CONFIG.playlist_length_limit
return error_json(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos")
return Errors.error_json(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos")
end
video_id = env.params.json["videoId"].try &.as(String)
# Sanity checks
unless video_id && validate_video_id(video_id)
return error_json(400, InvalidVideoID.new(video_id))
return Errors.error_json(400, InvalidVideoID.new(video_id))
end
begin
video = get_video(video_id)
rescue ex : NotFoundException
return error_json(404, ex)
return Errors.error_json(404, ex)
rescue ex
return error_json(500, ex)
return Errors.error_json(500, ex)
end
playlist_video = PlaylistVideo.new({
@ -355,15 +355,15 @@ module Invidious::Routes::API::V1::Authenticated
playlist = Invidious::Database::Playlists.select(id: plid)
if !playlist || playlist.author != user.email && playlist.privacy.private?
return error_json(404, "Playlist does not exist.")
return Errors.error_json(404, "Playlist does not exist.")
end
if playlist.author != user.email
return error_json(403, "Invalid user")
return Errors.error_json(403, "Invalid user")
end
if !playlist.index.includes? index
return error_json(404, "Playlist does not contain index")
return Errors.error_json(404, "Playlist does not contain index")
end
Invidious::Database::PlaylistVideos.delete(index, plid)
@ -411,7 +411,7 @@ module Invidious::Routes::API::V1::Authenticated
callback_url = env.params.json["callbackUrl"]?.try &.as(String)
expire = env.params.json["expire"]?.try &.as(Int64)
else
return error_json(400, "Invalid or missing header 'Content-Type'")
return Errors.error_json(400, "Invalid or missing header 'Content-Type'")
end
if callback_url && callback_url.empty?
@ -427,7 +427,7 @@ module Invidious::Routes::API::V1::Authenticated
# Used by template bellow.
# ameba:disable Lint/UselessAssign
csrf_token = generate_response(sid, {":authorize_token"}, HMAC_KEY, use_nonce: true)
csrf_token = Invidious::Helpers::Tokens.generate_response(sid, {":authorize_token"}, HMAC_KEY, use_nonce: true)
return templated "user/authorize_token"
else
env.response.content_type = "application/json"
@ -436,12 +436,12 @@ module Invidious::Routes::API::V1::Authenticated
authorized_scopes = [] of String
scopes.each do |scope|
if scopes_include_scope(superset_scopes, scope)
if Invidious::Helpers::Tokens.scopes_include_scope(superset_scopes, scope)
authorized_scopes << scope
end
end
access_token = generate_token(user.email, authorized_scopes, expire, HMAC_KEY)
access_token = Invidious::Helpers::Tokens.generate_token(user.email, authorized_scopes, expire, HMAC_KEY)
if callback_url
access_token = URI.encode_www_form(access_token)
@ -473,10 +473,10 @@ module Invidious::Routes::API::V1::Authenticated
# Allow tokens to revoke other tokens with correct scope
if session == env.get("session").as(String)
Invidious::Database::SessionIDs.delete(sid: session)
elsif scopes_include_scope(scopes, "GET:tokens")
elsif Invidious::Helpers::Tokens.scopes_include_scope(scopes, "GET:tokens")
Invidious::Database::SessionIDs.delete(sid: session)
else
return error_json(400, "Cannot revoke session #{session}")
return Errors.error_json(400, "Cannot revoke session #{session}")
end
env.response.status_code = 204
@ -489,6 +489,6 @@ module Invidious::Routes::API::V1::Authenticated
topics = raw_topics.try &.split(",").uniq!.first(1000)
topics ||= [] of String
Helpers.create_notification_stream(env, topics, CONNECTION_CHANNEL)
Invidious::Helpers.create_notification_stream(env, topics, CONNECTION_CHANNEL)
end
end

62
src/invidious/routes/api/v1/channels.cr
View File

@ -3,14 +3,14 @@ module Invidious::Routes::API::V1::Channels
# This sets the `channel` variable, or handles Exceptions.
private macro get_channel
begin
channel = get_about_info(ucid)
channel = Invidious::Channels::About.get_about_info(ucid)
rescue ex : ChannelRedirect
env.response.headers["Location"] = env.request.resource.gsub(ucid, ex.channel_id)
return error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id})
return Errors.error_json(302, "Channel is unavailable", {"authorId" => ex.channel_id})
rescue ex : NotFoundException
return error_json(404, ex)
return Errors.error_json(404, ex)
rescue ex
return error_json(500, ex)
return Errors.error_json(500, ex)
end
end
@ -29,8 +29,8 @@ module Invidious::Routes::API::V1::Channels
if channel.is_age_gated
begin
playlist = get_playlist(channel.ucid.sub("UC", "UULF"))
videos = get_playlist_videos(playlist, offset: 0)
playlist = Invidious::Playlists::Playlists.get_playlist(channel.ucid.sub("UC", "UULF"))
videos = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: 0)
rescue ex : InfoException
# playlist doesnt exist.
videos = [] of PlaylistVideo
@ -39,7 +39,7 @@ module Invidious::Routes::API::V1::Channels
begin
videos, _ = Channel::Tabs.get_videos(channel, sort_by: sort_by)
rescue ex
return error_json(500, ex)
return Errors.error_json(500, ex)
end
end
@ -96,7 +96,7 @@ module Invidious::Routes::API::V1::Channels
json.field "autoGenerated", channel.auto_generated
json.field "ageGated", channel.is_age_gated
json.field "isFamilyFriendly", channel.is_family_friendly
json.field "description", Helpers.html_to_content(channel.description_html)
json.field "description", Invidious::Helpers.html_to_content(channel.description_html)
json.field "descriptionHtml", channel.description_html
json.field "allowedRegions", channel.allowed_regions
@ -117,7 +117,7 @@ module Invidious::Routes::API::V1::Channels
json.array do
# Fetch related channels
begin
related_channels, _ = fetch_related_channels(channel)
related_channels, _ = Invidious::Channels::About.fetch_related_channels(channel)
rescue ex
related_channels = [] of SearchChannel
end
@ -155,8 +155,8 @@ module Invidious::Routes::API::V1::Channels
if channel.is_age_gated
begin
playlist = get_playlist(channel.ucid.sub("UC", "UULF"))
videos = get_playlist_videos(playlist, offset: 0)
playlist = Invidious::Playlists::Playlists.get_playlist(channel.ucid.sub("UC", "UULF"))
videos = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: 0)
rescue ex : InfoException
# playlist doesnt exist.
videos = [] of PlaylistVideo
@ -168,7 +168,7 @@ module Invidious::Routes::API::V1::Channels
channel, continuation: continuation, sort_by: sort_by
)
rescue ex
return error_json(500, ex)
return Errors.error_json(500, ex)
end
end
@ -201,8 +201,8 @@ module Invidious::Routes::API::V1::Channels
if channel.is_age_gated
begin
playlist = get_playlist(channel.ucid.sub("UC", "UUSH"))
videos = get_playlist_videos(playlist, offset: 0)
playlist = Invidious::Playlists::Playlists.get_playlist(channel.ucid.sub("UC", "UUSH"))
videos = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: 0)
rescue ex : InfoException
# playlist doesnt exist.
videos = [] of PlaylistVideo
@ -214,7 +214,7 @@ module Invidious::Routes::API::V1::Channels
channel, continuation: continuation, sort_by: sort_by
)
rescue ex
return error_json(500, ex)
return Errors.error_json(500, ex)
end
end
@ -247,8 +247,8 @@ module Invidious::Routes::API::V1::Channels
if channel.is_age_gated
begin
playlist = get_playlist(channel.ucid.sub("UC", "UULV"))
videos = get_playlist_videos(playlist, offset: 0)
playlist = Invidious::Playlists::Playlists.get_playlist(channel.ucid.sub("UC", "UULV"))
videos = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: 0)
rescue ex : InfoException
# playlist doesnt exist.
videos = [] of PlaylistVideo
@ -260,7 +260,7 @@ module Invidious::Routes::API::V1::Channels
channel, continuation: continuation, sort_by: sort_by
)
rescue ex
return error_json(500, ex)
return Errors.error_json(500, ex)
end
end
@ -292,7 +292,7 @@ module Invidious::Routes::API::V1::Channels
channel = nil # Make the compiler happy
get_channel()
items, next_continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
items, next_continuation = Invidious::Channels::Playlists.fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
JSON.build do |json|
json.object do
@ -321,7 +321,7 @@ module Invidious::Routes::API::V1::Channels
channel = nil # Make the compiler happy
get_channel()
items, next_continuation = fetch_channel_podcasts(channel.ucid, channel.author, continuation)
items, next_continuation = Invidious::Channels::Playlists.fetch_channel_podcasts(channel.ucid, channel.author, continuation)
JSON.build do |json|
json.object do
@ -350,7 +350,7 @@ module Invidious::Routes::API::V1::Channels
channel = nil # Make the compiler happy
get_channel()
items, next_continuation = fetch_channel_releases(channel.ucid, channel.author, continuation)
items, next_continuation = Invidious::Channels::Playlists.fetch_channel_releases(channel.ucid, channel.author, continuation)
JSON.build do |json|
json.object do
@ -379,7 +379,7 @@ module Invidious::Routes::API::V1::Channels
channel = nil # Make the compiler happy
get_channel()
items, next_continuation = fetch_channel_courses(channel.ucid, channel.author, continuation)
items, next_continuation = Invidious::Channels::Playlists.fetch_channel_courses(channel.ucid, channel.author, continuation)
JSON.build do |json|
json.object do
@ -413,9 +413,9 @@ module Invidious::Routes::API::V1::Channels
# sort_by = env.params.query["sort_by"]?.try &.downcase
begin
fetch_channel_community(ucid, continuation, locale, format, thin_mode)
Invidious::Channels::Community.fetch_channel_community(ucid, continuation, locale, format, thin_mode)
rescue ex
return error_json(500, ex)
return Errors.error_json(500, ex)
end
end
@ -434,16 +434,16 @@ module Invidious::Routes::API::V1::Channels
if ucid.nil?
response = YoutubeAPI.resolve_url("https://www.youtube.com/post/#{id}")
return error_json(400, "Invalid post ID") if response["error"]?
ucid = decode_ucid_from_post_protobuf(response.dig("endpoint", "browseEndpoint", "params").as_s)
return Errors.error_json(400, "Invalid post ID") if response["error"]?
ucid = Invidious::Channels::Community.decode_ucid_from_post_protobuf(response.dig("endpoint", "browseEndpoint", "params").as_s)
else
ucid = ucid.to_s
end
begin
fetch_channel_community_post(ucid, id, locale, format, thin_mode)
Invidious::Channels::Community.fetch_channel_community_post(ucid, id, locale, format, thin_mode)
rescue ex
return error_json(500, ex)
return Errors.error_json(500, ex)
end
end
@ -487,9 +487,9 @@ module Invidious::Routes::API::V1::Channels
continuation = env.params.query["continuation"]?
begin
items, next_continuation = fetch_related_channels(channel, continuation)
items, next_continuation = Invidious::Channels::About.fetch_related_channels(channel, continuation)
rescue ex
return error_json(500, ex)
return Errors.error_json(500, ex)
end
JSON.build do |json|
@ -520,7 +520,7 @@ module Invidious::Routes::API::V1::Channels
begin
search_results = query.process
rescue ex
return error_json(400, ex)
return Errors.error_json(400, ex)
end
JSON.build do |json|

4
src/invidious/routes/api/v1/feeds.cr
View File

@ -8,9 +8,9 @@ module Invidious::Routes::API::V1::Feeds
trending_type = env.params.query["type"]?
begin
trending, _ = fetch_trending(trending_type, region, locale)
trending, _ = Invidious::Feeds::Trending.fetch(trending_type, region, locale)
rescue ex
return error_json(500, ex)
return Errors.error_json(500, ex)
end
videos = JSON.build do |json|

18
src/invidious/routes/api/v1/misc.cr
View File

@ -50,16 +50,16 @@ module Invidious::Routes::API::V1::Misc
end
begin
playlist = get_playlist(plid)
playlist = Invidious::Playlists::Playlists.get_playlist(plid)
rescue ex : InfoException
return error_json(404, ex)
return Errors.error_json(404, ex)
rescue ex
return error_json(404, "Playlist does not exist.")
return Errors.error_json(404, "Playlist does not exist.")
end
user = env.get?("user").try &.as(User)
if !playlist || playlist.privacy.private? && playlist.author != user.try &.email
return error_json(404, "Playlist does not exist.")
return Errors.error_json(404, "Playlist does not exist.")
end
# includes into the playlist a maximum of 20 videos, before the offset
@ -88,7 +88,7 @@ module Invidious::Routes::API::V1::Misc
end
if format == "html"
playlist_html = template_playlist(json_response, listen)
playlist_html = Invidious::Playlists::Playlists.template_playlist(json_response, listen)
index, next_video = json_response["videos"].as_a.skip(1 + lookback).select { |video| !video["author"].as_s.empty? }[0]?.try { |v| {v["index"], v["videoId"]} } || {nil, nil}
response = {
@ -127,7 +127,7 @@ module Invidious::Routes::API::V1::Misc
mix.videos = mix.videos[index..-1]
rescue ex
return error_json(500, ex)
return Errors.error_json(500, ex)
end
response = JSON.build do |json|
@ -178,14 +178,14 @@ module Invidious::Routes::API::V1::Misc
env.response.content_type = "application/json"
url = env.params.query["url"]?
return error_json(400, "Missing URL to resolve") if !url
return Errors.error_json(400, "Missing URL to resolve") if !url
begin
resolved_url = YoutubeAPI.resolve_url(url.as(String))
endpoint = resolved_url["endpoint"]
page_type = endpoint.dig?("commandMetadata", "webCommandMetadata", "webPageType").try &.as_s || ""
if page_type == "WEB_PAGE_TYPE_UNKNOWN"
return error_json(400, "Unknown url")
return Errors.error_json(400, "Unknown url")
end
sub_endpoint = endpoint["watchEndpoint"]? || endpoint["browseEndpoint"]? || endpoint
@ -204,7 +204,7 @@ module Invidious::Routes::API::V1::Misc
post_id = nil
end
rescue ex
return error_json(500, ex)
return Errors.error_json(500, ex)
end
JSON.build do |json|
json.object do

8
src/invidious/routes/api/v1/search.cr
View File

@ -10,7 +10,7 @@ module Invidious::Routes::API::V1::Search
begin
search_results = query.process
rescue ex
return error_json(400, ex)
return Errors.error_json(400, ex)
end
JSON.build do |json|
@ -53,7 +53,7 @@ module Invidious::Routes::API::V1::Search
end
end
rescue ex
return error_json(500, ex)
return Errors.error_json(500, ex)
end
end
@ -67,9 +67,9 @@ module Invidious::Routes::API::V1::Search
env.response.content_type = "application/json"
begin
results = Invidious::Hashtag.fetch(hashtag, page, region)
results = Invidious::Search::Hashtag.fetch_hashtag(hashtag, page, region)
rescue ex
return error_json(400, ex)
return Errors.error_json(400, ex)
end
JSON.build do |json|

36
src/invidious/routes/api/v1/videos.cr
View File

@ -16,9 +16,9 @@ module Invidious::Routes::API::V1::Videos
begin
video = get_video(id, region: region)
rescue ex : NotFoundException
return error_json(404, ex)
return Errors.error_json(404, ex)
rescue ex
return error_json(500, ex)
return Errors.error_json(500, ex)
end
return JSON.build do |json|
@ -34,7 +34,7 @@ module Invidious::Routes::API::V1::Videos
# Sanity checks
unless validate_video_id(id)
return error_json(400, InvalidVideoID.new(id))
return Errors.error_json(400, InvalidVideoID.new(id))
end
# See https://github.com/ytdl-org/youtube-dl/blob/6ab30ff50bf6bd0585927cb73c7421bef184f87a/youtube_dl/extractor/youtube.py#L1354
@ -301,7 +301,7 @@ module Invidious::Routes::API::V1::Videos
annotations = response.body
Helpers.cache_annotation(video_id, annotations)
Invidious::Helpers.cache_annotation(video_id, annotations)
end
else # "youtube"
response = YT_POOL.client &.get("/annotations_invideo?video_id=#{video_id}")
@ -354,9 +354,9 @@ module Invidious::Routes::API::V1::Videos
begin
comments = Comments.fetch_youtube(id, continuation, format, locale, thin_mode, region, sort_by: sort_by)
rescue ex : NotFoundException
return error_json(404, ex)
return Errors.error_json(404, ex)
rescue ex
return error_json(500, ex)
return Errors.error_json(500, ex)
end
return comments
@ -371,7 +371,7 @@ module Invidious::Routes::API::V1::Videos
end
if !reddit_thread || !comments
return error_json(404, "No reddit threads found")
return Errors.error_json(404, "No reddit threads found")
end
if format == "json"
@ -404,10 +404,10 @@ module Invidious::Routes::API::V1::Videos
proxy = {"1", "true"}.any? &.== env.params.query["local"]?
response = YoutubeAPI.resolve_url("https://www.youtube.com/clip/#{clip_id}")
return error_json(400, "Invalid clip ID") if response["error"]?
return Errors.error_json(400, "Invalid clip ID") if response["error"]?
video_id = response.dig?("endpoint", "watchEndpoint", "videoId").try &.as_s
return error_json(400, "Invalid clip ID") if video_id.nil?
return Errors.error_json(400, "Invalid clip ID") if video_id.nil?
start_time = nil
end_time = nil
@ -420,9 +420,9 @@ module Invidious::Routes::API::V1::Videos
begin
video = get_video(video_id, region: region)
rescue ex : NotFoundException
return error_json(404, ex)
return Errors.error_json(404, ex)
rescue ex
return error_json(500, ex)
return Errors.error_json(500, ex)
end
return JSON.build do |json|
@ -454,9 +454,9 @@ module Invidious::Routes::API::V1::Videos
begin
video = get_video(id)
rescue ex : NotFoundException
return error_json(404, ex)
return Errors.error_json(404, ex)
rescue ex
return error_json(500, ex)
return Errors.error_json(500, ex)
end
response = JSON.build do |json|
@ -494,14 +494,14 @@ module Invidious::Routes::API::V1::Videos
begin
video = get_video(id)
rescue ex : NotFoundException
return error_json(404, ex)
return Errors.error_json(404, ex)
rescue ex
return error_json(500, ex)
return Errors.error_json(500, ex)
end
target_transcript = video.captions.select(&.name.== label)
if target_transcript.empty?
return error_json(404, NotFoundException.new("Requested transcript does not exist"))
return Errors.error_json(404, NotFoundException.new("Requested transcript does not exist"))
else
target_transcript = target_transcript[0]
lang, auto_generated = target_transcript.language_code, target_transcript.auto_generated
@ -515,9 +515,9 @@ module Invidious::Routes::API::V1::Videos
YoutubeAPI.get_transcript(params), lang, auto_generated
)
rescue ex : NotFoundException
return error_json(404, ex)
return Errors.error_json(404, ex)
rescue ex
return error_json(500, ex)
return Errors.error_json(500, ex)
end
return transcript.to_json

6
src/invidious/routes/before_all.cr
View File

@ -1,4 +1,6 @@
module Invidious::Routes::BeforeAll
extend self
struct CompanionCSP
property companion_urls : String = ""
@ -89,7 +91,7 @@ module Invidious::Routes::BeforeAll
if email = Database::SessionIDs.select_email(sid)
user = Database::Users.select!(email: email)
csrf_token = generate_response(sid, {
csrf_token = Invidious::Helpers::Tokens.generate_response(sid, {
":authorize_token",
":playlist_ajax",
":signout",
@ -107,7 +109,7 @@ module Invidious::Routes::BeforeAll
end
end
dark_mode = convert_theme(env.params.query["dark_mode"]?) || preferences.dark_mode.to_s
dark_mode = Invidious::User::Converters.convert_theme(env.params.query["dark_mode"]?) || preferences.dark_mode.to_s
thin_mode = env.params.query["thin_mode"]?
thin_mode = (thin_mode == "true") || preferences.thin_mode
locale = env.params.query["hl"]? || preferences.locale

46
src/invidious/routes/channels.cr
View File

@ -23,7 +23,7 @@ module Invidious::Routes::Channels
sort_by ||= "last"
sort_options = {"last", "oldest", "newest"}
items, next_continuation = fetch_channel_playlists(
items, next_continuation = Invidious::Channels::Playlists.fetch_channel_playlists(
channel.ucid, channel.author, continuation, sort_by
)
@ -42,8 +42,8 @@ module Invidious::Routes::Channels
sort_by = ""
sort_options = [] of String
begin
playlist = get_playlist(channel.ucid.sub("UC", "UULF"))
items = get_playlist_videos(playlist, offset: 0)
playlist = Invidious::Playlists::Playlists.get_playlist(channel.ucid.sub("UC", "UULF"))
items = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: 0)
rescue ex : InfoException
# playlist doesnt exist.
items = [] of PlaylistVideo
@ -77,8 +77,8 @@ module Invidious::Routes::Channels
sort_by = ""
sort_options = [] of String
begin
playlist = get_playlist(channel.ucid.sub("UC", "UUSH"))
items = get_playlist_videos(playlist, offset: 0)
playlist = Invidious::Playlists::Playlists.get_playlist(channel.ucid.sub("UC", "UUSH"))
items = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: 0)
rescue ex : InfoException
# playlist doesnt exist.
items = [] of PlaylistVideo
@ -112,8 +112,8 @@ module Invidious::Routes::Channels
sort_by = ""
sort_options = [] of String
begin
playlist = get_playlist(channel.ucid.sub("UC", "UULV"))
items = get_playlist_videos(playlist, offset: 0)
playlist = Invidious::Playlists::Playlists.get_playlist(channel.ucid.sub("UC", "UULV"))
items = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: 0)
rescue ex : InfoException
# playlist doesnt exist.
items = [] of PlaylistVideo
@ -146,7 +146,7 @@ module Invidious::Routes::Channels
return env.redirect "/channel/#{channel.ucid}"
end
items, next_continuation = fetch_channel_playlists(
items, next_continuation = Invidious::Channels::Playlists.fetch_channel_playlists(
channel.ucid, channel.author, continuation, (sort_by || "last")
)
@ -166,7 +166,7 @@ module Invidious::Routes::Channels
sort_by = ""
sort_options = [] of String
items, next_continuation = fetch_channel_podcasts(
items, next_continuation = Invidious::Channels::Playlists.fetch_channel_podcasts(
channel.ucid, channel.author, continuation
)
@ -186,7 +186,7 @@ module Invidious::Routes::Channels
sort_by = ""
sort_options = [] of String
items, next_continuation = fetch_channel_releases(
items, next_continuation = Invidious::Channels::Playlists.fetch_channel_releases(
channel.ucid, channel.author, continuation
)
@ -206,7 +206,7 @@ module Invidious::Routes::Channels
sort_by = ""
sort_options = [] of String
items, next_continuation = fetch_channel_courses(
items, next_continuation = Invidious::Channels::Playlists.fetch_channel_courses(
channel.ucid, channel.author, continuation
)
@ -247,7 +247,7 @@ module Invidious::Routes::Channels
sort_options = [] of String
begin
items = JSON.parse(fetch_channel_community(ucid, continuation, locale, "json", thin_mode))
items = JSON.parse(Invidious::Channels::Community.fetch_channel_community(ucid, continuation, locale, "json", thin_mode))
rescue ex : InfoException
env.response.status_code = 500
error_message = ex.message
@ -255,7 +255,7 @@ module Invidious::Routes::Channels
env.response.status_code = 404
error_message = ex.message
rescue ex
return error_template(500, ex)
return Errors.error_template(500, ex)
end
templated "community"
@ -280,14 +280,14 @@ module Invidious::Routes::Channels
if !ucid.nil?
ucid = ucid.to_s
post_response = fetch_channel_community_post(ucid, id, locale, "json", thin_mode)
post_response = Invidious::Channels::Community.fetch_channel_community_post(ucid, id, locale, "json", thin_mode)
else
# resolve the url to get the author's UCID
response = YoutubeAPI.resolve_url("https://www.youtube.com/post/#{id}")
return error_template(400, "Invalid post ID") if response["error"]?
return Errors.error_template(400, "Invalid post ID") if response["error"]?
ucid = decode_ucid_from_post_protobuf(response.dig("endpoint", "browseEndpoint", "params").as_s)
post_response = fetch_channel_community_post(ucid, id, locale, "json", thin_mode)
ucid = Invidious::Channels::Community.decode_ucid_from_post_protobuf(response.dig("endpoint", "browseEndpoint", "params").as_s)
post_response = Invidious::Channels::Community.fetch_channel_community_post(ucid, id, locale, "json", thin_mode)
end
post_response = JSON.parse(post_response)
@ -309,7 +309,7 @@ module Invidious::Routes::Channels
return env.redirect "/channel/#{channel.ucid}"
end
items, next_continuation = fetch_related_channels(channel, continuation)
items, next_continuation = Invidious::Channels::About.fetch_related_channels(channel, continuation)
# Featured/related channels can't be sorted
sort_options = [] of String
@ -354,7 +354,7 @@ module Invidious::Routes::Channels
resolved_url = YoutubeAPI.resolve_url("https://www.youtube.com#{env.request.path}#{yt_url_params.size > 0 ? "?#{yt_url_params}" : ""}")
ucid = resolved_url["endpoint"]["browseEndpoint"]["browseId"]
rescue ex : InfoException | KeyError
return error_template(404, I18n.translate(locale, "This channel does not exist."))
return Errors.error_template(404, I18n.translate(locale, "This channel does not exist."))
end
selected_tab = env.params.url["tab"]?
@ -380,7 +380,7 @@ module Invidious::Routes::Channels
user = env.params.query["user"]?
if !user
return error_template(404, "This channel does not exist.")
return Errors.error_template(404, "This channel does not exist.")
else
env.redirect "/user/#{user}#{uri_params}"
end
@ -433,13 +433,13 @@ module Invidious::Routes::Channels
continuation = env.params.query["continuation"]?
begin
channel = get_about_info(ucid)
channel = Invidious::Channels::About.get_about_info(ucid)
rescue ex : ChannelRedirect
return env.redirect env.request.resource.gsub(ucid, ex.channel_id)
rescue ex : NotFoundException
return error_template(404, ex)
return Errors.error_template(404, ex)
rescue ex
return error_template(500, ex)
return Errors.error_template(500, ex)
end
env.set "search", "channel:#{ucid} "

22
src/invidious/routes/embed.cr
View File

@ -5,9 +5,9 @@ module Invidious::Routes::Embed
locale = env.get("preferences").as(Preferences).locale
if plid = env.params.query["list"]?.try &.gsub(/[^a-zA-Z0-9_-]/, "")
begin
playlist = get_playlist(plid)
playlist = Invidious::Playlists::Playlists.get_playlist(plid)
offset = env.params.query["index"]?.try &.to_i? || 0
videos = get_playlist_videos(playlist, offset: offset)
videos = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: offset)
if videos.empty?
url = "/playlist?list=#{plid}"
raise NotFoundException.new(I18n.translate(locale, "error_video_not_in_playlist", url))
@ -15,9 +15,9 @@ module Invidious::Routes::Embed
first_playlist_video = videos[0].as(PlaylistVideo)
rescue ex : NotFoundException
return error_template(404, ex)
return Errors.error_template(404, ex)
rescue ex
return error_template(500, ex)
return Errors.error_template(500, ex)
end
url = "/embed/#{first_playlist_video.id}?#{env.params.query}"
@ -66,9 +66,9 @@ module Invidious::Routes::Embed
if plid
begin
playlist = get_playlist(plid)
playlist = Invidious::Playlists::Playlists.get_playlist(plid)
offset = env.params.query["index"]?.try &.to_i? || 0
videos = get_playlist_videos(playlist, offset: offset)
videos = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: offset)
if videos.empty?
url = "/playlist?list=#{plid}"
raise NotFoundException.new(I18n.translate(locale, "error_video_not_in_playlist", url))
@ -76,9 +76,9 @@ module Invidious::Routes::Embed
first_playlist_video = videos[0].as(PlaylistVideo)
rescue ex : NotFoundException
return error_template(404, ex)
return Errors.error_template(404, ex)
rescue ex
return error_template(500, ex)
return Errors.error_template(500, ex)
end
url = "/embed/#{first_playlist_video.id}"
@ -101,7 +101,7 @@ module Invidious::Routes::Embed
env.params.query.delete_all("channel")
if !video_id || video_id == "live_stream"
return error_template(500, "Video is unavailable.")
return Errors.error_template(500, "Video is unavailable.")
end
url = "/embed/#{video_id}"
@ -135,9 +135,9 @@ module Invidious::Routes::Embed
begin
video = get_video(id, region: params.region)
rescue ex : NotFoundException
return error_template(404, ex)
return Errors.error_template(404, ex)
rescue ex
return error_template(500, ex)
return Errors.error_template(500, ex)
end
if preferences.annotations_subscribed &&

18
src/invidious/routes/feeds.cr
View File

@ -53,9 +53,9 @@ module Invidious::Routes::Feeds
region ||= preferences.region
begin
trending, plid = fetch_trending(trending_type, region, locale)
trending, plid = Invidious::Feeds::Trending.fetch(trending_type, region, locale)
rescue ex
return error_template(500, ex)
return Errors.error_template(500, ex)
end
templated "feeds/trending"
@ -91,7 +91,7 @@ module Invidious::Routes::Feeds
page = env.params.query["page"]?.try &.to_i?
page ||= 1
videos, notifications = get_subscription_feed(user, max_results, page)
videos, notifications = Invidious::User::Users.get_subscription_feed(user, max_results, page)
if CONFIG.enable_user_notifications
# "updated" here is used for delivering new notifications, so if
@ -150,7 +150,7 @@ module Invidious::Routes::Feeds
if env.params.url["ucid"].matches?(/^[\w-]+$/)
ucid = env.params.url["ucid"]
else
return error_atom(400, InfoException.new("Invalid channel ucid provided."))
return Errors.error_atom(400, InfoException.new("Invalid channel ucid provided."))
end
params = HTTP::Params.parse(env.params.query["params"]? || "")
@ -162,7 +162,7 @@ module Invidious::Routes::Feeds
}
response = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}")
return error_atom(404, NotFoundException.new("Channel does not exist.")) if response.status_code == 404
return Errors.error_atom(404, NotFoundException.new("Channel does not exist.")) if response.status_code == 404
rss = XML.parse(response.body)
videos = rss.xpath_nodes("//default:feed/default:entry", namespaces).map do |entry|
@ -250,7 +250,7 @@ module Invidious::Routes::Feeds
params = HTTP::Params.parse(env.params.query["params"]? || "")
videos, notifications = get_subscription_feed(user, max_results, page)
videos, notifications = Invidious::User::Users.get_subscription_feed(user, max_results, page)
XML.build(indent: " ", encoding: "UTF-8") do |xml|
xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015",
@ -281,11 +281,11 @@ module Invidious::Routes::Feeds
if plid.starts_with? "IV"
if playlist = Invidious::Database::Playlists.select(id: plid)
videos = get_playlist_videos(playlist, offset: 0)
videos = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: 0)
user = env.get?("user").try &.as(User)
if !playlist || playlist.privacy.private? && playlist.author != user.try &.email
return error_atom(404, "Playlist does not exist.")
return Errors.error_atom(404, "Playlist does not exist.")
end
return XML.build(indent: " ", encoding: "UTF-8") do |xml|
@ -317,7 +317,7 @@ module Invidious::Routes::Feeds
end
response = YT_POOL.client &.get("/feeds/videos.xml?playlist_id=#{plid}")
return error_atom(404, NotFoundException.new("Playlist does not exist.")) if response.status_code == 404
return Errors.error_atom(404, NotFoundException.new("Playlist does not exist.")) if response.status_code == 404
document = XML.parse(response.body)
document.xpath_nodes(%q(//*[@href]|//*[@url])).each do |node|

4
src/invidious/routes/images.cr
View File

@ -106,7 +106,7 @@ module Invidious::Routes::Images
break
end
Helpers.proxy_file(response, env)
Invidious::Helpers.proxy_file(response, env)
end
rescue ex
end
@ -158,6 +158,6 @@ module Invidious::Routes::Images
return env.response.headers.delete("Transfer-Encoding")
end
return Helpers.proxy_file(response, env)
return Invidious::Helpers.proxy_file(response, env)
end
end

30
src/invidious/routes/login.cr
View File

@ -11,7 +11,7 @@ module Invidious::Routes::Login
return env.redirect referer if user
if !CONFIG.login_enabled
return error_template(400, "Login has been disabled by administrator.")
return Errors.error_template(400, "Login has been disabled by administrator.")
end
email = nil
@ -31,7 +31,7 @@ module Invidious::Routes::Login
referer = get_referer(env, "/feed/subscriptions")
if !CONFIG.login_enabled
return error_template(403, "Login has been disabled by administrator.")
return Errors.error_template(403, "Login has been disabled by administrator.")
end
# https://stackoverflow.com/a/574698
@ -44,11 +44,11 @@ module Invidious::Routes::Login
case account_type
when "invidious"
if email.nil? || email.empty?
return error_template(401, "User ID is a required field")
return Errors.error_template(401, "User ID is a required field")
end
if password.nil? || password.empty?
return error_template(401, "Password is a required field")
return Errors.error_template(401, "Password is a required field")
end
user = Invidious::Database::Users.select(email: email)
@ -64,7 +64,7 @@ module Invidious::Routes::Login
env.response.cookies["SID"] = Invidious::User::Cookies.sid(CONFIG.domain, sid)
end
else
return error_template(401, "Wrong username or password")
return Errors.error_template(401, "Wrong username or password")
end
# Since this user has already registered, we don't want to overwrite their preferences
@ -75,16 +75,16 @@ module Invidious::Routes::Login
end
else
if !CONFIG.registration_enabled
return error_template(400, "Registration has been disabled by administrator.")
return Errors.error_template(400, "Registration has been disabled by administrator.")
end
if password.empty?
return error_template(401, "Password cannot be empty")
return Errors.error_template(401, "Password cannot be empty")
end
# See https://security.stackexchange.com/a/39851
if password.bytesize > 55
return error_template(400, "Password cannot be longer than 55 characters")
return Errors.error_template(400, "Password cannot be longer than 55 characters")
end
password = password.byte_slice(0, 55)
@ -102,11 +102,11 @@ module Invidious::Routes::Login
answer = OpenSSL::HMAC.hexdigest(:sha256, HMAC_KEY, answer)
begin
validate_request(tokens[0], answer, env.request, HMAC_KEY, locale)
Invidious::Helpers::Tokens.validate_request(tokens[0], answer, env.request, HMAC_KEY, locale)
rescue ex : InfoException
return error_template(400, InfoException.new("Erroneous CAPTCHA"))
return Errors.error_template(400, InfoException.new("Erroneous CAPTCHA"))
rescue ex
return error_template(400, ex)
return Errors.error_template(400, ex)
end
else
return templated "user/login"
@ -114,7 +114,7 @@ module Invidious::Routes::Login
end
sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
user, sid = create_user(sid, email, password)
user, sid = Invidious::User::Users.create_user(sid, email, password)
if language_header = env.request.headers["Accept-Language"]?
if language = ANG.language_negotiator.best(language_header, I18n::LOCALES.keys)
@ -126,7 +126,7 @@ module Invidious::Routes::Login
Invidious::Database::SessionIDs.insert(sid, email)
view_name = "subscriptions_#{sha256(user.email)}"
PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{MATERIALIZED_VIEW_SQL.call(user.email)}")
PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS #{Invidious::User::Users::MATERIALIZED_VIEW_SQL.call(user.email)}")
if alt = CONFIG.alternative_domains.index(host)
env.response.cookies["SID"] = Invidious::User::Cookies.sid(CONFIG.alternative_domains[alt], sid)
@ -166,9 +166,9 @@ module Invidious::Routes::Login
token = env.params.body["csrf_token"]?
begin
validate_request(token, sid, env.request, HMAC_KEY, locale)
Invidious::Helpers::Tokens.validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
return error_template(400, ex)
return Errors.error_template(400, ex)
end
Invidious::Database::SessionIDs.delete(sid: sid)

2
src/invidious/routes/notifications.cr
View File

@ -18,7 +18,7 @@ module Invidious::Routes::Notifications
if redirect
return env.redirect referer
else
return error_json(403, "No such user")
return Errors.error_json(403, "No such user")
end
end

86
src/invidious/routes/playlists.cr
View File

@ -12,7 +12,7 @@ module Invidious::Routes::Playlists
user = user.as(User)
sid = sid.as(String)
csrf_token = generate_response(sid, {":create_playlist"}, HMAC_KEY)
csrf_token = Invidious::Helpers::Tokens.generate_response(sid, {":create_playlist"}, HMAC_KEY)
templated "create_playlist"
end
@ -31,26 +31,26 @@ module Invidious::Routes::Playlists
token = env.params.body["csrf_token"]?
begin
validate_request(token, sid, env.request, HMAC_KEY, locale)
Invidious::Helpers::Tokens.validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
return error_template(400, ex)
return Errors.error_template(400, ex)
end
title = env.params.body["title"]?.try &.as(String)
if !title || title.empty?
return error_template(400, "Title cannot be empty.")
return Errors.error_template(400, "Title cannot be empty.")
end
privacy = PlaylistPrivacy.parse?(env.params.body["privacy"]?.try &.as(String) || "")
if !privacy
return error_template(400, "Invalid privacy setting.")
return Errors.error_template(400, "Invalid privacy setting.")
end
if Invidious::Database::Playlists.count_owned_by(user.email) >= 100
return error_template(400, "User cannot have more than 100 playlists.")
return Errors.error_template(400, "User cannot have more than 100 playlists.")
end
playlist = create_playlist(title, privacy, user)
playlist = Invidious::Playlists::Playlists.create_playlist(title, privacy, user)
env.redirect "/playlist?list=#{playlist.id}"
end
@ -67,13 +67,13 @@ module Invidious::Routes::Playlists
playlist_id = env.params.query["list"]
begin
playlist = get_playlist(playlist_id)
playlist = Invidious::Playlists::Playlists.get_playlist(playlist_id)
rescue ex : NotFoundException
return error_template(404, ex)
return Errors.error_template(404, ex)
rescue ex
return error_template(500, ex)
return Errors.error_template(500, ex)
end
subscribe_playlist(user, playlist)
Invidious::Playlists::Playlists.subscribe_playlist(user, playlist)
env.redirect "/playlist?list=#{playlist.id}"
end
@ -92,7 +92,7 @@ module Invidious::Routes::Playlists
plid = env.params.query["list"]?
if !plid || plid.empty?
return error_template(400, "A playlist ID is required")
return Errors.error_template(400, "A playlist ID is required")
end
playlist = Invidious::Database::Playlists.select(id: plid)
@ -100,7 +100,7 @@ module Invidious::Routes::Playlists
return env.redirect referer
end
csrf_token = generate_response(sid, {":delete_playlist"}, HMAC_KEY)
csrf_token = Invidious::Helpers::Tokens.generate_response(sid, {":delete_playlist"}, HMAC_KEY)
templated "delete_playlist"
end
@ -122,9 +122,9 @@ module Invidious::Routes::Playlists
token = env.params.body["csrf_token"]?
begin
validate_request(token, sid, env.request, HMAC_KEY, locale)
Invidious::Helpers::Tokens.validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
return error_template(400, ex)
return Errors.error_template(400, ex)
end
playlist = Invidious::Database::Playlists.select(id: plid)
@ -163,12 +163,12 @@ module Invidious::Routes::Playlists
end
begin
items = get_playlist_videos(playlist, offset: (page - 1) * 100)
items = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: (page - 1) * 100)
rescue ex
items = [] of PlaylistVideo
end
csrf_token = generate_response(sid, {":edit_playlist"}, HMAC_KEY)
csrf_token = Invidious::Helpers::Tokens.generate_response(sid, {":edit_playlist"}, HMAC_KEY)
# Pagination
page_nav_html = Frontend::Pagination.nav_numeric(locale,
@ -197,9 +197,9 @@ module Invidious::Routes::Playlists
token = env.params.body["csrf_token"]?
begin
validate_request(token, sid, env.request, HMAC_KEY, locale)
Invidious::Helpers::Tokens.validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
return error_template(400, ex)
return Errors.error_template(400, ex)
end
playlist = Invidious::Database::Playlists.select(id: plid)
@ -286,7 +286,7 @@ module Invidious::Routes::Playlists
if redirect
return env.redirect referer
else
return error_json(403, "No such user")
return Errors.error_json(403, "No such user")
end
end
@ -295,26 +295,26 @@ module Invidious::Routes::Playlists
token = env.params.body["csrf_token"]?
begin
validate_request(token, sid, env.request, HMAC_KEY, locale)
Invidious::Helpers::Tokens.validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
if redirect
return error_template(400, ex)
return Errors.error_template(400, ex)
else
return error_json(400, ex)
return Errors.error_json(400, ex)
end
end
begin
playlist_id = env.params.query["playlist_id"]
playlist = get_playlist(playlist_id).as(InvidiousPlaylist)
playlist = Invidious::Playlists::Playlists.get_playlist(playlist_id).as(InvidiousPlaylist)
raise "Invalid user" if playlist.author != user.email
rescue ex : NotFoundException
return error_json(404, ex)
return Errors.error_json(404, ex)
rescue ex
if redirect
return error_template(400, ex)
return Errors.error_template(400, ex)
else
return error_json(400, ex)
return Errors.error_json(400, ex)
end
end
@ -322,9 +322,9 @@ module Invidious::Routes::Playlists
when "add_video"
if playlist.index.size >= CONFIG.playlist_length_limit
if redirect
return error_template(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos")
return Errors.error_template(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos")
else
return error_json(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos")
return Errors.error_json(400, "Playlist cannot have more than #{CONFIG.playlist_length_limit} videos")
end
end
@ -333,12 +333,12 @@ module Invidious::Routes::Playlists
begin
video = get_video(video_id)
rescue ex : NotFoundException
return error_json(404, ex)
return Errors.error_json(404, ex)
rescue ex
if redirect
return error_template(500, ex)
return Errors.error_template(500, ex)
else
return error_json(500, ex)
return Errors.error_json(500, ex)
end
end
@ -359,7 +359,7 @@ module Invidious::Routes::Playlists
when "remove_video"
index = env.params.query["set_video_id"].to_i64?
if index.nil? || !playlist.index.includes? index
return error_json(404, "Playlist does not contain index")
return Errors.error_json(404, "Playlist does not contain index")
end
Invidious::Database::PlaylistVideos.delete(index, playlist_id)
@ -367,9 +367,9 @@ module Invidious::Routes::Playlists
when "move_video_before"
# TODO: Playlist stub
when nil
return error_json(400, "Missing action")
return Errors.error_json(400, "Missing action")
else
return error_json(400, "Unsupported action #{action}")
return Errors.error_json(400, "Unsupported action #{action}")
end
if redirect
@ -399,11 +399,11 @@ module Invidious::Routes::Playlists
end
begin
playlist = get_playlist(plid)
playlist = Invidious::Playlists::Playlists.get_playlist(plid)
rescue ex : NotFoundException
return error_template(404, ex)
return Errors.error_template(404, ex)
rescue ex
return error_template(500, ex)
return Errors.error_template(500, ex)
end
if playlist.is_a? InvidiousPlaylist
@ -419,17 +419,17 @@ module Invidious::Routes::Playlists
end
if playlist.privacy == PlaylistPrivacy::Private && playlist.author != user.try &.email
return error_template(403, "This playlist is private.")
return Errors.error_template(403, "This playlist is private.")
end
begin
if playlist.is_a? InvidiousPlaylist
items = get_playlist_videos(playlist, offset: (page - 1) * 100)
items = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: (page - 1) * 100)
else
items = get_playlist_videos(playlist, offset: (page - 1) * 200)
items = Invidious::Playlists::Playlists.get_playlist_videos(playlist, offset: (page - 1) * 200)
end
rescue ex
return error_template(500, "Error encountered while retrieving playlist videos.<br>#{ex.message}")
return Errors.error_template(500, "Error encountered while retrieving playlist videos.<br>#{ex.message}")
end
if playlist.author == user.try &.email
@ -460,7 +460,7 @@ module Invidious::Routes::Playlists
begin
mix = fetch_mix(rdid, continuation, locale: locale)
rescue ex
return error_template(500, ex)
return Errors.error_template(500, ex)
end
templated "mix"

8
src/invidious/routes/preferences.cr
View File

@ -329,7 +329,7 @@ module Invidious::Routes::PreferencesRoute
if !success
haltf(env, status_code: 415,
response: error_template(415, "Invalid subscription file uploaded")
response: Errors.error_template(415, "Invalid subscription file uploaded")
)
end
when "import_youtube_pl"
@ -338,7 +338,7 @@ module Invidious::Routes::PreferencesRoute
if !success
haltf(env, status_code: 415,
response: error_template(415, "Invalid playlist file uploaded")
response: Errors.error_template(415, "Invalid playlist file uploaded")
)
end
when "import_youtube_wh"
@ -347,7 +347,7 @@ module Invidious::Routes::PreferencesRoute
if !success
haltf(env, status_code: 415,
response: error_template(415, "Invalid watch history file uploaded")
response: Errors.error_template(415, "Invalid watch history file uploaded")
)
end
when "import_freetube"
@ -359,7 +359,7 @@ module Invidious::Routes::PreferencesRoute
if !success
haltf(env, status_code: 415,
response: error_template(415, "Uploaded file is too large")
response: Errors.error_template(415, "Uploaded file is too large")
)
end
else nil # Ignore

10
src/invidious/routes/search.cr
View File

@ -71,9 +71,9 @@ module Invidious::Routes::Search
items = query.process
end
rescue ex : ChannelSearchException
return error_template(404, "Unable to find channel with id of '#{HTML.escape(ex.channel)}'. Are you sure that's an actual channel id? It should look like 'UC4QobU6STFB0P71PMvOGN5A'.")
return Errors.error_template(404, "Unable to find channel with id of '#{HTML.escape(ex.channel)}'. Are you sure that's an actual channel id? It should look like 'UC4QobU6STFB0P71PMvOGN5A'.")
rescue ex
return error_template(500, ex)
return Errors.error_template(500, ex)
end
redirect_url = Invidious::Frontend::Misc.redirect_url(env)
@ -100,7 +100,7 @@ module Invidious::Routes::Search
hashtag = env.params.url["hashtag"]?
if hashtag.nil? || hashtag.empty?
return error_template(400, "Invalid request")
return Errors.error_template(400, "Invalid request")
end
page = env.params.query["page"]?
@ -112,9 +112,9 @@ module Invidious::Routes::Search
end
begin
items = Invidious::Hashtag.fetch(hashtag, page)
items = Invidious::Search::Hashtag.fetch_hashtag(hashtag, page)
rescue ex
return error_template(500, ex)
return Errors.error_template(500, ex)
end
# Pagination

12
src/invidious/routes/subscriptions.cr
View File

@ -14,7 +14,7 @@ module Invidious::Routes::Subscriptions
if redirect
return env.redirect referer
else
return error_json(403, "No such user")
return Errors.error_json(403, "No such user")
end
end
@ -23,12 +23,12 @@ module Invidious::Routes::Subscriptions
token = env.params.body["csrf_token"]?
begin
validate_request(token, sid, env.request, HMAC_KEY, locale)
Invidious::Helpers::Tokens.validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
if redirect
return error_template(400, ex)
return Errors.error_template(400, ex)
else
return error_json(400, ex)
return Errors.error_json(400, ex)
end
end
@ -38,13 +38,13 @@ module Invidious::Routes::Subscriptions
case action = env.params.query["action"]?
when "create_subscription_to_channel"
if !user.subscriptions.includes? channel_id
get_channel(channel_id)
Invidious::Channels::Channels.get_channel(channel_id)
Invidious::Database::Users.subscribe_channel(user, channel_id)
end
when "remove_subscriptions"
Invidious::Database::Users.unsubscribe_channel(user, channel_id)
else
return error_json(400, "Unsupported action #{action}")
return Errors.error_json(400, "Unsupported action #{action}")
end
if redirect

20
src/invidious/routes/video_playback.cr
View File

@ -24,7 +24,7 @@ module Invidious::Routes::VideoPlayback
# Sanity check, to avoid being used as an open proxy
if !host.matches?(/[\w-]+\.(?:googlevideo|c\.youtube)\.com/)
return error_template(400, "Invalid \"host\" parameter.")
return Errors.error_template(400, "Invalid \"host\" parameter.")
end
host = "https://#{host}"
@ -83,7 +83,7 @@ module Invidious::Routes::VideoPlayback
# Remove the Range header added previously.
headers.delete("Range") if range_header.nil?
playback_statistics = Helpers.get_playback_statistic
playback_statistics = Invidious::Helpers.get_playback_statistic
playback_statistics["totalRequests"] += 1
if response.status_code >= 400
@ -95,7 +95,7 @@ module Invidious::Routes::VideoPlayback
if url.includes? "&file=seg.ts"
if CONFIG.disabled?("livestreams")
return error_template(403, "Administrator has disabled this endpoint.")
return Errors.error_template(403, "Administrator has disabled this endpoint.")
end
begin
@ -120,7 +120,7 @@ module Invidious::Routes::VideoPlayback
else
if query_params["title"]? && CONFIG.disabled?("downloads") ||
CONFIG.disabled?("dash")
return error_template(403, "Administrator has disabled this endpoint.")
return Errors.error_template(403, "Administrator has disabled this endpoint.")
end
content_length = nil
@ -195,7 +195,7 @@ module Invidious::Routes::VideoPlayback
end
end
Helpers.proxy_file(resp, env)
Invidious::Helpers.proxy_file(resp, env)
end
rescue ex
if ex.message != "Error reading socket: Connection reset by peer"
@ -269,11 +269,11 @@ module Invidious::Routes::VideoPlayback
# Sanity checks
unless id && validate_video_id(id)
return error_template(400, InvalidVideoID.new(id))
return Errors.error_template(400, InvalidVideoID.new(id))
end
if !itag.nil? && (itag <= 0 || itag >= 1000)
return error_template(400, "Invalid itag")
return Errors.error_template(400, "Invalid itag")
end
region = env.params.query["region"]?
@ -282,15 +282,15 @@ module Invidious::Routes::VideoPlayback
title = env.params.query["title"]?
if title && CONFIG.disabled?("downloads")
return error_template(403, "Administrator has disabled this endpoint.")
return Errors.error_template(403, "Administrator has disabled this endpoint.")
end
begin
video = get_video(id, region: region)
rescue ex : NotFoundException
return error_template(404, ex)
return Errors.error_template(404, ex)
rescue ex
return error_template(500, ex)
return Errors.error_template(500, ex)
end
if itag.nil?

32
src/invidious/routes/watch.cr
View File

@ -15,11 +15,11 @@ module Invidious::Routes::Watch
id = env.params.query["v"]
if env.params.query["v"].empty?
return error_template(400, "Invalid parameters.")
return Errors.error_template(400, "Invalid parameters.")
end
unless validate_video_id(id)
return error_template(400, InvalidVideoID.new(id))
return Errors.error_template(400, InvalidVideoID.new(id))
end
else
return env.redirect "/"
@ -48,10 +48,10 @@ module Invidious::Routes::Watch
video = get_video(id, region: params.region)
rescue ex : NotFoundException
LOGGER.error("get_video not found: #{id} : #{ex.message}")
return error_template(404, ex)
return Errors.error_template(404, ex)
rescue ex
LOGGER.error("get_video: #{id} : #{ex.message}")
return error_template(500, ex)
return Errors.error_template(500, ex)
end
if preferences.annotations_subscribed &&
@ -220,7 +220,7 @@ module Invidious::Routes::Watch
if redirect
return env.redirect referer
else
return error_json(403, "No such user")
return Errors.error_json(403, "No such user")
end
end
@ -235,12 +235,12 @@ module Invidious::Routes::Watch
end
begin
validate_request(token, sid, env.request, HMAC_KEY, locale)
Invidious::Helpers::Tokens.validate_request(token, sid, env.request, HMAC_KEY, locale)
rescue ex
if redirect
return error_template(400, ex)
return Errors.error_template(400, ex)
else
return error_json(400, ex)
return Errors.error_json(400, ex)
end
end
@ -250,7 +250,7 @@ module Invidious::Routes::Watch
when "mark_unwatched"
Invidious::Database::Users.mark_unwatched(user, id)
else
return error_json(400, "Unsupported action #{action}")
return Errors.error_json(400, "Unsupported action #{action}")
end
if redirect
@ -264,10 +264,10 @@ module Invidious::Routes::Watch
def self.clip(env)
clip_id = env.params.url["clip"]?
return error_template(400, "A clip ID is required") if !clip_id
return Errors.error_template(400, "A clip ID is required") if !clip_id
response = YoutubeAPI.resolve_url("https://www.youtube.com/clip/#{clip_id}")
return error_template(400, "Invalid clip ID") if response["error"]?
return Errors.error_template(400, "Invalid clip ID") if response["error"]?
if video_id = response.dig?("endpoint", "watchEndpoint", "videoId")
if params = response.dig?("endpoint", "watchEndpoint", "params").try &.as_s
@ -278,16 +278,16 @@ module Invidious::Routes::Watch
return env.redirect "/watch?v=#{video_id}&#{env.params.query}"
else
return error_template(404, "The requested clip doesn't exist")
return Errors.error_template(404, "The requested clip doesn't exist")
end
end
def self.download(env)
if CONFIG.disabled?("downloads")
return error_template(403, "Administrator has disabled this endpoint.")
return Errors.error_template(403, "Administrator has disabled this endpoint.")
end
if CONFIG.invidious_companion.present?
return error_template(403, "Downloads should be routed through Companion when present")
return Errors.error_template(403, "Downloads should be routed through Companion when present")
end
title = env.params.body["title"]? || ""
@ -295,7 +295,7 @@ module Invidious::Routes::Watch
selection = env.params.body["download_widget"]?
if title.empty? || video_id.empty? || selection.nil?
return error_template(400, "Missing form data")
return Errors.error_template(400, "Missing form data")
end
download_widget = JSON.parse(selection)
@ -326,7 +326,7 @@ module Invidious::Routes::Watch
return Invidious::Routes::VideoPlayback.latest_version(env)
else
return error_template(400, "Invalid label or itag")
return Errors.error_template(400, "Invalid label or itag")
end
end
end

56
src/invidious/search/ctoken.cr
View File

@ -1,32 +1,36 @@
def produce_channel_search_continuation(ucid, query, page)
if page <= 1
idx = 0_i64
else
idx = 30_i64 * (page - 1)
end
module Invidious::Search::CToken
extend self
object = {
"80226972:embedded" => {
"2:string" => ucid,
"3:base64" => {
"2:string" => "search",
"6:varint" => 1_i64,
"7:varint" => 1_i64,
"12:varint" => 1_i64,
"15:base64" => {
"3:varint" => idx,
def produce_channel_search_continuation(ucid, query, page)
if page <= 1
idx = 0_i64
else
idx = 30_i64 * (page - 1)
end
object = {
"80226972:embedded" => {
"2:string" => ucid,
"3:base64" => {
"2:string" => "search",
"6:varint" => 1_i64,
"7:varint" => 1_i64,
"12:varint" => 1_i64,
"15:base64" => {
"3:varint" => idx,
},
"23:varint" => 0_i64,
},
"23:varint" => 0_i64,
"11:string" => query,
"35:string" => "browse-feed#{ucid}search",
},
"11:string" => query,
"35:string" => "browse-feed#{ucid}search",
},
}
}
continuation = object.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
continuation = object.try { |i| Protodec::Any.cast_json(i) }
.try { |i| Protodec::Any.from_json(i) }
.try { |i| Base64.urlsafe_encode(i) }
.try { |i| URI.encode_www_form(i) }
return continuation
return continuation
end
end

4
src/invidious/hashtag.cr → src/invidious/search/hashtag.cr
View File

@ -1,7 +1,7 @@
module Invidious::Hashtag
module Invidious::Search::Hashtag
extend self
def fetch(hashtag : String, page : Int, region : String? = nil) : Array(SearchItem)
def fetch_hashtag(hashtag : String, page : Int, region : String? = nil) : Array(SearchItem)
cursor = (page - 1) * 60
ctoken = generate_continuation(hashtag, cursor)

74
src/invidious/search/processors.cr
View File

@ -1,45 +1,44 @@
module Invidious::Search
module Processors
extend self
module Invidious::Search::Processors
extend self
# Regular search (`/search` endpoint)
def regular(query : Query) : Array(SearchItem)
search_params = query.filters.to_yt_params(page: query.page)
# Regular search (`/search` endpoint)
def regular(query : Query) : Array(SearchItem)
search_params = query.filters.to_yt_params(page: query.page)
client_config = YoutubeAPI::ClientConfig.new(region: query.region)
initial_data = YoutubeAPI.search(query.text, search_params, client_config: client_config)
client_config = YoutubeAPI::ClientConfig.new(region: query.region)
initial_data = YoutubeAPI.search(query.text, search_params, client_config: client_config)
items, _ = extract_items(initial_data)
return items.reject!(Category)
items, _ = extract_items(initial_data)
return items.reject!(Category)
end
# Search a youtube channel
# TODO: clean code, and rely more on YoutubeAPI
def channel(query : Query) : Array(SearchItem)
response = YT_POOL.client &.get("/channel/#{query.channel}")
if response.status_code == 404
response = YT_POOL.client &.get("/user/#{query.channel}")
response = YT_POOL.client &.get("/c/#{query.channel}") if response.status_code == 404
initial_data = Invidious::Helpers.extract_initial_data(response.body)
ucid = initial_data.dig?("header", "c4TabbedHeaderRenderer", "channelId").try(&.as_s?)
raise ChannelSearchException.new(query.channel) if !ucid
else
ucid = query.channel
end
# Search a youtube channel
# TODO: clean code, and rely more on YoutubeAPI
def channel(query : Query) : Array(SearchItem)
response = YT_POOL.client &.get("/channel/#{query.channel}")
continuation = Invidious::Search::CToken.produce_channel_search_continuation(ucid, query.text, query.page)
response_json = YoutubeAPI.browse(continuation)
if response.status_code == 404
response = YT_POOL.client &.get("/user/#{query.channel}")
response = YT_POOL.client &.get("/c/#{query.channel}") if response.status_code == 404
initial_data = Helpers.extract_initial_data(response.body)
ucid = initial_data.dig?("header", "c4TabbedHeaderRenderer", "channelId").try(&.as_s?)
raise ChannelSearchException.new(query.channel) if !ucid
else
ucid = query.channel
end
items, _ = extract_items(response_json, "", ucid)
return items.reject!(Category)
end
continuation = produce_channel_search_continuation(ucid, query.text, query.page)
response_json = YoutubeAPI.browse(continuation)
# Search inside of user subscriptions
def subscriptions(query : Query, user : Invidious::User) : Array(ChannelVideo)
view_name = "subscriptions_#{sha256(user.email)}"
items, _ = extract_items(response_json, "", ucid)
return items.reject!(Category)
end
# Search inside of user subscriptions
def subscriptions(query : Query, user : Invidious::User) : Array(ChannelVideo)
view_name = "subscriptions_#{sha256(user.email)}"
return PG_DB.query_all("
return PG_DB.query_all("
SELECT id,title,published,updated,ucid,author,length_seconds
FROM (
SELECT *,
@ -48,9 +47,8 @@ module Invidious::Search
as document
FROM #{view_name}
) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;",
query.text, (query.page - 1) * 20,
as: ChannelVideo
)
end
query.text, (query.page - 1) * 20,
as: ChannelVideo
)
end
end

48
src/invidious/trending.cr
View File

@ -1,48 +0,0 @@
def fetch_trending(trending_type, region, locale)
region ||= "US"
region = region.upcase
plid = nil
browse_id = ""
case trending_type.try &.downcase
when "gaming"
browse_id = "UCOpNcN46UbXVtpKMrmU4Abg"
params = "Egh0cmVuZGluZw%3D%3D"
when "livestreams"
browse_id = "UC4R8DWoMoI7CAwX8_LjQHig"
params = "EgdsaXZldGFikgEDCKEK"
else
# Livestreams is the default one as Youtube removed
# the aggregated trending page
# https://github.com/iv-org/invidious/issues/5397#issuecomment-3218928458
browse_id = "UC4R8DWoMoI7CAwX8_LjQHig"
params = "EgdsaXZldGFikgEDCKEK"
end
client_config = YoutubeAPI::ClientConfig.new(region: region)
initial_data = YoutubeAPI.browse(browse_id, params: params, client_config: client_config)
items, _ = extract_items(initial_data)
extracted = [] of SearchItem
deduplicate = items.size > 1
items.each do |itm|
if itm.is_a?(Category)
# Ignore the smaller categories, as they generally contain a sponsored
# channel, which brings a lot of noise on the trending page.
# See: https://github.com/iv-org/invidious/issues/2989
next if (itm.contents.size < 24 && deduplicate)
extracted.concat itm.contents.select(SearchItem)
else
extracted << itm
end
end
# Deduplicate items before returning results
return extracted.select(SearchVideo | ProblematicTimelineItem).uniq!(&.id), plid
end

2
src/invidious/user/captcha.cr
View File

@ -55,7 +55,7 @@ struct Invidious::User
return {
question: image,
tokens: {generate_response(answer, {":login"}, key, use_nonce: true)},
tokens: {Invidious::Helpers::Tokens.generate_response(answer, {":login"}, key, use_nonce: true)},
}
end
end

24
src/invidious/user/converters.cr
View File

@ -1,12 +1,16 @@
def convert_theme(theme)
case theme
when "true"
"dark"
when "false"
"light"
when "", nil
nil
else
theme
module Invidious::User::Converters
extend self
def convert_theme(theme)
case theme
when "true"
"dark"
when "false"
"light"
when "", nil
nil
else
theme
end
end
end

2
src/invidious/user/exports.cr
View File

@ -15,7 +15,7 @@ struct Invidious::User
playlists.each do |playlist|
json.object do
json.field "title", playlist.title
json.field "description", Helpers.html_to_content(playlist.description_html)
json.field "description", Invidious::Helpers.html_to_content(playlist.description_html)
json.field "privacy", playlist.privacy.to_s
json.field "videos" do
json.array do

4
src/invidious/user/imports.cr
View File

@ -43,7 +43,7 @@ struct Invidious::User
description = "This is the default description of an imported playlist. Feel Free to change it as you see fit."
privacy = PlaylistPrivacy::Private
playlist = create_playlist(title, privacy, user)
playlist = Invidious::Playlists::Playlists.create_playlist(title, privacy, user)
Invidious::Database::Playlists.update_description(playlist.id, description)
# Add each video to the playlist from the body content
@ -117,7 +117,7 @@ struct Invidious::User
next if !description
next if !privacy
playlist = create_playlist(title, privacy, user)
playlist = Invidious::Playlists::Playlists.create_playlist(title, privacy, user)
Invidious::Database::Playlists.update_description(playlist.id, description)
item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx|

109
src/invidious/user/users.cr Normal file
View File

@ -0,0 +1,109 @@
require "crypto/bcrypt/password"
module Invidious::User::Users
extend self
# Materialized views may not be defined using bound parameters (`$1` as used elsewhere)
MATERIALIZED_VIEW_SQL = ->(email : String) { "SELECT cv.* FROM channel_videos cv WHERE EXISTS (SELECT subscriptions FROM users u WHERE cv.ucid = ANY (u.subscriptions) AND u.email = E'#{email.gsub({'\'' => "\\'", '\\' => "\\\\"})}') ORDER BY published DESC" }
def create_user(sid, email, password)
password = Crypto::Bcrypt::Password.create(password, cost: 10)
token = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
user = Invidious::User.new({
updated: Time.utc,
notifications: [] of String,
subscriptions: [] of String,
email: email,
preferences: Preferences.new(CONFIG.default_user_preferences.to_tuple),
password: password.to_s,
token: token,
watched: [] of String,
feed_needs_update: true,
})
return user, sid
end
def get_subscription_feed(user, max_results = 40, page = 1)
limit = max_results.clamp(0, MAX_ITEMS_PER_PAGE)
offset = (page - 1) * limit
notifications = Invidious::Database::Users.select_notifications(user)
view_name = "subscriptions_#{sha256(user.email)}"
if user.preferences.notifications_only && !notifications.empty?
# Only show notifications
notifications = Invidious::Database::ChannelVideos.select(notifications)
videos = [] of ChannelVideo
notifications.sort_by!(&.published).reverse!
case user.preferences.sort
when "alphabetically"
notifications.sort_by!(&.title)
when "alphabetically - reverse"
notifications.sort_by!(&.title).reverse!
when "channel name"
notifications.sort_by!(&.author)
when "channel name - reverse"
notifications.sort_by!(&.author).reverse!
else nil # Ignore
end
else
if user.preferences.latest_only
if user.preferences.unseen_only
# Show latest video from a channel that a user hasn't watched
# "unseen_only" isn't really correct here, more accurate would be "unwatched_only"
if user.watched.empty?
values = "'{}'"
else
values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}"
end
videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY ucid, published DESC", as: ChannelVideo)
else
# Show latest video from each channel
videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} ORDER BY ucid, published DESC", as: ChannelVideo)
end
videos.sort_by!(&.published).reverse!
else
if user.preferences.unseen_only
# Only show unwatched
if user.watched.empty?
values = "'{}'"
else
values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}"
end
videos = PG_DB.query_all("SELECT * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo)
else
# Sort subscriptions as normal
videos = PG_DB.query_all("SELECT * FROM #{view_name} ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo)
end
end
case user.preferences.sort
when "published - reverse"
videos.sort_by!(&.published)
when "alphabetically"
videos.sort_by!(&.title)
when "alphabetically - reverse"
videos.sort_by!(&.title).reverse!
when "channel name"
videos.sort_by!(&.author)
when "channel name - reverse"
videos.sort_by!(&.author).reverse!
else nil # Ignore
end
notifications = Invidious::Database::Users.select_notifications(user)
notifications = videos.select { |v| notifications.includes? v.id }
videos = videos - notifications
end
return videos, notifications
end
end

106
src/invidious/users.cr
View File

@ -1,106 +0,0 @@
require "crypto/bcrypt/password"
# Materialized views may not be defined using bound parameters (`$1` as used elsewhere)
MATERIALIZED_VIEW_SQL = ->(email : String) { "SELECT cv.* FROM channel_videos cv WHERE EXISTS (SELECT subscriptions FROM users u WHERE cv.ucid = ANY (u.subscriptions) AND u.email = E'#{email.gsub({'\'' => "\\'", '\\' => "\\\\"})}') ORDER BY published DESC" }
def create_user(sid, email, password)
password = Crypto::Bcrypt::Password.create(password, cost: 10)
token = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
user = Invidious::User.new({
updated: Time.utc,
notifications: [] of String,
subscriptions: [] of String,
email: email,
preferences: Preferences.new(CONFIG.default_user_preferences.to_tuple),
password: password.to_s,
token: token,
watched: [] of String,
feed_needs_update: true,
})
return user, sid
end
def get_subscription_feed(user, max_results = 40, page = 1)
limit = max_results.clamp(0, MAX_ITEMS_PER_PAGE)
offset = (page - 1) * limit
notifications = Invidious::Database::Users.select_notifications(user)
view_name = "subscriptions_#{sha256(user.email)}"
if user.preferences.notifications_only && !notifications.empty?
# Only show notifications
notifications = Invidious::Database::ChannelVideos.select(notifications)
videos = [] of ChannelVideo
notifications.sort_by!(&.published).reverse!
case user.preferences.sort
when "alphabetically"
notifications.sort_by!(&.title)
when "alphabetically - reverse"
notifications.sort_by!(&.title).reverse!
when "channel name"
notifications.sort_by!(&.author)
when "channel name - reverse"
notifications.sort_by!(&.author).reverse!
else nil # Ignore
end
else
if user.preferences.latest_only
if user.preferences.unseen_only
# Show latest video from a channel that a user hasn't watched
# "unseen_only" isn't really correct here, more accurate would be "unwatched_only"
if user.watched.empty?
values = "'{}'"
else
values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}"
end
videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY ucid, published DESC", as: ChannelVideo)
else
# Show latest video from each channel
videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} ORDER BY ucid, published DESC", as: ChannelVideo)
end
videos.sort_by!(&.published).reverse!
else
if user.preferences.unseen_only
# Only show unwatched
if user.watched.empty?
values = "'{}'"
else
values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}"
end
videos = PG_DB.query_all("SELECT * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo)
else
# Sort subscriptions as normal
videos = PG_DB.query_all("SELECT * FROM #{view_name} ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo)
end
end
case user.preferences.sort
when "published - reverse"
videos.sort_by!(&.published)
when "alphabetically"
videos.sort_by!(&.title)
when "alphabetically - reverse"
videos.sort_by!(&.title).reverse!
when "channel name"
videos.sort_by!(&.author)
when "channel name - reverse"
videos.sort_by!(&.author).reverse!
else nil # Ignore
end
notifications = Invidious::Database::Users.select_notifications(user)
notifications = videos.select { |v| notifications.includes? v.id }
videos = videos - notifications
end
return videos, notifications
end

2
src/invidious/videos/parser.cr
View File

@ -87,7 +87,7 @@ module Invidious::Videos::Parser
# Although technically not a call to /videoplayback the fact that YouTube is returning the
# wrong video means that we should count it as a failure.
Helpers.get_playback_statistic["totalRequests"] += 1
Invidious::Helpers.get_playback_statistic["totalRequests"] += 1
return {
"version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),

0
src/invidious/videos.cr → src/invidious/videos/videos.cr
View File

2
src/invidious/views/components/item.ecr
View File

@ -106,7 +106,7 @@
</div>
<details>
<summary class="pure-button pure-button-secondary"><%=I18n.translate(locale, "timeline_parse_error_show_technical_details")%></summary>
<pre class="error-issue-template"><%=get_issue_template(env, item.parse_exception)[1]%></pre>
<pre class="error-issue-template"><%=Errors.get_issue_template(env, item.parse_exception)[1]%></pre>
</details>
</div>
<% else %>