mirror of
https://github.com/iv-org/invidious.git
synced 2026-09-06 09:02:45 -05:00
fix(captions): prefer regional language codes over base fallback
Keep region-specific LANGUAGES names such as English (United States) mapped to en-us, then fall back to en. Rank the regional code ahead of en-GB and other base-language matches.
This commit is contained in:
XZH
parent
3df2266294
commit
c4e2bc34c5
@ -27,7 +27,6 @@ Spectator.describe Invidious::Videos::Captions do
|
|||||||
expect(matched.map(&.name)).to eq(["English (United States)", "English (United Kingdom)"])
|
expect(matched.map(&.name)).to eq(["English (United States)", "English (United Kingdom)"])
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
||||||
it "matches English preference to a code-labeled en track" do
|
it "matches English preference to a code-labeled en track" do
|
||||||
code_only = Invidious::Videos::Captions::Metadata.new("en", "en", "http://x", true)
|
code_only = Invidious::Videos::Captions::Metadata.new("en", "en", "http://x", true)
|
||||||
arabic = Invidious::Videos::Captions::Metadata.new("ar", "ar", "http://x", true)
|
arabic = Invidious::Videos::Captions::Metadata.new("ar", "ar", "http://x", true)
|
||||||
@ -44,6 +43,14 @@ Spectator.describe Invidious::Videos::Captions do
|
|||||||
expect(matched.map(&.name)).to eq(["en-US"])
|
expect(matched.map(&.name)).to eq(["en-US"])
|
||||||
end
|
end
|
||||||
|
|
||||||
|
it "prefers en-US over en-GB for English (United States)" do
|
||||||
|
us = Invidious::Videos::Captions::Metadata.new("en-US", "en-US", "http://x", false)
|
||||||
|
uk = Invidious::Videos::Captions::Metadata.new("en-GB", "en-GB", "http://x", false)
|
||||||
|
|
||||||
|
matched = Invidious::Videos::Captions.matching([uk, us], ["English (United States)"])
|
||||||
|
expect(matched.map(&.name)).to eq(["en-US", "en-GB"])
|
||||||
|
end
|
||||||
|
|
||||||
it "ignores blank preference slots" do
|
it "ignores blank preference slots" do
|
||||||
arabic = Invidious::Videos::Captions::Metadata.new("Arabic (auto-generated)", "ar", "http://x", true)
|
arabic = Invidious::Videos::Captions::Metadata.new("Arabic (auto-generated)", "ar", "http://x", true)
|
||||||
matched = Invidious::Videos::Captions.matching([arabic], ["", "", ""])
|
matched = Invidious::Videos::Captions.matching([arabic], ["", "", ""])
|
||||||
|
|||||||
@ -101,173 +101,114 @@ module Invidious::Videos
|
|||||||
names.any? { |name| name_matches?(caption, name) }
|
names.any? { |name| name_matches?(caption, name) }
|
||||||
end
|
end
|
||||||
|
|
||||||
# Map display-language preferences (e.g. "English") onto ISO-ish codes so
|
# Common LANGUAGES names mapped to ISO 639-1 / BCP-47 codes.
|
||||||
# they can match caption names/codes like "en" or "en-US".
|
# Auto-generated suffixes are stripped before lookup; regional
|
||||||
|
# qualifiers keep their more specific code and fall back to the base.
|
||||||
NAME_TO_CODE = {
|
NAME_TO_CODE = {
|
||||||
"afrikaans" => "af",
|
"arabic" => "ar",
|
||||||
"albanian" => "sq",
|
"chinese" => "zh",
|
||||||
"amharic" => "am",
|
"chinese (china)" => "zh-cn",
|
||||||
"arabic" => "ar",
|
"chinese (hong kong)" => "zh-hk",
|
||||||
"armenian" => "hy",
|
"chinese (simplified)" => "zh-hans",
|
||||||
"azerbaijani" => "az",
|
"chinese (taiwan)" => "zh-tw",
|
||||||
"bangla" => "bn",
|
"chinese (traditional)" => "zh-hant",
|
||||||
"basque" => "eu",
|
"dutch" => "nl",
|
||||||
"belarusian" => "be",
|
"english" => "en",
|
||||||
"bosnian" => "bs",
|
"english (united kingdom)" => "en-gb",
|
||||||
"bulgarian" => "bg",
|
"english (united states)" => "en-us",
|
||||||
"burmese" => "my",
|
"filipino" => "fil",
|
||||||
"cantonese" => "yue",
|
"french" => "fr",
|
||||||
"catalan" => "ca",
|
"german" => "de",
|
||||||
"cebuano" => "ceb",
|
"indonesian" => "id",
|
||||||
"chinese" => "zh",
|
"italian" => "it",
|
||||||
"corsican" => "co",
|
"japanese" => "ja",
|
||||||
"croatian" => "hr",
|
"korean" => "ko",
|
||||||
"czech" => "cs",
|
"portuguese" => "pt",
|
||||||
"danish" => "da",
|
"portuguese (brazil)" => "pt-br",
|
||||||
"dutch" => "nl",
|
"russian" => "ru",
|
||||||
"english" => "en",
|
"spanish" => "es",
|
||||||
"esperanto" => "eo",
|
"spanish (latin america)" => "es-419",
|
||||||
"estonian" => "et",
|
"spanish (mexico)" => "es-mx",
|
||||||
"filipino" => "fil",
|
"spanish (spain)" => "es-es",
|
||||||
"finnish" => "fi",
|
"turkish" => "tr",
|
||||||
"french" => "fr",
|
"vietnamese" => "vi",
|
||||||
"galician" => "gl",
|
|
||||||
"georgian" => "ka",
|
|
||||||
"german" => "de",
|
|
||||||
"greek" => "el",
|
|
||||||
"gujarati" => "gu",
|
|
||||||
"haitian creole" => "ht",
|
|
||||||
"hausa" => "ha",
|
|
||||||
"hawaiian" => "haw",
|
|
||||||
"hebrew" => "he",
|
|
||||||
"hindi" => "hi",
|
|
||||||
"hmong" => "hmn",
|
|
||||||
"hungarian" => "hu",
|
|
||||||
"icelandic" => "is",
|
|
||||||
"igbo" => "ig",
|
|
||||||
"indonesian" => "id",
|
|
||||||
"interlingue" => "ie",
|
|
||||||
"irish" => "ga",
|
|
||||||
"italian" => "it",
|
|
||||||
"japanese" => "ja",
|
|
||||||
"javanese" => "jv",
|
|
||||||
"kannada" => "kn",
|
|
||||||
"kazakh" => "kk",
|
|
||||||
"khmer" => "km",
|
|
||||||
"korean" => "ko",
|
|
||||||
"kurdish" => "ku",
|
|
||||||
"kyrgyz" => "ky",
|
|
||||||
"lao" => "lo",
|
|
||||||
"latin" => "la",
|
|
||||||
"latvian" => "lv",
|
|
||||||
"lithuanian" => "lt",
|
|
||||||
"luxembourgish" => "lb",
|
|
||||||
"macedonian" => "mk",
|
|
||||||
"malagasy" => "mg",
|
|
||||||
"malay" => "ms",
|
|
||||||
"malayalam" => "ml",
|
|
||||||
"maltese" => "mt",
|
|
||||||
"maori" => "mi",
|
|
||||||
"marathi" => "mr",
|
|
||||||
"mongolian" => "mn",
|
|
||||||
"nepali" => "ne",
|
|
||||||
"norwegian bokmål" => "nb",
|
|
||||||
"norwegian bokmal" => "nb",
|
|
||||||
"nyanja" => "ny",
|
|
||||||
"pashto" => "ps",
|
|
||||||
"persian" => "fa",
|
|
||||||
"polish" => "pl",
|
|
||||||
"portuguese" => "pt",
|
|
||||||
"punjabi" => "pa",
|
|
||||||
"romanian" => "ro",
|
|
||||||
"russian" => "ru",
|
|
||||||
"samoan" => "sm",
|
|
||||||
"scottish gaelic" => "gd",
|
|
||||||
"serbian" => "sr",
|
|
||||||
"shona" => "sn",
|
|
||||||
"sindhi" => "sd",
|
|
||||||
"sinhala" => "si",
|
|
||||||
"slovak" => "sk",
|
|
||||||
"slovenian" => "sl",
|
|
||||||
"somali" => "so",
|
|
||||||
"southern sotho" => "st",
|
|
||||||
"spanish" => "es",
|
|
||||||
"sundanese" => "su",
|
|
||||||
"swahili" => "sw",
|
|
||||||
"swedish" => "sv",
|
|
||||||
"tajik" => "tg",
|
|
||||||
"tamil" => "ta",
|
|
||||||
"telugu" => "te",
|
|
||||||
"thai" => "th",
|
|
||||||
"turkish" => "tr",
|
|
||||||
"ukrainian" => "uk",
|
|
||||||
"urdu" => "ur",
|
|
||||||
"uzbek" => "uz",
|
|
||||||
"vietnamese" => "vi",
|
|
||||||
"welsh" => "cy",
|
|
||||||
"western frisian" => "fy",
|
|
||||||
"xhosa" => "xh",
|
|
||||||
"yiddish" => "yi",
|
|
||||||
"yoruba" => "yo",
|
|
||||||
"zulu" => "zu",
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private def self.name_matches?(caption : Metadata, name : String) : Bool
|
private def self.name_matches?(caption : Metadata, name : String) : Bool
|
||||||
needle = name.strip.downcase
|
!match_index(caption, name).nil?
|
||||||
return false if needle.empty?
|
|
||||||
|
|
||||||
caption_name = caption.name.downcase
|
|
||||||
lang = caption.language_code.downcase
|
|
||||||
base_lang = lang.split("-")[0]
|
|
||||||
|
|
||||||
return true if caption_name == needle ||
|
|
||||||
caption_name.starts_with?(needle + " (") ||
|
|
||||||
caption_name.starts_with?(needle + " - ") ||
|
|
||||||
lang == needle ||
|
|
||||||
base_lang == needle
|
|
||||||
|
|
||||||
# Display-language preferences such as "English" must also match
|
|
||||||
# code-labeled tracks like "en" / "en-US".
|
|
||||||
needle_codes = canonical_codes(needle)
|
|
||||||
caption_codes = canonical_codes(caption_name)
|
|
||||||
caption_codes << lang
|
|
||||||
caption_codes << base_lang
|
|
||||||
needle_codes.any? { |code| caption_codes.includes?(code) }
|
|
||||||
end
|
end
|
||||||
|
|
||||||
private def self.canonical_codes(value : String) : Array(String)
|
# Tokens from most specific to least: "English (United States)" ->
|
||||||
raw = value.strip.downcase
|
# "english (united states)", "en-us", "english", "en".
|
||||||
return [] of String if raw.empty?
|
private def self.match_tokens(name : String) : Array(String)
|
||||||
|
token = name.strip.downcase
|
||||||
|
return [] of String if token.empty?
|
||||||
|
|
||||||
codes = [] of String
|
lookup = token
|
||||||
base_name = raw.split(" - ")[0].split(" (")[0].strip
|
if lookup.ends_with?(" (auto-generated)")
|
||||||
|
lookup = lookup.rchop(" (auto-generated)")
|
||||||
if mapped = NAME_TO_CODE[base_name]?
|
|
||||||
codes << mapped
|
|
||||||
end
|
end
|
||||||
|
|
||||||
if raw.includes?("-")
|
tokens = [lookup]
|
||||||
codes << raw
|
if mapped = NAME_TO_CODE[lookup]?
|
||||||
codes << raw.split("-")[0]
|
tokens << mapped
|
||||||
elsif raw.size.in?(2..3) && raw.chars.all?(&.ascii_letter?)
|
|
||||||
codes << raw
|
|
||||||
end
|
end
|
||||||
|
|
||||||
codes.uniq!
|
base_name = lookup.split(" - ")[0].split(" (")[0].strip
|
||||||
codes
|
if !base_name.empty? && base_name != lookup
|
||||||
|
tokens << base_name
|
||||||
|
if mapped = NAME_TO_CODE[base_name]?
|
||||||
|
tokens << mapped
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
if lookup.includes?("-")
|
||||||
|
tokens << lookup.split("-", 2)[0]
|
||||||
|
elsif lookup.size.in?(2..3) && lookup.chars.all?(&.ascii_letter?)
|
||||||
|
tokens << lookup
|
||||||
|
end
|
||||||
|
|
||||||
|
tokens.uniq
|
||||||
end
|
end
|
||||||
|
|
||||||
private def self.rank(caption : Metadata, names : Array(String)) : Tuple(Int32, Int32)
|
private def self.match_index(caption : Metadata, name : String) : Int32?
|
||||||
|
needles = match_tokens(name)
|
||||||
|
return nil if needles.empty?
|
||||||
|
|
||||||
|
caption_name = caption.name.strip.downcase
|
||||||
|
lang = caption.language_code.strip.downcase
|
||||||
|
base_lang = lang.split("-", 2)[0]
|
||||||
|
caption_tokens = match_tokens(caption.name)
|
||||||
|
caption_tokens << lang
|
||||||
|
caption_tokens << base_lang
|
||||||
|
|
||||||
|
needles.each_with_index do |needle, index|
|
||||||
|
if caption_name == needle ||
|
||||||
|
caption_name.starts_with?(needle + " (") ||
|
||||||
|
caption_name.starts_with?(needle + " - ") ||
|
||||||
|
lang == needle ||
|
||||||
|
(base_lang == needle && !needle.includes?("-")) ||
|
||||||
|
caption_tokens.includes?(needle)
|
||||||
|
return index
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
nil
|
||||||
|
end
|
||||||
|
|
||||||
|
private def self.rank(caption : Metadata, names : Array(String)) : Tuple(Int32, Int32, Int32)
|
||||||
pref_rank = names.size
|
pref_rank = names.size
|
||||||
|
specificity = Int32::MAX
|
||||||
names.each_with_index do |name, index|
|
names.each_with_index do |name, index|
|
||||||
if name_matches?(caption, name)
|
if found = match_index(caption, name)
|
||||||
pref_rank = index
|
pref_rank = index
|
||||||
|
specificity = found
|
||||||
break
|
break
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
auto_rank = caption.auto_generated ? 1 : 0
|
auto_rank = caption.auto_generated ? 1 : 0
|
||||||
{pref_rank, auto_rank}
|
{pref_rank, specificity, auto_rank}
|
||||||
end
|
end
|
||||||
|
|
||||||
# List of all caption languages available on Youtube.
|
# List of all caption languages available on Youtube.
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user