mirror of
https://github.com/iv-org/invidious.git
synced 2026-09-06 00:52:45 -05:00
fix(captions): prefer regional language codes over base fallback
Keep region-specific LANGUAGES names such as English (United States) mapped to en-us, then fall back to en. Rank the regional code ahead of en-GB and other base-language matches.
This commit is contained in:
XZH
parent
3df2266294
commit
c4e2bc34c5
@ -27,7 +27,6 @@ Spectator.describe Invidious::Videos::Captions do
|
||||
expect(matched.map(&.name)).to eq(["English (United States)", "English (United Kingdom)"])
|
||||
end
|
||||
|
||||
|
||||
it "matches English preference to a code-labeled en track" do
|
||||
code_only = Invidious::Videos::Captions::Metadata.new("en", "en", "http://x", true)
|
||||
arabic = Invidious::Videos::Captions::Metadata.new("ar", "ar", "http://x", true)
|
||||
@ -44,6 +43,14 @@ Spectator.describe Invidious::Videos::Captions do
|
||||
expect(matched.map(&.name)).to eq(["en-US"])
|
||||
end
|
||||
|
||||
it "prefers en-US over en-GB for English (United States)" do
|
||||
us = Invidious::Videos::Captions::Metadata.new("en-US", "en-US", "http://x", false)
|
||||
uk = Invidious::Videos::Captions::Metadata.new("en-GB", "en-GB", "http://x", false)
|
||||
|
||||
matched = Invidious::Videos::Captions.matching([uk, us], ["English (United States)"])
|
||||
expect(matched.map(&.name)).to eq(["en-US", "en-GB"])
|
||||
end
|
||||
|
||||
it "ignores blank preference slots" do
|
||||
arabic = Invidious::Videos::Captions::Metadata.new("Arabic (auto-generated)", "ar", "http://x", true)
|
||||
matched = Invidious::Videos::Captions.matching([arabic], ["", "", ""])
|
||||
|
||||
@ -101,173 +101,114 @@ module Invidious::Videos
|
||||
names.any? { |name| name_matches?(caption, name) }
|
||||
end
|
||||
|
||||
# Map display-language preferences (e.g. "English") onto ISO-ish codes so
|
||||
# they can match caption names/codes like "en" or "en-US".
|
||||
# Common LANGUAGES names mapped to ISO 639-1 / BCP-47 codes.
|
||||
# Auto-generated suffixes are stripped before lookup; regional
|
||||
# qualifiers keep their more specific code and fall back to the base.
|
||||
NAME_TO_CODE = {
|
||||
"afrikaans" => "af",
|
||||
"albanian" => "sq",
|
||||
"amharic" => "am",
|
||||
"arabic" => "ar",
|
||||
"armenian" => "hy",
|
||||
"azerbaijani" => "az",
|
||||
"bangla" => "bn",
|
||||
"basque" => "eu",
|
||||
"belarusian" => "be",
|
||||
"bosnian" => "bs",
|
||||
"bulgarian" => "bg",
|
||||
"burmese" => "my",
|
||||
"cantonese" => "yue",
|
||||
"catalan" => "ca",
|
||||
"cebuano" => "ceb",
|
||||
"chinese" => "zh",
|
||||
"corsican" => "co",
|
||||
"croatian" => "hr",
|
||||
"czech" => "cs",
|
||||
"danish" => "da",
|
||||
"dutch" => "nl",
|
||||
"english" => "en",
|
||||
"esperanto" => "eo",
|
||||
"estonian" => "et",
|
||||
"filipino" => "fil",
|
||||
"finnish" => "fi",
|
||||
"french" => "fr",
|
||||
"galician" => "gl",
|
||||
"georgian" => "ka",
|
||||
"german" => "de",
|
||||
"greek" => "el",
|
||||
"gujarati" => "gu",
|
||||
"haitian creole" => "ht",
|
||||
"hausa" => "ha",
|
||||
"hawaiian" => "haw",
|
||||
"hebrew" => "he",
|
||||
"hindi" => "hi",
|
||||
"hmong" => "hmn",
|
||||
"hungarian" => "hu",
|
||||
"icelandic" => "is",
|
||||
"igbo" => "ig",
|
||||
"indonesian" => "id",
|
||||
"interlingue" => "ie",
|
||||
"irish" => "ga",
|
||||
"italian" => "it",
|
||||
"japanese" => "ja",
|
||||
"javanese" => "jv",
|
||||
"kannada" => "kn",
|
||||
"kazakh" => "kk",
|
||||
"khmer" => "km",
|
||||
"korean" => "ko",
|
||||
"kurdish" => "ku",
|
||||
"kyrgyz" => "ky",
|
||||
"lao" => "lo",
|
||||
"latin" => "la",
|
||||
"latvian" => "lv",
|
||||
"lithuanian" => "lt",
|
||||
"luxembourgish" => "lb",
|
||||
"macedonian" => "mk",
|
||||
"malagasy" => "mg",
|
||||
"malay" => "ms",
|
||||
"malayalam" => "ml",
|
||||
"maltese" => "mt",
|
||||
"maori" => "mi",
|
||||
"marathi" => "mr",
|
||||
"mongolian" => "mn",
|
||||
"nepali" => "ne",
|
||||
"norwegian bokmål" => "nb",
|
||||
"norwegian bokmal" => "nb",
|
||||
"nyanja" => "ny",
|
||||
"pashto" => "ps",
|
||||
"persian" => "fa",
|
||||
"polish" => "pl",
|
||||
"portuguese" => "pt",
|
||||
"punjabi" => "pa",
|
||||
"romanian" => "ro",
|
||||
"russian" => "ru",
|
||||
"samoan" => "sm",
|
||||
"scottish gaelic" => "gd",
|
||||
"serbian" => "sr",
|
||||
"shona" => "sn",
|
||||
"sindhi" => "sd",
|
||||
"sinhala" => "si",
|
||||
"slovak" => "sk",
|
||||
"slovenian" => "sl",
|
||||
"somali" => "so",
|
||||
"southern sotho" => "st",
|
||||
"spanish" => "es",
|
||||
"sundanese" => "su",
|
||||
"swahili" => "sw",
|
||||
"swedish" => "sv",
|
||||
"tajik" => "tg",
|
||||
"tamil" => "ta",
|
||||
"telugu" => "te",
|
||||
"thai" => "th",
|
||||
"turkish" => "tr",
|
||||
"ukrainian" => "uk",
|
||||
"urdu" => "ur",
|
||||
"uzbek" => "uz",
|
||||
"vietnamese" => "vi",
|
||||
"welsh" => "cy",
|
||||
"western frisian" => "fy",
|
||||
"xhosa" => "xh",
|
||||
"yiddish" => "yi",
|
||||
"yoruba" => "yo",
|
||||
"zulu" => "zu",
|
||||
"arabic" => "ar",
|
||||
"chinese" => "zh",
|
||||
"chinese (china)" => "zh-cn",
|
||||
"chinese (hong kong)" => "zh-hk",
|
||||
"chinese (simplified)" => "zh-hans",
|
||||
"chinese (taiwan)" => "zh-tw",
|
||||
"chinese (traditional)" => "zh-hant",
|
||||
"dutch" => "nl",
|
||||
"english" => "en",
|
||||
"english (united kingdom)" => "en-gb",
|
||||
"english (united states)" => "en-us",
|
||||
"filipino" => "fil",
|
||||
"french" => "fr",
|
||||
"german" => "de",
|
||||
"indonesian" => "id",
|
||||
"italian" => "it",
|
||||
"japanese" => "ja",
|
||||
"korean" => "ko",
|
||||
"portuguese" => "pt",
|
||||
"portuguese (brazil)" => "pt-br",
|
||||
"russian" => "ru",
|
||||
"spanish" => "es",
|
||||
"spanish (latin america)" => "es-419",
|
||||
"spanish (mexico)" => "es-mx",
|
||||
"spanish (spain)" => "es-es",
|
||||
"turkish" => "tr",
|
||||
"vietnamese" => "vi",
|
||||
}
|
||||
|
||||
private def self.name_matches?(caption : Metadata, name : String) : Bool
|
||||
needle = name.strip.downcase
|
||||
return false if needle.empty?
|
||||
|
||||
caption_name = caption.name.downcase
|
||||
lang = caption.language_code.downcase
|
||||
base_lang = lang.split("-")[0]
|
||||
|
||||
return true if caption_name == needle ||
|
||||
caption_name.starts_with?(needle + " (") ||
|
||||
caption_name.starts_with?(needle + " - ") ||
|
||||
lang == needle ||
|
||||
base_lang == needle
|
||||
|
||||
# Display-language preferences such as "English" must also match
|
||||
# code-labeled tracks like "en" / "en-US".
|
||||
needle_codes = canonical_codes(needle)
|
||||
caption_codes = canonical_codes(caption_name)
|
||||
caption_codes << lang
|
||||
caption_codes << base_lang
|
||||
needle_codes.any? { |code| caption_codes.includes?(code) }
|
||||
!match_index(caption, name).nil?
|
||||
end
|
||||
|
||||
private def self.canonical_codes(value : String) : Array(String)
|
||||
raw = value.strip.downcase
|
||||
return [] of String if raw.empty?
|
||||
# Tokens from most specific to least: "English (United States)" ->
|
||||
# "english (united states)", "en-us", "english", "en".
|
||||
private def self.match_tokens(name : String) : Array(String)
|
||||
token = name.strip.downcase
|
||||
return [] of String if token.empty?
|
||||
|
||||
codes = [] of String
|
||||
base_name = raw.split(" - ")[0].split(" (")[0].strip
|
||||
|
||||
if mapped = NAME_TO_CODE[base_name]?
|
||||
codes << mapped
|
||||
lookup = token
|
||||
if lookup.ends_with?(" (auto-generated)")
|
||||
lookup = lookup.rchop(" (auto-generated)")
|
||||
end
|
||||
|
||||
if raw.includes?("-")
|
||||
codes << raw
|
||||
codes << raw.split("-")[0]
|
||||
elsif raw.size.in?(2..3) && raw.chars.all?(&.ascii_letter?)
|
||||
codes << raw
|
||||
tokens = [lookup]
|
||||
if mapped = NAME_TO_CODE[lookup]?
|
||||
tokens << mapped
|
||||
end
|
||||
|
||||
codes.uniq!
|
||||
codes
|
||||
base_name = lookup.split(" - ")[0].split(" (")[0].strip
|
||||
if !base_name.empty? && base_name != lookup
|
||||
tokens << base_name
|
||||
if mapped = NAME_TO_CODE[base_name]?
|
||||
tokens << mapped
|
||||
end
|
||||
end
|
||||
|
||||
if lookup.includes?("-")
|
||||
tokens << lookup.split("-", 2)[0]
|
||||
elsif lookup.size.in?(2..3) && lookup.chars.all?(&.ascii_letter?)
|
||||
tokens << lookup
|
||||
end
|
||||
|
||||
tokens.uniq
|
||||
end
|
||||
|
||||
private def self.rank(caption : Metadata, names : Array(String)) : Tuple(Int32, Int32)
|
||||
private def self.match_index(caption : Metadata, name : String) : Int32?
|
||||
needles = match_tokens(name)
|
||||
return nil if needles.empty?
|
||||
|
||||
caption_name = caption.name.strip.downcase
|
||||
lang = caption.language_code.strip.downcase
|
||||
base_lang = lang.split("-", 2)[0]
|
||||
caption_tokens = match_tokens(caption.name)
|
||||
caption_tokens << lang
|
||||
caption_tokens << base_lang
|
||||
|
||||
needles.each_with_index do |needle, index|
|
||||
if caption_name == needle ||
|
||||
caption_name.starts_with?(needle + " (") ||
|
||||
caption_name.starts_with?(needle + " - ") ||
|
||||
lang == needle ||
|
||||
(base_lang == needle && !needle.includes?("-")) ||
|
||||
caption_tokens.includes?(needle)
|
||||
return index
|
||||
end
|
||||
end
|
||||
|
||||
nil
|
||||
end
|
||||
|
||||
private def self.rank(caption : Metadata, names : Array(String)) : Tuple(Int32, Int32, Int32)
|
||||
pref_rank = names.size
|
||||
specificity = Int32::MAX
|
||||
names.each_with_index do |name, index|
|
||||
if name_matches?(caption, name)
|
||||
if found = match_index(caption, name)
|
||||
pref_rank = index
|
||||
specificity = found
|
||||
break
|
||||
end
|
||||
end
|
||||
|
||||
auto_rank = caption.auto_generated ? 1 : 0
|
||||
{pref_rank, auto_rank}
|
||||
{pref_rank, specificity, auto_rank}
|
||||
end
|
||||
|
||||
# List of all caption languages available on Youtube.
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user