diff --git a/spec/invidious/videos/captions_matching_spec.cr b/spec/invidious/videos/captions_matching_spec.cr index b8f61a12c..ec36a9754 100644 --- a/spec/invidious/videos/captions_matching_spec.cr +++ b/spec/invidious/videos/captions_matching_spec.cr @@ -27,7 +27,6 @@ Spectator.describe Invidious::Videos::Captions do expect(matched.map(&.name)).to eq(["English (United States)", "English (United Kingdom)"]) end - it "matches English preference to a code-labeled en track" do code_only = Invidious::Videos::Captions::Metadata.new("en", "en", "http://x", true) arabic = Invidious::Videos::Captions::Metadata.new("ar", "ar", "http://x", true) @@ -44,6 +43,14 @@ Spectator.describe Invidious::Videos::Captions do expect(matched.map(&.name)).to eq(["en-US"]) end + it "prefers en-US over en-GB for English (United States)" do + us = Invidious::Videos::Captions::Metadata.new("en-US", "en-US", "http://x", false) + uk = Invidious::Videos::Captions::Metadata.new("en-GB", "en-GB", "http://x", false) + + matched = Invidious::Videos::Captions.matching([uk, us], ["English (United States)"]) + expect(matched.map(&.name)).to eq(["en-US", "en-GB"]) + end + it "ignores blank preference slots" do arabic = Invidious::Videos::Captions::Metadata.new("Arabic (auto-generated)", "ar", "http://x", true) matched = Invidious::Videos::Captions.matching([arabic], ["", "", ""]) diff --git a/src/invidious/videos/caption.cr b/src/invidious/videos/caption.cr index ba67e2de6..7340ffcb7 100644 --- a/src/invidious/videos/caption.cr +++ b/src/invidious/videos/caption.cr @@ -101,173 +101,114 @@ module Invidious::Videos names.any? { |name| name_matches?(caption, name) } end - # Map display-language preferences (e.g. "English") onto ISO-ish codes so - # they can match caption names/codes like "en" or "en-US". + # Common LANGUAGES names mapped to ISO 639-1 / BCP-47 codes. + # Auto-generated suffixes are stripped before lookup; regional + # qualifiers keep their more specific code and fall back to the base. NAME_TO_CODE = { - "afrikaans" => "af", - "albanian" => "sq", - "amharic" => "am", - "arabic" => "ar", - "armenian" => "hy", - "azerbaijani" => "az", - "bangla" => "bn", - "basque" => "eu", - "belarusian" => "be", - "bosnian" => "bs", - "bulgarian" => "bg", - "burmese" => "my", - "cantonese" => "yue", - "catalan" => "ca", - "cebuano" => "ceb", - "chinese" => "zh", - "corsican" => "co", - "croatian" => "hr", - "czech" => "cs", - "danish" => "da", - "dutch" => "nl", - "english" => "en", - "esperanto" => "eo", - "estonian" => "et", - "filipino" => "fil", - "finnish" => "fi", - "french" => "fr", - "galician" => "gl", - "georgian" => "ka", - "german" => "de", - "greek" => "el", - "gujarati" => "gu", - "haitian creole" => "ht", - "hausa" => "ha", - "hawaiian" => "haw", - "hebrew" => "he", - "hindi" => "hi", - "hmong" => "hmn", - "hungarian" => "hu", - "icelandic" => "is", - "igbo" => "ig", - "indonesian" => "id", - "interlingue" => "ie", - "irish" => "ga", - "italian" => "it", - "japanese" => "ja", - "javanese" => "jv", - "kannada" => "kn", - "kazakh" => "kk", - "khmer" => "km", - "korean" => "ko", - "kurdish" => "ku", - "kyrgyz" => "ky", - "lao" => "lo", - "latin" => "la", - "latvian" => "lv", - "lithuanian" => "lt", - "luxembourgish" => "lb", - "macedonian" => "mk", - "malagasy" => "mg", - "malay" => "ms", - "malayalam" => "ml", - "maltese" => "mt", - "maori" => "mi", - "marathi" => "mr", - "mongolian" => "mn", - "nepali" => "ne", - "norwegian bokmål" => "nb", - "norwegian bokmal" => "nb", - "nyanja" => "ny", - "pashto" => "ps", - "persian" => "fa", - "polish" => "pl", - "portuguese" => "pt", - "punjabi" => "pa", - "romanian" => "ro", - "russian" => "ru", - "samoan" => "sm", - "scottish gaelic" => "gd", - "serbian" => "sr", - "shona" => "sn", - "sindhi" => "sd", - "sinhala" => "si", - "slovak" => "sk", - "slovenian" => "sl", - "somali" => "so", - "southern sotho" => "st", - "spanish" => "es", - "sundanese" => "su", - "swahili" => "sw", - "swedish" => "sv", - "tajik" => "tg", - "tamil" => "ta", - "telugu" => "te", - "thai" => "th", - "turkish" => "tr", - "ukrainian" => "uk", - "urdu" => "ur", - "uzbek" => "uz", - "vietnamese" => "vi", - "welsh" => "cy", - "western frisian" => "fy", - "xhosa" => "xh", - "yiddish" => "yi", - "yoruba" => "yo", - "zulu" => "zu", + "arabic" => "ar", + "chinese" => "zh", + "chinese (china)" => "zh-cn", + "chinese (hong kong)" => "zh-hk", + "chinese (simplified)" => "zh-hans", + "chinese (taiwan)" => "zh-tw", + "chinese (traditional)" => "zh-hant", + "dutch" => "nl", + "english" => "en", + "english (united kingdom)" => "en-gb", + "english (united states)" => "en-us", + "filipino" => "fil", + "french" => "fr", + "german" => "de", + "indonesian" => "id", + "italian" => "it", + "japanese" => "ja", + "korean" => "ko", + "portuguese" => "pt", + "portuguese (brazil)" => "pt-br", + "russian" => "ru", + "spanish" => "es", + "spanish (latin america)" => "es-419", + "spanish (mexico)" => "es-mx", + "spanish (spain)" => "es-es", + "turkish" => "tr", + "vietnamese" => "vi", } private def self.name_matches?(caption : Metadata, name : String) : Bool - needle = name.strip.downcase - return false if needle.empty? - - caption_name = caption.name.downcase - lang = caption.language_code.downcase - base_lang = lang.split("-")[0] - - return true if caption_name == needle || - caption_name.starts_with?(needle + " (") || - caption_name.starts_with?(needle + " - ") || - lang == needle || - base_lang == needle - - # Display-language preferences such as "English" must also match - # code-labeled tracks like "en" / "en-US". - needle_codes = canonical_codes(needle) - caption_codes = canonical_codes(caption_name) - caption_codes << lang - caption_codes << base_lang - needle_codes.any? { |code| caption_codes.includes?(code) } + !match_index(caption, name).nil? end - private def self.canonical_codes(value : String) : Array(String) - raw = value.strip.downcase - return [] of String if raw.empty? + # Tokens from most specific to least: "English (United States)" -> + # "english (united states)", "en-us", "english", "en". + private def self.match_tokens(name : String) : Array(String) + token = name.strip.downcase + return [] of String if token.empty? - codes = [] of String - base_name = raw.split(" - ")[0].split(" (")[0].strip - - if mapped = NAME_TO_CODE[base_name]? - codes << mapped + lookup = token + if lookup.ends_with?(" (auto-generated)") + lookup = lookup.rchop(" (auto-generated)") end - if raw.includes?("-") - codes << raw - codes << raw.split("-")[0] - elsif raw.size.in?(2..3) && raw.chars.all?(&.ascii_letter?) - codes << raw + tokens = [lookup] + if mapped = NAME_TO_CODE[lookup]? + tokens << mapped end - codes.uniq! - codes + base_name = lookup.split(" - ")[0].split(" (")[0].strip + if !base_name.empty? && base_name != lookup + tokens << base_name + if mapped = NAME_TO_CODE[base_name]? + tokens << mapped + end + end + + if lookup.includes?("-") + tokens << lookup.split("-", 2)[0] + elsif lookup.size.in?(2..3) && lookup.chars.all?(&.ascii_letter?) + tokens << lookup + end + + tokens.uniq end - private def self.rank(caption : Metadata, names : Array(String)) : Tuple(Int32, Int32) + private def self.match_index(caption : Metadata, name : String) : Int32? + needles = match_tokens(name) + return nil if needles.empty? + + caption_name = caption.name.strip.downcase + lang = caption.language_code.strip.downcase + base_lang = lang.split("-", 2)[0] + caption_tokens = match_tokens(caption.name) + caption_tokens << lang + caption_tokens << base_lang + + needles.each_with_index do |needle, index| + if caption_name == needle || + caption_name.starts_with?(needle + " (") || + caption_name.starts_with?(needle + " - ") || + lang == needle || + (base_lang == needle && !needle.includes?("-")) || + caption_tokens.includes?(needle) + return index + end + end + + nil + end + + private def self.rank(caption : Metadata, names : Array(String)) : Tuple(Int32, Int32, Int32) pref_rank = names.size + specificity = Int32::MAX names.each_with_index do |name, index| - if name_matches?(caption, name) + if found = match_index(caption, name) pref_rank = index + specificity = found break end end auto_rank = caption.auto_generated ? 1 : 0 - {pref_rank, auto_rank} + {pref_rank, specificity, auto_rank} end # List of all caption languages available on Youtube.