diff --git a/spec/invidious/comments/content_spec.cr b/spec/invidious/comments/content_spec.cr new file mode 100644 index 00000000..40d47ba7 --- /dev/null +++ b/spec/invidious/comments/content_spec.cr @@ -0,0 +1,118 @@ +require "../../parsers_helper" + +Spectator.describe "strip_url_trailing_punctuation" do + it "strips trailing punctuation from URLs" do + expect(strip_url_trailing_punctuation("https://example.com.")).to eq("https://example.com") + expect(strip_url_trailing_punctuation("https://example.com,")).to eq("https://example.com") + expect(strip_url_trailing_punctuation("https://example.com...")).to eq("https://example.com") + expect(strip_url_trailing_punctuation("https://example.com/page)!?")).to eq("https://example.com/page") + end + + it "keeps URLs without trailing punctuation untouched" do + expect(strip_url_trailing_punctuation("https://example.com")).to eq("https://example.com") + expect(strip_url_trailing_punctuation("https://example.com/a.b?c=d,e#f")).to eq("https://example.com/a.b?c=d,e#f") + end + + it "keeps balanced closing parentheses" do + expect(strip_url_trailing_punctuation("https://en.wikipedia.org/wiki/Crystal_(programming_language)")) + .to eq("https://en.wikipedia.org/wiki/Crystal_(programming_language)") + + # Only the unmatched parenthesis is stripped + expect(strip_url_trailing_punctuation("https://en.wikipedia.org/wiki/Crystal_(programming_language))")) + .to eq("https://en.wikipedia.org/wiki/Crystal_(programming_language)") + end +end + +Spectator.describe "text_to_parsed_content" do + it "parses text without URLs" do + runs = text_to_parsed_content("Hello world").dig("runs").as_a + + expect(runs.size).to eq(1) + expect(runs[0].dig("text").as_s).to eq("Hello world\n") + end + + it "auto-links a bare URL" do + runs = text_to_parsed_content("https://example.com").dig("runs").as_a + + expect(runs.size).to eq(3) + expect(runs[0].dig("text").as_s).to eq("") + expect(runs[1].dig("text").as_s).to eq("https://example.com") + expect(runs[1].dig("navigationEndpoint", "urlEndpoint", "url").as_s).to eq("https://example.com") + expect(runs[2].dig("text").as_s).to eq("\n") + end + + it "excludes sentence punctuation from auto-linked URLs" do + runs = text_to_parsed_content("Check out https://example.com. Thanks!").dig("runs").as_a + + expect(runs.size).to eq(3) + expect(runs[0].dig("text").as_s).to eq("Check out ") + expect(runs[1].dig("navigationEndpoint", "urlEndpoint", "url").as_s).to eq("https://example.com") + expect(runs[2].dig("text").as_s).to eq(". Thanks!\n") + end + + it "excludes the closing parenthesis of enclosing text" do + runs = text_to_parsed_content("(visit my website at https://example.com)").dig("runs").as_a + + expect(runs.size).to eq(3) + expect(runs[0].dig("text").as_s).to eq("(visit my website at ") + expect(runs[1].dig("navigationEndpoint", "urlEndpoint", "url").as_s).to eq("https://example.com") + expect(runs[2].dig("text").as_s).to eq(")\n") + end + + it "keeps URLs containing balanced parentheses intact" do + runs = text_to_parsed_content("(see https://en.wikipedia.org/wiki/Crystal_(programming_language))").dig("runs").as_a + + expect(runs.size).to eq(3) + expect(runs[1].dig("navigationEndpoint", "urlEndpoint", "url").as_s) + .to eq("https://en.wikipedia.org/wiki/Crystal_(programming_language)") + expect(runs[2].dig("text").as_s).to eq(")\n") + end + + it "keeps punctuation inside URLs" do + runs = text_to_parsed_content("Docs: https://example.com/a.b?c=d,e#f! Read").dig("runs").as_a + + expect(runs.size).to eq(3) + expect(runs[1].dig("navigationEndpoint", "urlEndpoint", "url").as_s).to eq("https://example.com/a.b?c=d,e#f") + expect(runs[2].dig("text").as_s).to eq("! Read\n") + end + + it "does not include angle brackets in auto-linked URLs" do + runs = text_to_parsed_content("").dig("runs").as_a + + expect(runs.size).to eq(3) + expect(runs[0].dig("text").as_s).to eq("<") + expect(runs[1].dig("navigationEndpoint", "urlEndpoint", "url").as_s).to eq("https://example.com") + expect(runs[2].dig("text").as_s).to eq(">\n") + end + + it "auto-links multiple URLs on the same line" do + runs = text_to_parsed_content("See https://a.example, then https://b.example.").dig("runs").as_a + + expect(runs.size).to eq(5) + expect(runs[0].dig("text").as_s).to eq("See ") + expect(runs[1].dig("navigationEndpoint", "urlEndpoint", "url").as_s).to eq("https://a.example") + expect(runs[2].dig("text").as_s).to eq(", then ") + expect(runs[3].dig("navigationEndpoint", "urlEndpoint", "url").as_s).to eq("https://b.example") + expect(runs[4].dig("text").as_s).to eq(".\n") + end + + it "keeps the text surrounding a repeated URL" do + runs = text_to_parsed_content("https://example.com and https://example.com.").dig("runs").as_a + + expect(runs.size).to eq(5) + expect(runs[1].dig("navigationEndpoint", "urlEndpoint", "url").as_s).to eq("https://example.com") + expect(runs[2].dig("text").as_s).to eq(" and ") + expect(runs[3].dig("navigationEndpoint", "urlEndpoint", "url").as_s).to eq("https://example.com") + expect(runs[4].dig("text").as_s).to eq(".\n") + end + + it "handles URLs on multiple lines" do + runs = text_to_parsed_content("Line one https://example.com,\nLine two").dig("runs").as_a + + expect(runs.size).to eq(4) + expect(runs[0].dig("text").as_s).to eq("Line one ") + expect(runs[1].dig("navigationEndpoint", "urlEndpoint", "url").as_s).to eq("https://example.com") + expect(runs[2].dig("text").as_s).to eq(",\n") + expect(runs[3].dig("text").as_s).to eq("Line two\n") + end +end diff --git a/src/invidious/comments/content.cr b/src/invidious/comments/content.cr index 1f55bfe6..15f5059f 100644 --- a/src/invidious/comments/content.cr +++ b/src/invidious/comments/content.cr @@ -1,3 +1,33 @@ +# Strip punctuation at the end of an auto-detected URL that most likely +# belongs to the surrounding text rather than to the URL itself (e.g the +# period at the end of a sentence, or the closing parenthesis in +# "(see https://example.com)"). +# +# The rules mimic the autolink extension of GitHub Flavored Markdown +# (https://github.github.com/gfm/#extended-autolink-path-validation), +# with the addition of ';' since entity references can't appear in the +# plain text handled here. +# +# A trailing closing parenthesis is only stripped when it has no matching +# opening parenthesis within the URL, so that links like +# https://en.wikipedia.org/wiki/Crystal_(programming_language) are kept +# intact. +def strip_url_trailing_punctuation(url : String) : String + loop do + case url[-1]? + when '.', ',', ':', ';', '!', '?', '\'', '"', '*', '_', '~' + url = url.rchop + when ')' + break if url.count('(') >= url.count(')') + url = url.rchop + else + break + end + end + + return url +end + def text_to_parsed_content(text : String) : JSON::Any nodes = [] of JSON::Any # For each line convert line to array of nodes @@ -12,14 +42,16 @@ def text_to_parsed_content(text : String) : JSON::Any # For each match with url pattern, get last node and preserve # last node before create new node with url information # { 'text': match, 'navigationEndpoint': { 'urlEndpoint' : 'url': match } } - line.scan(/https?:\/\/[^ ]*/).each do |url_match| + line.scan(/https?:\/\/[^\s<>]+/).each do |url_match| + url = strip_url_trailing_punctuation(url_match[0]) + # Retrieve last node and update node without match last_node = current_nodes[-1].as_h - splitted_last_node = last_node["text"].as_s.split(url_match[0]) + splitted_last_node = last_node["text"].as_s.split(url, 2) last_node["text"] = JSON.parse(splitted_last_node[0].to_json) current_nodes[-1] = JSON.parse(last_node.to_json) # Create new node with match and navigation infos - current_node = {"text" => url_match[0], "navigationEndpoint" => {"urlEndpoint" => {"url" => url_match[0]}}} + current_node = {"text" => url, "navigationEndpoint" => {"urlEndpoint" => {"url" => url}}} current_nodes << (JSON.parse(current_node.to_json)) # If text remain after match create new simple node with text after match after_node = {"text" => splitted_last_node.size > 1 ? splitted_last_node[1] : ""}