Merge be4f5875a27d5f55484f3bffae290f2b46e5e410 into 6865cf208e575ded6723f1269ba5172c74445f55

spec/invidious/comments/content_spec.cr

@@ -0,0 +1,118 @@
+require "../../parsers_helper"
+
+Spectator.describe "strip_url_trailing_punctuation" do
+  it "strips trailing punctuation from URLs" do
+    expect(strip_url_trailing_punctuation("https://example.com.")).to eq("https://example.com")
+    expect(strip_url_trailing_punctuation("https://example.com,")).to eq("https://example.com")
+    expect(strip_url_trailing_punctuation("https://example.com...")).to eq("https://example.com")
+    expect(strip_url_trailing_punctuation("https://example.com/page)!?")).to eq("https://example.com/page")
+  end
+
+  it "keeps URLs without trailing punctuation untouched" do
+    expect(strip_url_trailing_punctuation("https://example.com")).to eq("https://example.com")
+    expect(strip_url_trailing_punctuation("https://example.com/a.b?c=d,e#f")).to eq("https://example.com/a.b?c=d,e#f")
+  end
+
+  it "keeps balanced closing parentheses" do
+    expect(strip_url_trailing_punctuation("https://en.wikipedia.org/wiki/Crystal_(programming_language)"))
+      .to eq("https://en.wikipedia.org/wiki/Crystal_(programming_language)")
+
+    # Only the unmatched parenthesis is stripped
+    expect(strip_url_trailing_punctuation("https://en.wikipedia.org/wiki/Crystal_(programming_language))"))
+      .to eq("https://en.wikipedia.org/wiki/Crystal_(programming_language)")
+  end
+end
+
+Spectator.describe "text_to_parsed_content" do
+  it "parses text without URLs" do
+    runs = text_to_parsed_content("Hello world").dig("runs").as_a
+
+    expect(runs.size).to eq(1)
+    expect(runs[0].dig("text").as_s).to eq("Hello world\n")
+  end
+
+  it "auto-links a bare URL" do
+    runs = text_to_parsed_content("https://example.com").dig("runs").as_a
+
+    expect(runs.size).to eq(3)
+    expect(runs[0].dig("text").as_s).to eq("")
+    expect(runs[1].dig("text").as_s).to eq("https://example.com")
+    expect(runs[1].dig("navigationEndpoint", "urlEndpoint", "url").as_s).to eq("https://example.com")
+    expect(runs[2].dig("text").as_s).to eq("\n")
+  end
+
+  it "excludes sentence punctuation from auto-linked URLs" do
+    runs = text_to_parsed_content("Check out https://example.com. Thanks!").dig("runs").as_a
+
+    expect(runs.size).to eq(3)
+    expect(runs[0].dig("text").as_s).to eq("Check out ")
+    expect(runs[1].dig("navigationEndpoint", "urlEndpoint", "url").as_s).to eq("https://example.com")
+    expect(runs[2].dig("text").as_s).to eq(". Thanks!\n")
+  end
+
+  it "excludes the closing parenthesis of enclosing text" do
+    runs = text_to_parsed_content("(visit my website at https://example.com)").dig("runs").as_a
+
+    expect(runs.size).to eq(3)
+    expect(runs[0].dig("text").as_s).to eq("(visit my website at ")
+    expect(runs[1].dig("navigationEndpoint", "urlEndpoint", "url").as_s).to eq("https://example.com")
+    expect(runs[2].dig("text").as_s).to eq(")\n")
+  end
+
+  it "keeps URLs containing balanced parentheses intact" do
+    runs = text_to_parsed_content("(see https://en.wikipedia.org/wiki/Crystal_(programming_language))").dig("runs").as_a
+
+    expect(runs.size).to eq(3)
+    expect(runs[1].dig("navigationEndpoint", "urlEndpoint", "url").as_s)
+      .to eq("https://en.wikipedia.org/wiki/Crystal_(programming_language)")
+    expect(runs[2].dig("text").as_s).to eq(")\n")
+  end
+
+  it "keeps punctuation inside URLs" do
+    runs = text_to_parsed_content("Docs: https://example.com/a.b?c=d,e#f! Read").dig("runs").as_a
+
+    expect(runs.size).to eq(3)
+    expect(runs[1].dig("navigationEndpoint", "urlEndpoint", "url").as_s).to eq("https://example.com/a.b?c=d,e#f")
+    expect(runs[2].dig("text").as_s).to eq("! Read\n")
+  end
+
+  it "does not include angle brackets in auto-linked URLs" do
+    runs = text_to_parsed_content("<https://example.com>").dig("runs").as_a
+
+    expect(runs.size).to eq(3)
+    expect(runs[0].dig("text").as_s).to eq("<")
+    expect(runs[1].dig("navigationEndpoint", "urlEndpoint", "url").as_s).to eq("https://example.com")
+    expect(runs[2].dig("text").as_s).to eq(">\n")
+  end
+
+  it "auto-links multiple URLs on the same line" do
+    runs = text_to_parsed_content("See https://a.example, then https://b.example.").dig("runs").as_a
+
+    expect(runs.size).to eq(5)
+    expect(runs[0].dig("text").as_s).to eq("See ")
+    expect(runs[1].dig("navigationEndpoint", "urlEndpoint", "url").as_s).to eq("https://a.example")
+    expect(runs[2].dig("text").as_s).to eq(", then ")
+    expect(runs[3].dig("navigationEndpoint", "urlEndpoint", "url").as_s).to eq("https://b.example")
+    expect(runs[4].dig("text").as_s).to eq(".\n")
+  end
+
+  it "keeps the text surrounding a repeated URL" do
+    runs = text_to_parsed_content("https://example.com and https://example.com.").dig("runs").as_a
+
+    expect(runs.size).to eq(5)
+    expect(runs[1].dig("navigationEndpoint", "urlEndpoint", "url").as_s).to eq("https://example.com")
+    expect(runs[2].dig("text").as_s).to eq(" and ")
+    expect(runs[3].dig("navigationEndpoint", "urlEndpoint", "url").as_s).to eq("https://example.com")
+    expect(runs[4].dig("text").as_s).to eq(".\n")
+  end
+
+  it "handles URLs on multiple lines" do
+    runs = text_to_parsed_content("Line one https://example.com,\nLine two").dig("runs").as_a
+
+    expect(runs.size).to eq(4)
+    expect(runs[0].dig("text").as_s).to eq("Line one ")
+    expect(runs[1].dig("navigationEndpoint", "urlEndpoint", "url").as_s).to eq("https://example.com")
+    expect(runs[2].dig("text").as_s).to eq(",\n")
+    expect(runs[3].dig("text").as_s).to eq("Line two\n")
+  end
+end

src/invidious/comments/content.cr

@@ -1,3 +1,33 @@
+# Strip punctuation at the end of an auto-detected URL that most likely
+# belongs to the surrounding text rather than to the URL itself (e.g the
+# period at the end of a sentence, or the closing parenthesis in
+# "(see https://example.com)").
+#
+# The rules mimic the autolink extension of GitHub Flavored Markdown
+# (https://github.github.com/gfm/#extended-autolink-path-validation),
+# with the addition of ';' since entity references can't appear in the
+# plain text handled here.
+#
+# A trailing closing parenthesis is only stripped when it has no matching
+# opening parenthesis within the URL, so that links like
+# https://en.wikipedia.org/wiki/Crystal_(programming_language) are kept
+# intact.
+def strip_url_trailing_punctuation(url : String) : String
+  loop do
+    case url[-1]?
+    when '.', ',', ':', ';', '!', '?', '\'', '"', '*', '_', '~'
+      url = url.rchop
+    when ')'
+      break if url.count('(') >= url.count(')')
+      url = url.rchop
+    else
+      break
+    end
+  end
+
+  return url
+end
+
 def text_to_parsed_content(text : String) : JSON::Any
   nodes = [] of JSON::Any
   # For each line convert line to array of nodes
@@ -12,14 +42,16 @@ def text_to_parsed_content(text : String) : JSON::Any
     # For each match with url pattern, get last node and preserve
     # last node before create new node with url information
     # { 'text': match, 'navigationEndpoint': { 'urlEndpoint' : 'url': match } }
-    line.scan(/https?:\/\/[^ ]*/).each do |url_match|
+    line.scan(/https?:\/\/[^\s<>]+/).each do |url_match|
+      url = strip_url_trailing_punctuation(url_match[0])
+
       # Retrieve last node and update node without match
       last_node = current_nodes[-1].as_h
-      splitted_last_node = last_node["text"].as_s.split(url_match[0])
+      splitted_last_node = last_node["text"].as_s.split(url, 2)
       last_node["text"] = JSON.parse(splitted_last_node[0].to_json)
       current_nodes[-1] = JSON.parse(last_node.to_json)
       # Create new node with match and navigation infos
-      current_node = {"text" => url_match[0], "navigationEndpoint" => {"urlEndpoint" => {"url" => url_match[0]}}}
+      current_node = {"text" => url, "navigationEndpoint" => {"urlEndpoint" => {"url" => url}}}
       current_nodes << (JSON.parse(current_node.to_json))
       # If text remain after match create new simple node with text after match
       after_node = {"text" => splitted_last_node.size > 1 ? splitted_last_node[1] : ""}