Merge pull request CloudBotIRC#141 from linuxdaemon/gonzobot+fix-url-regex

Update link announcer regex to be more precise

Author: edwardslabs

plugins/link_announcer.py

@@ -8,7 +8,36 @@
 from cloudbot.hook import Priority, Action
 
 # This will match any URL, blacklist removed and abstracted to a priority/halting system
-url_re = re.compile(r'https?://(?:[a-zA-Z]|[0-9]|[$-_@.&+~]|[!*(),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', re.I)
+url_re = re.compile(
+    r"""
+    https? # Scheme
+    ://
+    
+    # Username and Password
+    (?:
+        (?:[^\[\]?/<~#`!@$%^&*()=+}|:";',>{\s]|%[0-9A-F]{2})*
+        (?::(?:[^\[\]?/<~#`!@$%^&*()=+}|:";',>{\s]|%[0-9A-F]{2})*)?
+        @
+    )?
+    
+    # Domain
+    (?:
+        # TODO Add support for IDNA hostnames as specified by RFC5891
+        [\-.0-9A-Za-z]+|  # host
+        \d{1,3}(?:\.\d{1,3}){3}|  # IPv4
+        \[[A-F0-9]{0,4}(?::[A-F0-9]{0,4}){2,7}\]  # IPv6
+    )
+    
+    (?::\d*)?  # port
+    
+    (?:/(?:[A-Za-z0-9!$&-.:;=@_~\u00A0-\U0010FFFD]|%[A-F0-9]{2})*)*  # Path segment
+    
+    (?:\?(?:[A-Za-z0-9!$&-;=@_~\u00A0-\U0010FFFD]|%[A-F0-9]{2})*)?  # Query
+    
+    (?:\#(?:[A-Za-z0-9!$&-;=@_~\u00A0-\U0010FFFD]|%[A-F0-9]{2})*)?  # Fragment
+    """,
+    re.IGNORECASE | re.VERBOSE
+)
 
 HEADERS = {
     'Accept-Language': 'en-US,en;q=0.5',
@@ -32,6 +61,8 @@ def print_url_title(message, match):
         return
 
     html = BeautifulSoup(content, "lxml", from_encoding=encoding)
-    title = " ".join(html.title.text.strip().splitlines())
-    out = "Title: \x02{}\x02".format(title)
-    message(out)
+
+    if html.title:
+        title = html.title.text
+        out = "Title: \x02{}\x02".format(title)
+        message(out)

tests/plugin_tests/test_link_announcer.py

@@ -0,0 +1,74 @@
+from plugins.link_announcer import url_re
+
+MATCHES = (
+    "http://foo.com/blah_blah",
+    "http://foo.com/blah_blah/",
+    "http://foo.com/blah_blah_(wikipedia)",
+    "http://foo.com/blah_blah_(wikipedia)_(again)",
+    "http://www.example.com/wpstyle/?p=364",
+    "https://www.example.com/foo/?bar=baz&inga=42&quux",
+    "http://userid:[email protected]:8080",
+    "http://userid:[email protected]:8080/",
+    "http://[email protected]",
+    "http://[email protected]/",
+    "http://[email protected]:8080",
+    "http://[email protected]:8080/",
+    "http://userid:[email protected]",
+    "http://userid:[email protected]/",
+    "http://142.42.1.1/",
+    "http://142.42.1.1:8080/",
+    "http://foo.com/blah_(wikipedia)#cite-1",
+    "http://foo.com/blah_(wikipedia)_blah#cite-1",
+    "http://foo.com/unicode_(✪)_in_parens",
+    "http://foo.com/(something)?after=parens",
+    "http://code.google.com/events/#&product=browser",
+    "http://j.mp",
+    "http://foo.bar/?q=Test%20URL-encoded%20stuff",
+    "http://1337.net",
+    "http://a.b-c.de",
+    "http://223.255.255.254",
+)
+
+FAILS = (
+    "http://",
+    "http://?",
+    "http://??",
+    "http://??/",
+    "http://#",
+    "http://##",
+    "http://##/",
+    "http://foo.bar?q=Spaces should be encoded",
+    "//",
+    "//a",
+    "///a",
+    "///",
+    "http:///a",
+    "foo.com",
+    "rdar://1234",
+    "h://test",
+    "http:// shouldfail.com",
+    ":// should fail",
+    "http://foo.bar/foo(bar)baz quux",
+    "ftps://foo.bar/",
+)
+
+SEARCH = (
+    ("[https://example.com]", "https://example.com"),
+    ("<a hreh=\"https://example.com/test.page?#test\">", "https://example.com/test.page?#test"),
+    ("<https://www.example.com/this.is.a.test/blah.txt?a=1#123>", "https://www.example.com/this.is.a.test/blah.txt?a=1#123"),
+)
+
+
+def test_urls():
+    for url in MATCHES:
+        assert url_re.fullmatch(url), url
+
+    for url in FAILS:
+        match = url_re.fullmatch(url)
+        assert not match, match.group()
+
+
+def test_search():
+    for text, out in SEARCH:
+        match = url_re.search(text)
+        assert match and match.group() == out