Merge pull request #950 from itzzdev09/fix/url-domain-www-prefix

Strip only a whole www. prefix from parsed domains

src/mvt/common/url.py

@@ -349,7 +349,9 @@ class URL:
             return tld_obj
         if tld_obj is None:
             return ""
-        return tld_obj.parsed_url.netloc.lower().lstrip("www.")
+        # removeprefix, not lstrip: lstrip takes a set of characters, so it ate the
+        # leading "w"s and dots of any domain ("web.evil.com" -> "eb.evil.com").
+        return tld_obj.parsed_url.netloc.lower().removeprefix("www.")
 
     def get_top_level(self) -> str:
         """Get only the top-level domain from a URL.

tests/common/test_indicators.py

@@ -311,3 +311,31 @@ class TestIndicators:
         ind = Indicators(log=logging)
         ind.load_indicators_files([], load_default=False)
         assert ind.total_ioc_count == 9
+
+    def test_check_url_matches_w_prefixed_subdomain(self, tmp_path):
+        import json
+
+        stix_file = tmp_path / "w-domain.stix2"
+        stix_file.write_text(
+            json.dumps(
+                {
+                    "objects": [
+                        {
+                            "type": "indicator",
+                            "pattern": "[domain-name:value = 'web.evil.com']",
+                        }
+                    ]
+                }
+            ),
+            encoding="utf-8",
+        )
+        ind = Indicators(log=logging)
+        ind.load_indicators_files([str(stix_file)], load_default=False)
+
+        for url in (
+            "https://web.evil.com/path",
+            "https://www.web.evil.com/path",
+        ):
+            match = ind.check_url(url)
+            assert match is not None
+            assert match.ioc.value == "web.evil.com"

tests/common/test_url.py

@@ -22,3 +22,23 @@ def test_google_maps_url_is_not_shortened(url):
 
 def test_other_google_short_url_is_shortened():
     assert URL("https://goo.gl/example").check_if_shortened() is True
+
+
+@pytest.mark.parametrize(
+    "url, domain",
+    [
+        ("https://www.example.com/path", "example.com"),
+        # Only the whole "www." prefix comes off, not any leading "w" or "." character.
+        ("https://web.example.com", "web.example.com"),
+        ("https://wow.com", "wow.com"),
+        ("https://wired.com", "wired.com"),
+        ("https://www.wow.com", "wow.com"),
+    ],
+)
+def test_get_domain_strips_only_a_whole_www_prefix(url, domain):
+    assert URL(url).domain == domain
+
+
+def test_shortener_starting_with_w_is_detected():
+    assert URL("https://w3t.org/example").check_if_shortened() is True
+    assert URL("https://www.w3t.org/example").check_if_shortened() is True