add url boost for spectrum

Edouard360 · Edouard360 · commit 9a0d318fcc9b · 2021-10-20T18:46:40.000-07:00
diff --git a/readability/readability.py b/readability/readability.py
@@ -401,7 +401,7 @@ def score_paragraphs(self):
 
     def class_weight(self, e):
         weight = 0
-        for feature in [e.get("class", None), e.get("id", None)]:
+        for feature in [e.get("class", None), e.get("id", None), e.get("data-post-url")]:
             if feature:
                 if REGEXES["negativeRe"].search(feature):
                     weight -= 25
@@ -415,6 +415,9 @@ def class_weight(self, e):
                 if self.negative_keywords and self.negative_keywords.search(feature):
                     weight -= 25
 
+                if self.url in feature:
+                    weight += 500
+
         if self.positive_keywords and self.positive_keywords.match("tag-" + e.tag):
             weight += 25
 
diff --git a/tests/samples/spectrum.html b/tests/samples/spectrum.html
diff --git a/tests/test_article_only.py b/tests/test_article_only.py
@@ -124,3 +124,9 @@ def test_utf8_kanji(self):
         sample = load_sample("utf-8-kanji.sample.html")
         doc = Document(sample)
         res = doc.summary()
+
+    def test_spectrum_ieee(self):
+        sample = load_sample("spectrum.html")
+        doc = Document(sample, url="https://spectrum.ieee.org/event-based-camera-chips")
+        first_sentence = "The pixels in event-based cameras"
+        self.assertIn(first_sentence, doc.summary())
