added link scraper

mjhea0 · mjhea0 · commit 776ed9c75c98 · 2014-05-18T07:29:23.000-07:00
diff --git a/08_basic_email_web_crawler.py b/08_basic_email_web_crawler.py
@@ -6,6 +6,7 @@
 email_re = re.compile(r'([\w\.,]+@[\w\.,]+\.\w+)')
 link_re = re.compile(r'href="(.*?)"')
 
+
 def crawl(url, maxlevel):
 
     result = set()
@@ -25,21 +26,21 @@ def crawl(url, maxlevel):
             # Get an absolute URL for a link
             link = urlparse.urljoin(url, link)
 
-        # Find all emails on current page
-        result.update(email_re.findall(req.text))
+            # Find all emails on current page
+            result.update(email_re.findall(req.text))
 
-        print "Crawled level: {}".format(maxlevel)
+            print "Crawled level: {}".format(maxlevel)
 
-        # new level
-        maxlevel -= 1
+            # new level
+            maxlevel -= 1
 
-        # recurse 
-        crawl(link, maxlevel)
+            # recurse
+            crawl(link, maxlevel)
 
     return result
 
 emails = crawl('http://www.website_goes_here_dot_com', 2)
 
 print "\nScrapped e-mail addresses:"
 for email in emails:
-    print email
+    print email
diff --git a/09_basic_link_web_crawler.py b/09_basic_link_web_crawler.py
@@ -0,0 +1,44 @@
+import requests
+import re
+import urlparse
+
+# regex
+link_re = re.compile(r'href="(.*?)"')
+
+
+def crawl(url, maxlevel):
+
+    result = set()
+
+    while maxlevel > 0:
+
+        # Get the webpage
+        req = requests.get(url)
+
+        # Check if successful
+        if(req.status_code != 200):
+            return []
+
+        # Find and follow all the links
+        links = link_re.findall(req.text)
+        for link in links:
+            # Get an absolute URL for a link
+            link = urlparse.urljoin(url, link)
+            # add links to result set
+            result.update(link)
+
+            print "Crawled level: {}".format(maxlevel)
+
+            # new level
+            maxlevel -= 1
+
+            # recurse
+            crawl(link, maxlevel)
+
+    return result
+
+emails = crawl('http://www.website_goes_here_dot_com', 2)
+
+print "\nScrapped links:"
+for link in links:
+    print link
