From a6af890c1069bf147c8573e00ad41a42e46f865d Mon Sep 17 00:00:00 2001 From: "Achim D. Brucker" Date: Tue, 10 Jan 2017 16:29:32 +0000 Subject: [PATCH] Removed ua header. --- crawler.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/crawler.py b/crawler.py index 19e8bf1..bdb9626 100755 --- a/crawler.py +++ b/crawler.py @@ -73,8 +73,6 @@ class ExtensionCrawler: store_url = 'https://chrome.google.com/webstore' review_url = 'https://chrome.google.com/reviews/components' support_url = 'https://chrome.google.com/reviews/components' -# ua_header = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'} - ua_header = {} @@ -169,13 +167,13 @@ class ExtensionCrawler: '"internedValues":[]}}') response = requests.post( - self.support_url, data=payload.format(extid, "0", "100"),headers=self.ua_header) + self.support_url, data=payload.format(extid, "0", "100")) with open(os.path.join(extdir, 'support000-099.text'), 'w') as f: f.write(response.text) self.store_request_metadata(os.path.join(extdir, 'support000-099.text'),response) self.google_dos_protection(os.path.join(extdir, 'support000-099.text'),response) response = requests.post( - self.support_url, data=payload.format(extid, "100", "100"),headers=self.ua_header) + self.support_url, data=payload.format(extid, "100", "100")) with open(os.path.join(extdir, 'support100-199.text'), 'w') as f: f.write(str(response.text)) self.store_request_metadata(os.path.join(extdir, 'support100-199.text'),response) @@ -191,13 +189,13 @@ class ExtensionCrawler: '"internedKeys":[],' + '"internedValues":[]}}') response = requests.post( - self.review_url, data=payload.format(extid, "0", "100"),headers=self.ua_header) + self.review_url, data=payload.format(extid, "0", "100")) with open(os.path.join(extdir, 'reviews000-099.text'), 'w') as f: f.write(response.text) self.store_request_metadata(os.path.join(extdir, 'reviews000-099.text'),response) self.google_dos_protection(os.path.join(extdir, 'reviews000-099.text'),response) response = requests.post( - self.review_url, data=payload.format(extid, "100", "100"),headers=self.ua_header) + self.review_url, data=payload.format(extid, "100", "100")) with open(os.path.join(extdir, 'reviews100-199.text'), 'w') as f: f.write(response.text) self.store_request_metadata(os.path.join(extdir, 'reviews100-199.text'),response)