BrowserSecurity
/
ExtensionCrawler
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

More aggressive download heuristics.

This commit is contained in:
Achim D. Brucker 2018-04-03 22:20:09 +01:00
parent 7c688afee8
commit 0f1c53a011
2 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ExtensionCrawler/archive.py
View File

@ -346,7 +346,7 @@ def update_reviews(tar, date, ext_id):
try: try:
pages = [] pages = []
google_dos_protection() # google_dos_protection()
res = requests.post( res = requests.post(
const_review_url(), const_review_url(),
data=const_review_payload(ext_id, "0", "100"), data=const_review_payload(ext_id, "0", "100"),

6
ExtensionCrawler/util.py
View File

@ -23,10 +23,10 @@ import traceback
import logging import logging
def google_dos_protection(maxrange=0.3): def google_dos_protection(maxrange=0.15):
"""Wait a random number of seconds (between 0.5 to 0.5+maxrange) """Wait a random number of seconds (between 0.45 to 0.5+maxrange)
to avoid Google's bot detection""" to avoid Google's bot detection"""
sleep(0.5 + (random() * maxrange)) sleep(0.45 + (random() * maxrange))
def value_of(value, default): def value_of(value, default):