More aggressive download heuristics.
This commit is contained in:
parent
7c688afee8
commit
0f1c53a011
|
@ -346,7 +346,7 @@ def update_reviews(tar, date, ext_id):
|
|||
try:
|
||||
pages = []
|
||||
|
||||
google_dos_protection()
|
||||
# google_dos_protection()
|
||||
res = requests.post(
|
||||
const_review_url(),
|
||||
data=const_review_payload(ext_id, "0", "100"),
|
||||
|
|
|
@ -23,10 +23,10 @@ import traceback
|
|||
import logging
|
||||
|
||||
|
||||
def google_dos_protection(maxrange=0.3):
|
||||
"""Wait a random number of seconds (between 0.5 to 0.5+maxrange)
|
||||
def google_dos_protection(maxrange=0.15):
|
||||
"""Wait a random number of seconds (between 0.45 to 0.5+maxrange)
|
||||
to avoid Google's bot detection"""
|
||||
sleep(0.5 + (random() * maxrange))
|
||||
sleep(0.45 + (random() * maxrange))
|
||||
|
||||
|
||||
def value_of(value, default):
|
||||
|
|
Loading…
Reference in New Issue