BrowserSecurity
/
ExtensionCrawler
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Make sure semaphore is released if an exception occurs during http request.

This commit is contained in:
Michael Herzberg 2018-04-22 13:59:15 +01:00
parent 756dcb3ed1
commit 49ea3bb496
1 changed files with 16 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
ExtensionCrawler/request_manager.py
View File

@ -17,9 +17,13 @@ class RequestManager:
with self.lock:
self.sem.acquire()
time.sleep(max(0.0, self.last_restricted_request.value + 0.6 + (random.random() * 0.15) - time.time()))
yield None
self.last_request.value = time.time()
self.sem.release()
try:
yield
except Exception as e:
raise e
finally:
self.last_request.value = time.time()
self.sem.release()
@contextmanager
def restricted_request(self):
@ -27,8 +31,12 @@ class RequestManager:
for i in range(self.max_workers):
self.sem.acquire()
time.sleep(max(0.0, self.last_request.value + 0.6 + (random.random() * 0.15) - time.time()))
yield None
self.last_request.value = time.time()
self.last_restricted_request.value = time.time()
for i in range(self.max_workers):
self.sem.release()
try:
yield
except Exception as e:
raise e
finally:
self.last_request.value = time.time()
self.last_restricted_request.value = time.time()
for i in range(self.max_workers):
self.sem.release()