From 49ea3bb49646b94d5303338ada6d943e4fe29d07 Mon Sep 17 00:00:00 2001 From: Michael Herzberg Date: Sun, 22 Apr 2018 13:59:15 +0100 Subject: [PATCH] Make sure semaphore is released if an exception occurs during http request. --- ExtensionCrawler/request_manager.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/ExtensionCrawler/request_manager.py b/ExtensionCrawler/request_manager.py index c3b493b..d12672b 100644 --- a/ExtensionCrawler/request_manager.py +++ b/ExtensionCrawler/request_manager.py @@ -17,9 +17,13 @@ class RequestManager: with self.lock: self.sem.acquire() time.sleep(max(0.0, self.last_restricted_request.value + 0.6 + (random.random() * 0.15) - time.time())) - yield None - self.last_request.value = time.time() - self.sem.release() + try: + yield + except Exception as e: + raise e + finally: + self.last_request.value = time.time() + self.sem.release() @contextmanager def restricted_request(self): @@ -27,8 +31,12 @@ class RequestManager: for i in range(self.max_workers): self.sem.acquire() time.sleep(max(0.0, self.last_request.value + 0.6 + (random.random() * 0.15) - time.time())) - yield None - self.last_request.value = time.time() - self.last_restricted_request.value = time.time() - for i in range(self.max_workers): - self.sem.release() + try: + yield + except Exception as e: + raise e + finally: + self.last_request.value = time.time() + self.last_restricted_request.value = time.time() + for i in range(self.max_workers): + self.sem.release()