BrowserSecurity
/
ExtensionCrawler
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Process extensions in chunks.

This commit is contained in:
Achim D. Brucker 2018-04-06 21:34:09 +01:00
parent d5df43c5c3
commit 14a30a570d
1 changed files with 45 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
ExtensionCrawler/archive.py
View File

@ -539,24 +539,30 @@ def update_extension(archivedir, forums, ext_id):
res_reviews, res_support, sql_exception, sql_success) res_reviews, res_support, sql_exception, sql_success)
def chunks(l, n):
for i in range(0, len(l), n):
yield l[i:i+n]
def execute_parallel(archivedir, max_retry, timeout, max_workers, ext_ids): def execute_parallel(archivedir, max_retry, timeout, max_workers, ext_ids, chunksize=30000):
results=[] results=[]
for chunk in chunks(ext_ids, chunksize):
for n in range(max_retry): for n in range(max_retry):
if n > 0: if n > 0:
log_info("Attempt ({} out of {}): {} extensions excluding forums (parallel)".format( log_info("Attempt ({} out of {}): {} extensions excluding forums (parallel)".format(
n,max_retry,len(ext_timeouts)), 1) n,max_retry,len(ext_timeouts)), 1)
ext_ids=ext_timeouts chunk=ext_timeouts
ext_timeouts=[] ext_timeouts=[]
with ProcessPool(max_workers=1, max_tasks=100) as pool: with ProcessPool(max_workers=1, max_tasks=100) as pool:
future = pool.map(partial(update_extension, archivedir, False) future = pool.map(partial(update_extension, archivedir, False)
,ext_ids ,chunk
,timeout=timeout) ,timeout=timeout)
iterator = future.result() iterator = future.result()
ext_timeouts=[] ext_timeouts=[]
for ext_id in ext_ids: for ext_id in chunk:
try: try:
results.append(next(iterator)) results.append(next(iterator))
except StopIteration: except StopIteration:
@ -568,7 +574,7 @@ def execute_parallel(archivedir, max_retry, timeout, max_workers, ext_ids):
None, None, None, None, None, None,
None, None, False)) None, None, False))
except ProcessExpired as error: except ProcessExpired as error:
log_info("WorkerException: %s (%s). Exit code: %d" % (error, ext_id, error.exitcode)) log_info("WorkerException: %s (%s)self. Exit code: %d" % (error, ext_id, error.exitcode))
ext_timeouts.append(ext_id) ext_timeouts.append(ext_id)
results.append(UpdateResult(ext_id, False, error, results.append(UpdateResult(ext_id, False, error,
None, None, None, None, None, None,