BrowserSecurity
/
ExtensionCrawler
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Pebble: start processing results right away.

This commit is contained in:
Michael Herzberg 2018-04-10 16:15:33 +01:00
parent 6bee81b711
commit bbe575d07b
1 changed files with 17 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
ExtensionCrawler/archive.py
View File

@ -581,24 +581,23 @@ def execute_parallel_ProcessPool(archivedir, max_retry, timeout, max_workers, ex
future = pool.map(partial(update_extension, archivedir, forums) future = pool.map(partial(update_extension, archivedir, forums)
,ext_ids ,ext_ids
,timeout=timeout) ,timeout=timeout)
# wait for pool is finished before processing results. iterator = future.result()
iterator = future.result() ext_timeouts=[]
ext_timeouts=[] for ext_id in ext_ids:
for ext_id in ext_ids: try:
try: results.append(next(iterator))
results.append(next(iterator)) except StopIteration:
except StopIteration: break
break except TimeoutError as error:
except TimeoutError as error: log_warning("WorkerException: Processing of %s took longer than %d seconds" % (ext_id,error.args[1]))
log_warning("WorkerException: Processing of %s took longer than %d seconds" % (ext_id,error.args[1])) ext_timeouts.append(ext_id)
ext_timeouts.append(ext_id) except ProcessExpired as error:
except ProcessExpired as error: log_warning("WorkerException: %s (%s)self. Exit code: %d" % (error, ext_id, error.exitcode))
log_warning("WorkerException: %s (%s)self. Exit code: %d" % (error, ext_id, error.exitcode)) ext_timeouts.append(ext_id)
ext_timeouts.append(ext_id) except Exception as error:
except Exception as error: log_warning("WorkerException: Processing %s raised %s" % (ext_id, error))
log_warning("WorkerException: Processing %s raised %s" % (ext_id, error)) log_warning(error.traceback) # Python's traceback of remote process
log_warning(error.traceback) # Python's traceback of remote process ext_timeouts.append(ext_id)
ext_timeouts.append(ext_id)
return results return results