BrowserSecurity
/
ExtensionCrawler
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Removed queue length and reduced mysql insert batch size.

This commit is contained in:
Michael Herzberg 2018-07-28 19:02:38 +02:00
parent 4592cba9b2
commit 947ecf50d4
2 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ExtensionCrawler/dbbackend/mysql_backend.py
View File

@ -163,7 +163,7 @@ class MysqlBackend:
if table not in self.cache:
self.cache[table] = []
self.cache[table] += arglist
if len(self.cache[table]) >= 1000:
if len(self.cache[table]) >= 128:
self._do_insert(table, self.cache[table])
self.cache[table] = []
if self.cache_etags and table == "extension":

2
ExtensionCrawler/dbbackend/mysql_process.py
View File

@ -65,7 +65,7 @@ class MysqlProcessBackend:
def __init__(self, ext_id, **mysql_kwargs):
self.mysql_kwargs = mysql_kwargs
self.m = Manager()
self.queue = self.m.Queue(1000)
self.queue = self.m.Queue()
def __enter__(self):
self.p = Process(target=run, args=(self.mysql_kwargs, self.queue))