Group mysql inserts and don't compress them.
This commit is contained in:
parent
1884a02c33
commit
3bef0afe7a
|
@ -424,8 +424,7 @@ def update_db_incremental(tmptardir, ext_id, date, con=None):
|
|||
with MysqlBackend(
|
||||
ext_id,
|
||||
read_default_file=const_mysql_config_file(),
|
||||
charset='utf8mb4',
|
||||
compress=True) as con:
|
||||
charset='utf8mb4') as con:
|
||||
update_db_incremental_with_connection(tmptardir, ext_id, date, con)
|
||||
|
||||
|
||||
|
|
|
@ -19,6 +19,7 @@ import time
|
|||
import datetime
|
||||
from collections import OrderedDict
|
||||
from random import uniform
|
||||
import sys
|
||||
|
||||
import MySQLdb
|
||||
import _mysql_exceptions
|
||||
|
@ -64,13 +65,19 @@ class MysqlBackend:
|
|||
",".join(len(args[0]) * ["%s"]),
|
||||
",".join(
|
||||
["{c}=VALUES({c})".format(c=c) for c in sorted_arglist[0].keys()]))
|
||||
start = time.time()
|
||||
self.retry(lambda: self.cursor.executemany(query, args))
|
||||
log_info("* Inserted {} bytes into {}, taking {:.2f}s.".format(sum([sys.getsizeof(arg) for arg in args]),
|
||||
table, time.time() - start), 3)
|
||||
start = time.time()
|
||||
self.db.commit()
|
||||
log_info("* DB commit took {:.2f}s".format(time.time() - start), 2)
|
||||
|
||||
def _create_conn(self):
|
||||
if self.db is None:
|
||||
log_info("* self.db is None, open new connection ...", 3)
|
||||
self.db = MySQLdb.connect(**self.dbargs)
|
||||
self.db.autocommit(True)
|
||||
self.db.autocommit(False)
|
||||
log_info("* success", 4)
|
||||
if self.cursor is None:
|
||||
log_info("* self.cursor is None, assigning new cursor ...", 3)
|
||||
|
|
|
@ -70,8 +70,7 @@ def process_id(from_date, until_date, path):
|
|||
try_wait=10,
|
||||
maxtries=30,
|
||||
read_default_file=const_mysql_config_file(),
|
||||
charset='utf8mb4',
|
||||
compress=True) as con:
|
||||
charset='utf8mb4') as con:
|
||||
for date in sorted(os.listdir(iddir)):
|
||||
if (from_date is not None and date < from_date) or \
|
||||
(until_date is not None and date > until_date):
|
||||
|
|
Loading…
Reference in New Issue