Group mysql inserts and don't compress them.

ExtensionCrawler/db.py

@@ -424,8 +424,7 @@ def update_db_incremental(tmptardir, ext_id, date, con=None):
         with MysqlBackend(
                 ext_id,
                 read_default_file=const_mysql_config_file(),
-                charset='utf8mb4',
-                compress=True) as con:
+                charset='utf8mb4') as con:
             update_db_incremental_with_connection(tmptardir, ext_id, date, con)
 
 

ExtensionCrawler/dbbackend/mysql_backend.py

@@ -19,6 +19,7 @@ import time
 import datetime
 from collections import OrderedDict
 from random import uniform
+import sys
 
 import MySQLdb
 import _mysql_exceptions
@@ -64,13 +65,19 @@ class MysqlBackend:
                 ",".join(len(args[0]) * ["%s"]),
                 ",".join(
                     ["{c}=VALUES({c})".format(c=c) for c in sorted_arglist[0].keys()]))
+            start = time.time()
             self.retry(lambda: self.cursor.executemany(query, args))
+            log_info("* Inserted {} bytes into {}, taking {:.2f}s.".format(sum([sys.getsizeof(arg) for arg in args]),
+                                                                           table, time.time() - start), 3)
+        start = time.time()
+        self.db.commit()
+        log_info("* DB commit took {:.2f}s".format(time.time() - start), 2)
 
     def _create_conn(self):
         if self.db is None:
             log_info("* self.db is None,  open new connection ...", 3)
             self.db = MySQLdb.connect(**self.dbargs)
-            self.db.autocommit(True)
+            self.db.autocommit(False)
             log_info("* success", 4)
         if self.cursor is None:
             log_info("* self.cursor is None,  assigning new cursor ...", 3)

create-db

@@ -70,8 +70,7 @@ def process_id(from_date, until_date, path):
                         try_wait=10,
                         maxtries=30,
                         read_default_file=const_mysql_config_file(),
-                        charset='utf8mb4',
-                        compress=True) as con:
+                        charset='utf8mb4') as con:
                     for date in sorted(os.listdir(iddir)):
                         if (from_date is not None and date < from_date) or \
                                 (until_date is not None and date > until_date):