From 257afe92f02bf3ec31518ac897c3bd6572a19f05 Mon Sep 17 00:00:00 2001
From: Michael Herzberg <mail@michael-herzberg.de>
Date: Sun, 27 Aug 2017 23:00:28 +0100
Subject: [PATCH] Use selective insert.

---
 ExtensionCrawler/dbbackend/mysql_backend.py  | 23 +++++++-------------
 ExtensionCrawler/dbbackend/sqlite_backend.py |  4 ++--
 2 files changed, 10 insertions(+), 17 deletions(-)

diff --git a/ExtensionCrawler/dbbackend/mysql_backend.py b/ExtensionCrawler/dbbackend/mysql_backend.py
index 40af9a9..5e81ab4 100644
--- a/ExtensionCrawler/dbbackend/mysql_backend.py
+++ b/ExtensionCrawler/dbbackend/mysql_backend.py
@@ -39,15 +39,6 @@ class MysqlBackend:
             db = MySQLdb.connect(**self.dbargs)
         self.cursor = db.cursor()
 
-        self.columns = {}
-        self.cursor.execute(
-            "select table_name,column_name from information_schema.columns where table_schema=%s",
-            (self.dbargs["db"], ))
-        for table, column in self.cursor.fetchall():
-            if table not in self.columns:
-                self.columns[table] = []
-            self.columns[table] += [column]
-
         return self
 
     def __exit__(self, *args):
@@ -64,16 +55,18 @@ class MysqlBackend:
             return None
 
     def insertmany(self, table, arglist):
-        args = [
-            tuple([arg[k] for k in self.columns[table]]) for arg in arglist
-        ]
+        args = [tuple(arg.values()) for arg in arglist]
+
         # Looks like this, for example:
-        # INSERT INTO category VALUES (%s,%s,%s) ON DUPLICATE KEY UPDATE extid=VALUES(extid),date=VALUES(date),category=VALUES(category)
-        query = "INSERT INTO {} VALUES ({}) ON DUPLICATE KEY UPDATE {}".format(
+        # INSERT INTO category VALUES(extid,date,category) (%s,%s,%s)
+        #   ON DUPLICATE KEY UPDATE extid=VALUES(extid),date=VALUES(date)
+        #   ,category=VALUES(category)
+        query = "INSERT INTO {}({}) VALUES ({}) ON DUPLICATE KEY UPDATE {}".format(
             table,
+            ",".join(arglist[0].keys()),
             ",".join(len(args[0]) * ["%s"]),
             ",".join(
-                ["{c}=VALUES({c})".format(c=c) for c in self.columns[table]]))
+                ["{c}=VALUES({c})".format(c=c) for c in arglist[0].keys()]))
         self.cursor.executemany(query, args)
 
     def insert(self, table, **kwargs):
diff --git a/ExtensionCrawler/dbbackend/sqlite_backend.py b/ExtensionCrawler/dbbackend/sqlite_backend.py
index 7ab4118..5919dd1 100644
--- a/ExtensionCrawler/dbbackend/sqlite_backend.py
+++ b/ExtensionCrawler/dbbackend/sqlite_backend.py
@@ -154,8 +154,8 @@ class SqliteBackend:
 
     def insert(self, table, **kwargs):
         args = tuple(kwargs.values())
-        self.con.execute("INSERT OR REPLACE INTO {} VALUES ({})".format(
-            table, ",".join(len(args) * ["?"])), args)
+        self.con.execute("INSERT OR REPLACE INTO {}({}) VALUES ({})".format(
+            table, ",".join(kwargs.keys()), ",".join(len(args) * ["?"])), args)
 
     def insertmany(self, table, argslist):
         for arg in argslist: