Added md5sum to sqlite.

This commit is contained in:
Michael Herzberg 2017-07-31 20:38:21 +01:00
parent 9f6cfc22a3
commit b34d45c4dc
1 changed files with 6 additions and 2 deletions

View File

@ -22,6 +22,7 @@ from ExtensionCrawler.archive import *
import sqlite3
import re
import hashlib
from bs4 import BeautifulSoup
from zipfile import ZipFile
import json
@ -102,6 +103,7 @@ def setup_tables(con):
"""crx_etag TEXT,"""
"""path TEXT,"""
"""size INTEGER,"""
"""md5 TEXT,"""
"""PRIMARY KEY (crx_etag, path)"""
""")""")
con.execute("""CREATE TABLE status ("""
@ -331,8 +333,10 @@ def parse_and_insert_crx(ext_id, date, datepath, con, verbose, indent):
jsfiles = filter(lambda x: x.filename.endswith(".js"),
f.infolist())
for jsfile in jsfiles:
con.execute("INSERT INTO jsfile VALUES (?,?,?)",
(etag, jsfile.filename, int(jsfile.file_size)))
with f.open(jsfile) as f2:
md5 = hashlib.md5(f2.read()).hexdigest()
con.execute("INSERT INTO jsfile VALUES (?,?,?,?)",
(etag, jsfile.filename, int(jsfile.file_size),md5))
public_key = read_crx(crx_path).public_key