Added ratingValue and ratingCount to db.
This commit is contained in:
parent
5949150b00
commit
4c01b95f69
|
@ -28,12 +28,15 @@ import json
|
||||||
import os
|
import os
|
||||||
import glob
|
import glob
|
||||||
|
|
||||||
|
|
||||||
class SelfclosingSqliteDB:
|
class SelfclosingSqliteDB:
|
||||||
def __init__(self, filename):
|
def __init__(self, filename):
|
||||||
self.filename = filename
|
self.filename = filename
|
||||||
|
|
||||||
def __enter__(self):
|
def __enter__(self):
|
||||||
self.con = sqlite3.connect(self.filename)
|
self.con = sqlite3.connect(self.filename)
|
||||||
return self.con
|
return self.con
|
||||||
|
|
||||||
def __exit__(self, *args):
|
def __exit__(self, *args):
|
||||||
self.con.commit()
|
self.con.commit()
|
||||||
self.con.close()
|
self.con.close()
|
||||||
|
@ -83,6 +86,8 @@ def setup_tables(con):
|
||||||
"""version TEXT,"""
|
"""version TEXT,"""
|
||||||
"""description TEXT,"""
|
"""description TEXT,"""
|
||||||
"""downloads INTEGER,"""
|
"""downloads INTEGER,"""
|
||||||
|
"""rating REAL,"""
|
||||||
|
"""ratingcount INTEGER,"""
|
||||||
"""fulldescription TEXT,"""
|
"""fulldescription TEXT,"""
|
||||||
"""developer TEXT,"""
|
"""developer TEXT,"""
|
||||||
"""crx_etag TEXT,"""
|
"""crx_etag TEXT,"""
|
||||||
|
@ -96,7 +101,8 @@ def get_etag(ext_id, datepath, con, verbose, indent):
|
||||||
txt = ""
|
txt = ""
|
||||||
|
|
||||||
# Trying to parse etag file
|
# Trying to parse etag file
|
||||||
etagpath = next(iter(glob.glob(os.path.join(datepath, "*.crx.etag"))), None)
|
etagpath = next(
|
||||||
|
iter(glob.glob(os.path.join(datepath, "*.crx.etag"))), None)
|
||||||
if etagpath:
|
if etagpath:
|
||||||
with open(etagpath) as f:
|
with open(etagpath) as f:
|
||||||
return f.read(), txt
|
return f.read(), txt
|
||||||
|
@ -180,6 +186,16 @@ def parse_and_insert_overview(ext_id, date, datepath, con, verbose, indent):
|
||||||
"""<meta itemprop="version" content="(.*?)"\s*/>""", contents)
|
"""<meta itemprop="version" content="(.*?)"\s*/>""", contents)
|
||||||
version = match.group(1) if match else None
|
version = match.group(1) if match else None
|
||||||
|
|
||||||
|
match = re.search(
|
||||||
|
"""<meta itemprop="ratingValue" content="(.*?)"\s*/>""",
|
||||||
|
contents)
|
||||||
|
rating = float(match.group(1)) if match else None
|
||||||
|
|
||||||
|
match = re.search(
|
||||||
|
"""<meta itemprop="ratingCount" content="(.*?)"\s*/>""",
|
||||||
|
contents)
|
||||||
|
rating_count = int(match.group(1)) if match else None
|
||||||
|
|
||||||
# Extracts extension categories
|
# Extracts extension categories
|
||||||
match = re.search(
|
match = re.search(
|
||||||
"""Attribute name="category">(.+?)</Attribute>""", contents)
|
"""Attribute name="category">(.+?)</Attribute>""", contents)
|
||||||
|
@ -213,9 +229,11 @@ def parse_and_insert_overview(ext_id, date, datepath, con, verbose, indent):
|
||||||
etag, etag_msg = get_etag(ext_id, datepath, con, verbose, indent)
|
etag, etag_msg = get_etag(ext_id, datepath, con, verbose, indent)
|
||||||
txt = logmsg(verbose, txt, etag_msg)
|
txt = logmsg(verbose, txt, etag_msg)
|
||||||
|
|
||||||
con.execute("INSERT INTO extension VALUES (?,?,?,?,?,?,?,?,?,?)",
|
con.execute(
|
||||||
(ext_id, date, name, version, description, downloads,
|
"INSERT INTO extension VALUES (?,?,?,?,?,?,?,?,?,?,?,?)",
|
||||||
full_description, developer, etag, last_updated))
|
(ext_id, date, name, version, description, downloads, rating,
|
||||||
|
rating_count, full_description, developer, etag,
|
||||||
|
last_updated))
|
||||||
|
|
||||||
if categories:
|
if categories:
|
||||||
for category in categories:
|
for category in categories:
|
||||||
|
|
Loading…
Reference in New Issue