A Python crawler for extensions from the Chrome Web Store.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

126 lines
4.0 KiB

#!/usr/bin/env python3
#
# Copyright (C) 2017 The University of Sheffield, UK
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
"""Tool for extracting crx file from a tar archive."""
import os
import sys
import getopt
import tarfile
import datetime
import dateutil
import dateutil.parser
from ExtensionCrawler.archive import last_crx, get_local_archive_dir, last_etag
# Script should run with python 3.4 or 3.5
assert sys.version_info >= (3, 4) and sys.version_info < (3, 6)
def helpmsg():
"""Print help message."""
print("extract-crx [OPTION] extid")
print(" -h print this help text")
print(" -s silent (no log messages)")
print(" -e use etag instead of date in outoput")
print(" -w avoid ':' in filenames (useful on Windows)")
print(" -d=<DATE> date")
print(" -o=<DIR> output directory")
print(" -a=<DIR> archive directory")
def get_tarinfo(members, name, winfs = False, etag = None):
"""Select tarinfo object with a specified path/name."""
for tarinfo in members:
if tarinfo.name == name:
if winfs:
tarinfo.name = name.replace(":","-")
if etag is not None:
(path, crx) = os.path.split(tarinfo.name)
(path, _) = os.path.split(path)
tarinfo.name = os.path.join(path,etag,crx)
yield tarinfo
def main(argv):
"""Main function of the extension crawler."""
basedir = "archive"
verbose = True
date = None
extid = ""
useetag = False
output = ""
winfs = False
try:
opts, args = getopt.getopt(argv, "hsed:a:o:w", ["date=", "archive=", "output="])
except getopt.GetoptError:
helpmsg()
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
helpmsg()
sys.exit()
elif opt in ("-a", "--archive"):
basedir = arg
elif opt in ("-d", "--date"):
date = arg
elif opt in ("-o", "--output"):
output = arg
elif opt in ("-w", "--winfs"):
winfs = True
elif opt in ("-e", "--etag"):
useetag = True
elif opt == '-s':
verbose = False
if len(args) > 0:
extid = args[0]
else:
helpmsg()
sys.exit()
if date is not None:
dateobj = dateutil.parser.parse(date)
if dateobj.tzinfo is None or dateobj.tzinfo.utcoffset(dateobj) is None:
dateobj = dateobj.replace(tzinfo=datetime.timezone.utc)
last = last_crx(os.path.join(basedir, "data"), extid, dateobj)
else:
last = last_crx(os.path.join(basedir, "data"), extid)
if useetag:
etag = last_etag(os.path.join(basedir, "data"), extid, last)
else:
etag = None
tar = os.path.join(basedir, "data",
get_local_archive_dir(extid), extid + ".tar")
if last != "":
if os.path.exists(tar):
if verbose:
print("Extracting "+os.path.join(output, last))
with tarfile.open(tar, 'r') as archive:
archive.extractall(path=output, members=get_tarinfo(archive, last, winfs, etag))
elif verbose:
print("Cannot find archive " + tar)
elif verbose:
if os.path.exists(tar):
print("CRX not in archive" + tar)
else:
print("CRX does not exist: cannot find archive " + tar)
if __name__ == "__main__":
main(sys.argv[1:])