Extended const_basedir to check environment variable EXTENSION_ARCHIVE and modified main scripts to actually use const_basedir.

ExtensionCrawler/config.py

@@ -127,7 +127,12 @@ def archive_file(archivedir, ext_id):
 
 def const_basedir():
     """Top-level directory for the extension crawler archive."""
-    return "archive"
+    if "EXTENSION_ARCHIVE" in os.environ:
+        return os.environ.get("EXTENSION_ARCHIVE")
+    else:
+        return "archive"
+
+
 
 def const_parallel_downloads():
     """Number of parallel downloads."""

cdnjs-crawler

@@ -24,7 +24,7 @@ import sys
 
 from ExtensionCrawler.cdnjs import (update_jslib_archive, update_md5_map_file,
                                     update_sha1_map_file)
-from ExtensionCrawler.config import const_log_format
+from ExtensionCrawler.config import (const_log_format, const_basedir)
 
 # Script should run with python 3.4 or 3.5
 assert sys.version_info >= (3, 4) and sys.version_info < (3, 6)
@@ -42,7 +42,7 @@ def helpmsg():
 
 def main(argv):
     """Main function of the extension crawler."""
-    basedir = "archive"
+    basedir = const_basedir()
     verbose = True
     force = False
     clean = False

create-db

@@ -92,7 +92,7 @@ def find_from_file(archive, extidlistfile):
 
 
 def parse_args(argv):
-    archive = "archive"
+    archive = const_basedir()
     parallel = 8
     taskid = 1
     maxtaskid = 1

crx-extract

@@ -26,6 +26,8 @@ import datetime
 import dateutil
 import dateutil.parser
 from ExtensionCrawler.archive import last_crx, get_local_archive_dir, last_etag
+from ExtensionCrawler.config import  const_basedir
+
 
 # Script should run with python 3.4 or 3.5
 assert sys.version_info >= (3, 4) and sys.version_info < (3, 6)
@@ -59,7 +61,7 @@ def get_tarinfo(members, name, winfs = False, etag = None):
 
 def main(argv):
     """Main function of the extension crawler."""
-    basedir = "archive"
+    basedir = const_basedir()
     verbose = True
     date = None
     extid = ""

grepper

@@ -27,7 +27,7 @@ from multiprocessing import Pool, Lock
 from zipfile import ZipFile
 from functools import partial
 import re
-
+from ExtensionCrawler.config import const_basedir
 
 def help():
     print("grepper [OPTION] GREP [FILE]")
@@ -117,7 +117,7 @@ def init(l):
 
 
 def parse_args(argv):
-    archive = "archive"
+    archive = const_basedir()
     beautify = False
     parallel = 8
     taskid = 1