diff --git a/ExtensionCrawler/cdnjs.py b/ExtensionCrawler/cdnjs.py index e225ea8..72f0308 100644 --- a/ExtensionCrawler/cdnjs.py +++ b/ExtensionCrawler/cdnjs.py @@ -109,7 +109,6 @@ def update_lib(verbose, force, archive, lib): json.dump(cdnjs_lib_json, json_file) - def build_hash_map_of_lib(hashalg, archive, lib): """Build dictionary with file information using the file hash as key.""" dirname = os.path.join(archive, "fileinfo", "cdnjs", "lib") @@ -131,14 +130,17 @@ def build_hash_map_of_lib(hashalg, archive, lib): } return hash_map + def build_sha1_map_of_lib(archive, lib): """Build dictionary with file information using the file sha1 as key.""" return build_hash_map_of_lib("sha1", archive, lib) + def build_md5_map_of_lib(archive, lib): """Build dictionary with file information using the file md5 as key.""" return build_hash_map_of_lib("md5", archive, lib) + def build_hash_map(hashalg, archive): """Build file information dictionary using the file hash as key""" hash_map = None @@ -150,14 +152,31 @@ def build_hash_map(hashalg, archive): hash_map = lib_map return hash_map + def build_sha1_map(archive): """Build file information dictionary using the sha1 hash as key""" return build_hash_map("sha1", archive) + def build_md5_map(archive): """Build file information dictionary using the md5 hash as key""" return build_hash_map("md5", archive) + +def update_md5_map_file(archive): + """Update file containing md5 information for all files.""" + with open(os.path.join(archive, "fileinfo", "cdnjs-md5.json"), + "w") as json_file: + json.dump(build_md5_map(archive), json_file) + + +def update_sha1_map_file(archive): + """Update file containing sha1 information for all files.""" + with open(os.path.join(archive, "fileinfo", "cdnjs-sha1.json"), + "w") as json_file: + json.dump(build_sha1_map(archive), json_file) + + def delete_orphaned(archive, local_libs, cdnjs_current_libs): """Delete all orphaned local libaries.""" dirname = os.path.join(archive, "fileinfo", "cdnjs", "lib") diff --git a/cdnjs-crawler b/cdnjs-crawler index c26711c..0ac7d21 100755 --- a/cdnjs-crawler +++ b/cdnjs-crawler @@ -21,7 +21,7 @@ import getopt import sys -from ExtensionCrawler.cdnjs import update_jslib_archive, delete_orphaned +from ExtensionCrawler.cdnjs import update_jslib_archive, delete_orphaned, update_md5_map_file, update_sha1_map_file # Script should run with python 3.4 or 3.5 assert sys.version_info >= (3, 4) and sys.version_info < (3, 6) @@ -63,6 +63,8 @@ def main(argv): clean = True update_jslib_archive(verbose, force, clean, basedir) + update_sha1_map_file(basedir) + update_md5_map_file(basedir) if __name__ == "__main__":