BrowserSecurity
/
ExtensionCrawler
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added methods for generating/updating index files based on the file hash.

This commit is contained in:
Achim D. Brucker 2017-09-02 22:10:43 +01:00
parent 9ae5905973
commit 515a462938
2 changed files with 23 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
ExtensionCrawler/cdnjs.py
View File

@ -109,7 +109,6 @@ def update_lib(verbose, force, archive, lib):
json.dump(cdnjs_lib_json, json_file)
def build_hash_map_of_lib(hashalg, archive, lib):
"""Build dictionary with file information using the file hash as key."""
dirname = os.path.join(archive, "fileinfo", "cdnjs", "lib")
@ -131,14 +130,17 @@ def build_hash_map_of_lib(hashalg, archive, lib):
}
return hash_map
def build_sha1_map_of_lib(archive, lib):
"""Build dictionary with file information using the file sha1 as key."""
return build_hash_map_of_lib("sha1", archive, lib)
def build_md5_map_of_lib(archive, lib):
"""Build dictionary with file information using the file md5 as key."""
return build_hash_map_of_lib("md5", archive, lib)
def build_hash_map(hashalg, archive):
"""Build file information dictionary using the file hash as key"""
hash_map = None
@ -150,14 +152,31 @@ def build_hash_map(hashalg, archive):
hash_map = lib_map
return hash_map
def build_sha1_map(archive):
"""Build file information dictionary using the sha1 hash as key"""
return build_hash_map("sha1", archive)
def build_md5_map(archive):
"""Build file information dictionary using the md5 hash as key"""
return build_hash_map("md5", archive)
def update_md5_map_file(archive):
"""Update file containing md5 information for all files."""
with open(os.path.join(archive, "fileinfo", "cdnjs-md5.json"),
"w") as json_file:
json.dump(build_md5_map(archive), json_file)
def update_sha1_map_file(archive):
"""Update file containing sha1 information for all files."""
with open(os.path.join(archive, "fileinfo", "cdnjs-sha1.json"),
"w") as json_file:
json.dump(build_sha1_map(archive), json_file)
def delete_orphaned(archive, local_libs, cdnjs_current_libs):
"""Delete all orphaned local libaries."""
dirname = os.path.join(archive, "fileinfo", "cdnjs", "lib")

4
cdnjs-crawler
View File

@ -21,7 +21,7 @@
import getopt
import sys
from ExtensionCrawler.cdnjs import update_jslib_archive, delete_orphaned
from ExtensionCrawler.cdnjs import update_jslib_archive, delete_orphaned, update_md5_map_file, update_sha1_map_file
# Script should run with python 3.4 or 3.5
assert sys.version_info >= (3, 4) and sys.version_info < (3, 6)
@ -63,6 +63,8 @@ def main(argv):
clean = True
update_jslib_archive(verbose, force, clean, basedir)
update_sha1_map_file(basedir)
update_md5_map_file(basedir)
if __name__ == "__main__":