BrowserSecurity
/
ExtensionCrawler
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'production'

This commit is contained in:
Achim D. Brucker 2017-11-11 21:56:33 +00:00
parent 5cc7a92f90 9bd283f35a
commit 4cb49f2281
1 changed files with 12 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
ExtensionCrawler/cdnjs_git.py
View File

@ -297,10 +297,12 @@ def update_database(create_csv,
poolsize=16):
"""Update database for all files in files."""
logging.info("Updating data base")
with Pool(poolsize) as pool:
pool.map(
partial(update_database_for_file_chunked, create_csv, release_dic,
cdnjs_git_path), chunks(list(files), 200))
for chunk in chunks(list(files), 200):
update_database_for_file_chunked(create_csv, release_dic, cdnjs_git_path, chunk)
# with Pool(poolsize) as pool:
# pool.map(
# partial(update_database_for_file_chunked, create_csv, release_dic,
# cdnjs_git_path), chunks(list(files), 200))
def get_release_triple(git_path, libver):
@ -315,8 +317,12 @@ def get_release_triple(git_path, libver):
def build_release_date_dic(git_path, libvers, poolsize=16):
""""Build dictionary of release date with the tuple (library, version) as key."""
logging.info("Building release dictionary")
with Pool(poolsize) as pool:
libverdates = pool.map(partial(get_release_triple, git_path), libvers)
libverdates = []
for libver in libvers:
libverdates.append(get_release_triple(git_path, libver))
release_date_dic = {}
# with Pool(poolsize) as pool:
# libverdates = pool.map(partial(get_release_triple, git_path), libvers)
release_date_dic = {}
for (lib, ver, date) in libverdates:
release_date_dic[(lib, ver)] = date