The method pull_get_updated_lib_files(...) now also returns unique library/version information.
This commit is contained in:
parent
e3f1202e44
commit
12a1e282aa
|
@ -55,7 +55,6 @@ def get_add_date(git_path, filename):
|
||||||
def pull_get_list_changed_files(gitrepo):
|
def pull_get_list_changed_files(gitrepo):
|
||||||
"""Pull new updates from remote origin."""
|
"""Pull new updates from remote origin."""
|
||||||
files = []
|
files = []
|
||||||
libvers = [] # TODO
|
|
||||||
cdnjs_origin = gitrepo.remotes.origin
|
cdnjs_origin = gitrepo.remotes.origin
|
||||||
fetch_info = cdnjs_origin.pull()
|
fetch_info = cdnjs_origin.pull()
|
||||||
for single_fetch_info in fetch_info:
|
for single_fetch_info in fetch_info:
|
||||||
|
@ -63,8 +62,7 @@ def pull_get_list_changed_files(gitrepo):
|
||||||
single_fetch_info.old_commit):
|
single_fetch_info.old_commit):
|
||||||
if not diff.a_blob.path in files:
|
if not diff.a_blob.path in files:
|
||||||
files.append(diff.a_blob.path)
|
files.append(diff.a_blob.path)
|
||||||
return files, libvers
|
return files
|
||||||
|
|
||||||
|
|
||||||
def normalize_jsdata(str_data):
|
def normalize_jsdata(str_data):
|
||||||
"""Compute normalized code blocks of a JavaScript file"""
|
"""Compute normalized code blocks of a JavaScript file"""
|
||||||
|
@ -220,16 +218,22 @@ def get_file_libinfo(release_dic, git_path, libfile):
|
||||||
def pull_get_updated_lib_files(cdnjs_git_path):
|
def pull_get_updated_lib_files(cdnjs_git_path):
|
||||||
"""Pull repository and determine updated libraries."""
|
"""Pull repository and determine updated libraries."""
|
||||||
logging.info("Building file list (only updates)")
|
logging.info("Building file list (only updates)")
|
||||||
|
libvers = set()
|
||||||
files = []
|
files = []
|
||||||
cdnjs_repo = git.Repo(cdnjs_git_path)
|
cdnjs_repo = git.Repo(cdnjs_git_path)
|
||||||
for update in pull_get_list_changed_files(cdnjs_repo):
|
for update in pull_get_list_changed_files(cdnjs_repo):
|
||||||
if not (os.path.basename(update) in ["package.json", ".gitkeep"]):
|
if not (os.path.basename(update) in ["package.json", ".gitkeep"]):
|
||||||
if update.startswith("ajax"):
|
if update.startswith("ajax"):
|
||||||
files.append(update)
|
fname = os.path.join(cdnjs_git_path, update)
|
||||||
|
files.append(fname)
|
||||||
|
plist = path_to_list(update)
|
||||||
|
if len(plist) == 4:
|
||||||
|
libvers.add(fname)
|
||||||
del cdnjs_repo
|
del cdnjs_repo
|
||||||
gc.collect()
|
gc.collect()
|
||||||
logging.info("Found " + str(len(files)) + " files")
|
logging.info("Found " + str(len(files)) + " files")
|
||||||
return files
|
logging.info("Found " + str(len(libvers)) + " unique library/version combinations.")
|
||||||
|
return files, list(libvers)
|
||||||
|
|
||||||
|
|
||||||
def get_all_lib_files(cdnjs_git_path):
|
def get_all_lib_files(cdnjs_git_path):
|
||||||
|
@ -269,13 +273,6 @@ def update_database(release_dic, cdnjs_git_path, files, poolsize=16):
|
||||||
pool.map(partial(update_database_for_file, release_dic, cdnjs_git_path), files)
|
pool.map(partial(update_database_for_file, release_dic, cdnjs_git_path), files)
|
||||||
|
|
||||||
|
|
||||||
def pull_and_update_db(cdnjs_git_path, poolsize=16):
|
|
||||||
"""Pull repo and update database."""
|
|
||||||
files, libvers = pull_get_updated_lib_files(cdnjs_git_path)
|
|
||||||
release_dic = build_release_date_dic(cdnjs_git_path, libvers)
|
|
||||||
del libvers
|
|
||||||
gc.collect()
|
|
||||||
update_database(release_dic, cdnjs_git_path, files, poolsize)
|
|
||||||
|
|
||||||
def build_release_date_dic(git_path, libvers):
|
def build_release_date_dic(git_path, libvers):
|
||||||
""""Build dictionary of release date with the tuple (library, version) as key."""
|
""""Build dictionary of release date with the tuple (library, version) as key."""
|
||||||
|
@ -290,6 +287,14 @@ def build_release_date_dic(git_path, libvers):
|
||||||
release_date_dic[(lib,ver)] = date
|
release_date_dic[(lib,ver)] = date
|
||||||
return release_date_dic
|
return release_date_dic
|
||||||
|
|
||||||
|
def pull_and_update_db(cdnjs_git_path, poolsize=16):
|
||||||
|
"""Pull repo and update database."""
|
||||||
|
files, libvers = pull_get_updated_lib_files(cdnjs_git_path)
|
||||||
|
release_dic = build_release_date_dic(cdnjs_git_path, libvers)
|
||||||
|
del libvers
|
||||||
|
gc.collect()
|
||||||
|
update_database(release_dic, cdnjs_git_path, files, poolsize)
|
||||||
|
|
||||||
def update_db_all_libs(cdnjs_git_path, poolsize=16):
|
def update_db_all_libs(cdnjs_git_path, poolsize=16):
|
||||||
"""Update database entries for all libs in git repo."""
|
"""Update database entries for all libs in git repo."""
|
||||||
files, libvers = get_all_lib_files(cdnjs_git_path)
|
files, libvers = get_all_lib_files(cdnjs_git_path)
|
||||||
|
|
Loading…
Reference in New Issue