Clean-up of logging infrastructure.
This commit is contained in:
parent
f42f8e3d03
commit
cfeb29d95f
|
@ -55,7 +55,7 @@ def get_local_libs(archive):
|
||||||
glob.glob(os.path.join(dirname, "*.json")))))
|
glob.glob(os.path.join(dirname, "*.json")))))
|
||||||
|
|
||||||
|
|
||||||
def update_lib(verbose, force, archive, lib):
|
def update_lib(force, archive, lib):
|
||||||
"""Update information for a JavaScript library."""
|
"""Update information for a JavaScript library."""
|
||||||
name = lib['name']
|
name = lib['name']
|
||||||
try:
|
try:
|
||||||
|
@ -102,22 +102,19 @@ def update_lib(verbose, force, archive, lib):
|
||||||
|
|
||||||
for lib_ver in cdnjs_lib_json['assets']:
|
for lib_ver in cdnjs_lib_json['assets']:
|
||||||
version = lib_ver['version']
|
version = lib_ver['version']
|
||||||
if verbose:
|
logging.info(" Checking " + str(lib['name']) + " " + str(version))
|
||||||
logging.info(" Checking " + str(lib['name']) + " " + str(version))
|
|
||||||
files_with_hashes = []
|
files_with_hashes = []
|
||||||
if not force and version in local_versions:
|
if not force and version in local_versions:
|
||||||
if verbose:
|
logging.info(" Updating from local record.")
|
||||||
logging.info(" Updating from local record.")
|
|
||||||
old_record = next(x for x in local_lib_json['assets']
|
old_record = next(x for x in local_lib_json['assets']
|
||||||
if x['version'] == lib_ver['version'])
|
if x['version'] == lib_ver['version'])
|
||||||
files_with_hashes = old_record['files']
|
files_with_hashes = old_record['files']
|
||||||
else:
|
else:
|
||||||
if verbose:
|
logging.warning(" Updating from remote record (" + name + " " +
|
||||||
logging.info(" Updating from remote record.")
|
version + ").")
|
||||||
for jsfile in lib_ver['files']:
|
for jsfile in lib_ver['files']:
|
||||||
jsfile_url = get_jsfile_url(name, version, jsfile)
|
jsfile_url = get_jsfile_url(name, version, jsfile)
|
||||||
if verbose:
|
logging.info(" " + jsfile_url)
|
||||||
logging.info(" " + jsfile_url)
|
|
||||||
try:
|
try:
|
||||||
res_jsfile = requests.get(jsfile_url, timeout=10)
|
res_jsfile = requests.get(jsfile_url, timeout=10)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
@ -161,8 +158,7 @@ def update_lib(verbose, force, archive, lib):
|
||||||
cdnjs_lib_json['assets'] = cdnjs_lib_json['assets'] + outphased
|
cdnjs_lib_json['assets'] = cdnjs_lib_json['assets'] + outphased
|
||||||
|
|
||||||
output = os.path.join(dirname, name + ".json")
|
output = os.path.join(dirname, name + ".json")
|
||||||
if verbose:
|
logging.info(" Saving " + str(output))
|
||||||
logging.info(" Saving " + str(output))
|
|
||||||
with open(output, "w") as json_file:
|
with open(output, "w") as json_file:
|
||||||
json.dump(cdnjs_lib_json, json_file)
|
json.dump(cdnjs_lib_json, json_file)
|
||||||
|
|
||||||
|
@ -245,7 +241,7 @@ def delete_orphaned(archive, local_libs, cdnjs_current_libs):
|
||||||
os.remove(os.path.join(dirname, lib + ".json"))
|
os.remove(os.path.join(dirname, lib + ".json"))
|
||||||
|
|
||||||
|
|
||||||
def update_jslib_archive(verbose, force, clean, archive):
|
def update_jslib_archive(force, clean, archive):
|
||||||
"""Update information for all available JavaScript libraries."""
|
"""Update information for all available JavaScript libraries."""
|
||||||
cdnjs_all_libs_url = get_cdnjs_all_libs_url()
|
cdnjs_all_libs_url = get_cdnjs_all_libs_url()
|
||||||
try:
|
try:
|
||||||
|
@ -270,9 +266,8 @@ def update_jslib_archive(verbose, force, clean, archive):
|
||||||
os.makedirs(str(dirname), exist_ok=True)
|
os.makedirs(str(dirname), exist_ok=True)
|
||||||
with open(os.path.join(dirname, "cdnjs-libraries.json"), "w") as json_file:
|
with open(os.path.join(dirname, "cdnjs-libraries.json"), "w") as json_file:
|
||||||
json.dump(res.json(), json_file)
|
json.dump(res.json(), json_file)
|
||||||
if verbose:
|
logging.info("Found " + str(len(cdnjs_lib_catalog)) +
|
||||||
logging.info("Found " + str(len(cdnjs_lib_catalog)) +
|
" different libraries")
|
||||||
" different libraries")
|
|
||||||
|
|
||||||
with Pool(16) as p:
|
with Pool(32) as p:
|
||||||
p.map(partial(update_lib, verbose, force, archive), cdnjs_lib_catalog)
|
p.map(partial(update_lib, force, archive), cdnjs_lib_catalog)
|
||||||
|
|
|
@ -76,7 +76,7 @@ def main(argv):
|
||||||
logger.addHandler(ch)
|
logger.addHandler(ch)
|
||||||
logger.setLevel(loglevel)
|
logger.setLevel(loglevel)
|
||||||
|
|
||||||
update_jslib_archive(verbose, force, clean, basedir)
|
update_jslib_archive(force, clean, basedir)
|
||||||
update_sha1_map_file(basedir)
|
update_sha1_map_file(basedir)
|
||||||
update_md5_map_file(basedir)
|
update_md5_map_file(basedir)
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue