Use UTC-based time/dates for logging.
This commit is contained in:
parent
91a76091e3
commit
2fc154d643
|
@ -6,15 +6,15 @@
|
||||||
ARCHIVE=${1:-/srv/Shared/BrowserExtensions/archive}
|
ARCHIVE=${1:-/srv/Shared/BrowserExtensions/archive}
|
||||||
CRAWLERHOME=${2:-~/ExtensionCrawler}
|
CRAWLERHOME=${2:-~/ExtensionCrawler}
|
||||||
IMAGE=${3:-/shared/brucker_research1/Shared/BrowserExtensions/bin/ExtensionCrawler.img}
|
IMAGE=${3:-/shared/brucker_research1/Shared/BrowserExtensions/bin/ExtensionCrawler.img}
|
||||||
LOGPREFIX=$ARCHIVE/log/`date --iso-8601=ns`
|
LOGPREFIX=$ARCHIVE/log/`date --utc --iso-8601=ns`
|
||||||
LOG=$LOGPREFIX-global.log
|
LOG=$LOGPREFIX-global.log
|
||||||
|
|
||||||
date +'* Start Updating Extensions Archive (%c)' | tee $LOG
|
date --utc +'* Start Updating Extensions Archive (%c)' | tee $LOG
|
||||||
|
|
||||||
# Update extensions
|
# Update extensions
|
||||||
singularity exec --bind /srv/:/srv/ $IMAGE crawler -p 40 -d -a $ARCHIVE > $LOGPREFIX.log
|
singularity exec --bind /srv/:/srv/ $IMAGE crawler -p 40 -d -a $ARCHIVE > $LOGPREFIX.log
|
||||||
|
|
||||||
date +'* Update Finished (%c)' | tee -a $LOG
|
date --utc +'* Update Finished (%c)' | tee -a $LOG
|
||||||
|
|
||||||
|
|
||||||
ERRORS=`grep ERROR $LOGPREFIX.log | sort -k 5,5 -u | wc -l`
|
ERRORS=`grep ERROR $LOGPREFIX.log | sort -k 5,5 -u | wc -l`
|
||||||
|
|
|
@ -2,24 +2,24 @@
|
||||||
|
|
||||||
ARCHIVE=${1:-/srv/Shared/BrowserExtensions/archive}
|
ARCHIVE=${1:-/srv/Shared/BrowserExtensions/archive}
|
||||||
TMPDIR=${TMPDIR:-/tmp}
|
TMPDIR=${TMPDIR:-/tmp}
|
||||||
LOGPREFIX=$ARCHIVE/log/`date --iso-8601=ns`
|
LOGPREFIX=$ARCHIVE/log/`date --utc --iso-8601=ns`
|
||||||
LOG=$LOGPREFIX-cdnjs.log
|
LOG=$LOGPREFIX-cdnjs.log
|
||||||
|
|
||||||
SING_IMG=/shared/brucker_research1/Shared/BrowserExtensions/archive/filedb/ExtensionCrawler-cdnjs.img
|
SING_IMG=/shared/brucker_research1/Shared/BrowserExtensions/archive/filedb/ExtensionCrawler-cdnjs.img
|
||||||
date +'* Create backup of disk image (%c)' | tee -a $LOG
|
date --utc +'* Create backup of disk image (%c)' | tee -a $LOG
|
||||||
cp $SING_IMG $SING_IMG.bak
|
cp $SING_IMG $SING_IMG.bak
|
||||||
SING_EXEC="singularity exec -w --pwd /opt/ExtensionCrawler -B $TMPDIR:/tmp $SING_IMG"
|
SING_EXEC="singularity exec -w --pwd /opt/ExtensionCrawler -B $TMPDIR:/tmp $SING_IMG"
|
||||||
ls "$SING_IMG" > /dev/null
|
ls "$SING_IMG" > /dev/null
|
||||||
|
|
||||||
# Update production branch of WebCrawler repository
|
# Update production branch of WebCrawler repository
|
||||||
date +'* Updating WebCrawler repository (%c)' | tee -a $LOG
|
date --utc +'* Updating WebCrawler repository (%c)' | tee -a $LOG
|
||||||
$SING_EXEC git fetch >> $LOG
|
$SING_EXEC git fetch >> $LOG
|
||||||
$SING_EXEC git checkout production >> $LOG 2>&1
|
$SING_EXEC git checkout production >> $LOG 2>&1
|
||||||
$SING_EXEC git pull >> $LOG 2>&1
|
$SING_EXEC git pull >> $LOG 2>&1
|
||||||
# $SING_EXEC pip3 install --system -e ../ExtensionCrawler
|
# $SING_EXEC pip3 install --system -e ../ExtensionCrawler
|
||||||
|
|
||||||
# Update cdnjs git repository and update cdnjs data base table
|
# Update cdnjs git repository and update cdnjs data base table
|
||||||
date +'* Updating CDNJS repository (%c)' | tee -a $LOG
|
date --utc +'* Updating CDNJS repository (%c)' | tee -a $LOG
|
||||||
$SING_EXEC ./cdnjs-git-miner -v -u -a /opt/archive >> $LOG
|
$SING_EXEC ./cdnjs-git-miner -v -u -a /opt/archive >> $LOG
|
||||||
date +'* Successfully updated CDNJS repository (%c)' | tee -a $LOG
|
date --utc +'* Successfully updated CDNJS repository (%c)' | tee -a $LOG
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue