diff --git a/scripts/global_update.sh b/scripts/global_update.sh index 40333c0..63c2afd 100755 --- a/scripts/global_update.sh +++ b/scripts/global_update.sh @@ -6,15 +6,15 @@ ARCHIVE=${1:-/srv/Shared/BrowserExtensions/archive} CRAWLERHOME=${2:-~/ExtensionCrawler} IMAGE=${3:-/shared/brucker_research1/Shared/BrowserExtensions/bin/ExtensionCrawler.img} -LOGPREFIX=$ARCHIVE/log/`date --iso-8601=ns` +LOGPREFIX=$ARCHIVE/log/`date --utc --iso-8601=ns` LOG=$LOGPREFIX-global.log -date +'* Start Updating Extensions Archive (%c)' | tee $LOG +date --utc +'* Start Updating Extensions Archive (%c)' | tee $LOG # Update extensions singularity exec --bind /srv/:/srv/ $IMAGE crawler -p 40 -d -a $ARCHIVE > $LOGPREFIX.log -date +'* Update Finished (%c)' | tee -a $LOG +date --utc +'* Update Finished (%c)' | tee -a $LOG ERRORS=`grep ERROR $LOGPREFIX.log | sort -k 5,5 -u | wc -l` diff --git a/scripts/update_cdnjs.sh b/scripts/update_cdnjs.sh index deb63cc..8a9891d 100755 --- a/scripts/update_cdnjs.sh +++ b/scripts/update_cdnjs.sh @@ -2,24 +2,24 @@ ARCHIVE=${1:-/srv/Shared/BrowserExtensions/archive} TMPDIR=${TMPDIR:-/tmp} -LOGPREFIX=$ARCHIVE/log/`date --iso-8601=ns` +LOGPREFIX=$ARCHIVE/log/`date --utc --iso-8601=ns` LOG=$LOGPREFIX-cdnjs.log SING_IMG=/shared/brucker_research1/Shared/BrowserExtensions/archive/filedb/ExtensionCrawler-cdnjs.img -date +'* Create backup of disk image (%c)' | tee -a $LOG +date --utc +'* Create backup of disk image (%c)' | tee -a $LOG cp $SING_IMG $SING_IMG.bak SING_EXEC="singularity exec -w --pwd /opt/ExtensionCrawler -B $TMPDIR:/tmp $SING_IMG" ls "$SING_IMG" > /dev/null # Update production branch of WebCrawler repository -date +'* Updating WebCrawler repository (%c)' | tee -a $LOG +date --utc +'* Updating WebCrawler repository (%c)' | tee -a $LOG $SING_EXEC git fetch >> $LOG $SING_EXEC git checkout production >> $LOG 2>&1 $SING_EXEC git pull >> $LOG 2>&1 # $SING_EXEC pip3 install --system -e ../ExtensionCrawler # Update cdnjs git repository and update cdnjs data base table -date +'* Updating CDNJS repository (%c)' | tee -a $LOG +date --utc +'* Updating CDNJS repository (%c)' | tee -a $LOG $SING_EXEC ./cdnjs-git-miner -v -u -a /opt/archive >> $LOG -date +'* Successfully updated CDNJS repository (%c)' | tee -a $LOG +date --utc +'* Successfully updated CDNJS repository (%c)' | tee -a $LOG