ExtensionCrawler/scripts/update_cdnjs.sh

26 lines
987 B
Bash
Raw Normal View History

2017-10-15 13:06:20 +00:00
#!/bin/bash
2017-10-17 16:44:13 +00:00
ARCHIVE=${1:-/srv/Shared/BrowserExtensions/archive}
2017-10-18 19:20:42 +00:00
TMPDIR=${TMPDIR:-/tmp}
2017-10-17 05:50:48 +00:00
LOGPREFIX=$ARCHIVE/log/`date --iso-8601=ns`
LOG=$LOGPREFIX-cdnjs.log
2017-10-15 13:06:20 +00:00
SING_IMG=/shared/brucker_research1/Shared/BrowserExtensions/archive/filedb/ExtensionCrawler-cdnjs.img
2017-11-12 14:49:51 +00:00
date +'* Create backup of disk image (%c)' | tee -a $LOG
2017-10-17 14:42:03 +00:00
cp $SING_IMG $SING_IMG.bak
2017-10-15 13:06:20 +00:00
SING_EXEC="singularity exec -w --pwd /opt/ExtensionCrawler -B $TMPDIR:/tmp $SING_IMG"
ls "$SING_IMG" > /dev/null
# Update production branch of WebCrawler repository
2017-11-12 14:49:51 +00:00
date +'* Updating WebCrawler repository (%c)' | tee -a $LOG
$SING_EXEC git fetch >> $LOG
$SING_EXEC git checkout production >> $LOG 2>&1
$SING_EXEC git pull >> $LOG 2>&1
# $SING_EXEC pip3 install --system -e ../ExtensionCrawler
2017-10-15 13:06:20 +00:00
# Update cdnjs git repository and update cdnjs data base table
2017-11-12 14:49:51 +00:00
date +'* Updating CDNJS repository (%c)' | tee -a $LOG
2017-11-12 14:07:25 +00:00
$SING_EXEC ./cdnjs-git-miner -v -u -a /opt/archive >> $LOG
2017-11-12 14:49:51 +00:00
date +'* Successfully updated CDNJS repository (%c)' | tee -a $LOG
2017-10-15 13:06:20 +00:00