diff --git a/scripts/global_update.sh b/scripts/global_update.sh index 2f71cb0..0b8607a 100755 --- a/scripts/global_update.sh +++ b/scripts/global_update.sh @@ -10,57 +10,58 @@ export PATH=/usr/local/bin:$PATH ARCHIVE=${1:-/srv/Shared/BrowserExtensions/} CRAWLERHOME=${2:-~/ExtensionCrawler} LOGPREFIX=$ARCHIVE/log/`date --iso-8601=ns` +LOG=$LOGPREFIX-global.log -date +'* Start Updating Extensions Archive (%c)' +date +'* Start Updating Extensions Archive (%c)' | tee $LOG SQLITE=`which sqlite3` # Update extensions (cd $CRAWLERHOME; (./crawler -d -a $ARCHIVE > $LOGPREFIX.log)) -date +"* Start Creating aa-ac.sqlite Data Base (%c) using $SQLITE" +date +"* Start Creating aa-ac.sqlite Data Base (%c) using $SQLITE" | tee $LOG # Update small database rm -f $ARCHIVE/db/aa-ac.sqlite (cd $CRAWLERHOME; (./scripts/generate_small_db.sh $ARCHIVE/data $ARCHIVE/db/aa-ac.sqlite &> $LOGPREFIX-sqlite-aa-ac.log)) if [ $? -ne "0" ]; then - echo " Creation of aa-ac.sqlite failed - see log file for details" + echo " Creation of aa-ac.sqlite failed - see log file for details" | tee $LOG else SIZE=`du -k $ARCHIVE/db/aa-ac.sqlite | cut -f1` - echo " Created aa-ac.sqlite successfully ($SIZE kb)" + echo " Created aa-ac.sqlite successfully ($SIZE kb)" | tee $LOG fi if [ -f "$ARCHIVE"/db/aa-ac.sqlite ]; then - date +'* Start Compressing aa-ac.sqlite Data Base (%c)' + date +'* Start Compressing aa-ac.sqlite Data Base (%c)' | tee $LOG rm -f $ARCHIVE/db/aa-ac.sqlite.bz2 pbzip2 -f "$ARCHIVE"/db/aa-ac.sqlite if [ $? -ne "0" ]; then - echo " Creation of aa-ac.sqlite.bz2 failed" + echo " Creation of aa-ac.sqlite.bz2 failed" | tee $LOG else SIZE=`du -k $ARCHIVE/db/aa-ac.sqlite.bz2 | cut -f1` - echo " Created aa-ac.sqlite.bz2 successfully ($SIZE kb)" + echo " Created aa-ac.sqlite.bz2 successfully ($SIZE kb)" | tee $LOG fi fi -date +"* Start Creating full.sqlite Data Base (%c) using $SQLITE" +date +"* Start Creating full.sqlite Data Base (%c) using $SQLITE" | tee $LOG # Update full database rm -f $ARCHIVE/db/full.sqlite find "$ARCHIVE"/data/ -name "*.sqlite" -exec "$CRAWLERHOME/scripts/merge_dbs.sh" "{}" "$ARCHIVE"/db/full.sqlite \; &> $LOGPREFIX-sqlite-full.log if [ $? -ne "0" ]; then - echo " Creation of full.sqlite failed - see log file for details" + echo " Creation of full.sqlite failed - see log file for details" | tee $LOG else SIZE=`du -k $ARCHIVE/db/full.sqlite | cut -f1` - echo " Created full.sqlite successfully ($SIZE kb)" + echo " Created full.sqlite successfully ($SIZE kb)" | tee $LOG fi if [ -f "$ARCHIVE"/db/full.sqlite ]; then - date +'* Start Compressing full.sqlite Data Base (%c)' + date +'* Start Compressing full.sqlite Data Base (%c)' | tee $LOG rm -f $ARCHIVE/db/full.sqlite.bz2 pbzip2 -f "$ARCHIVE"/db/full.sqlite if [ $? -ne "0" ]; then - echo " Creation of full.sqlite.bz2 failed" + echo " Creation of full.sqlite.bz2 failed" | tee $LOG else SIZE=`du -k $ARCHIVE/db/full.sqlite.bz2 | cut -f1` - echo " Created full.sqlite.bz2 successfully ($SIZE kb)" + echo " Created full.sqlite.bz2 successfully ($SIZE kb)" | tee $LOG fi fi @@ -69,5 +70,5 @@ fi # pbzip2 -f $f &> /dev/null # done -date +'* Update Finished (%c)' +date +'* Update Finished (%c)' | tee $LOG