BrowserSecurity
/
ExtensionCrawler
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Create log file for profiling.

This commit is contained in:
Achim D. Brucker 2017-07-20 07:51:30 +01:00
parent c73f4cda3e
commit 052748af07
1 changed files with 15 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
scripts/global_update.sh
View File

@ -10,57 +10,58 @@ export PATH=/usr/local/bin:$PATH
ARCHIVE=${1:-/srv/Shared/BrowserExtensions/}
CRAWLERHOME=${2:-~/ExtensionCrawler}
LOGPREFIX=$ARCHIVE/log/`date --iso-8601=ns`
LOG=$LOGPREFIX-global.log
date +'* Start Updating Extensions Archive (%c)'
date +'* Start Updating Extensions Archive (%c)' | tee $LOG
SQLITE=`which sqlite3`
# Update extensions
(cd $CRAWLERHOME; (./crawler -d -a $ARCHIVE > $LOGPREFIX.log))
date +"* Start Creating aa-ac.sqlite Data Base (%c) using $SQLITE"
date +"* Start Creating aa-ac.sqlite Data Base (%c) using $SQLITE" | tee $LOG
# Update small database
rm -f $ARCHIVE/db/aa-ac.sqlite
(cd $CRAWLERHOME; (./scripts/generate_small_db.sh $ARCHIVE/data $ARCHIVE/db/aa-ac.sqlite &> $LOGPREFIX-sqlite-aa-ac.log))
if [ $? -ne "0" ]; then
echo " Creation of aa-ac.sqlite failed - see log file for details"
echo " Creation of aa-ac.sqlite failed - see log file for details" | tee $LOG
else
SIZE=`du -k $ARCHIVE/db/aa-ac.sqlite | cut -f1`
echo " Created aa-ac.sqlite successfully ($SIZE kb)"
echo " Created aa-ac.sqlite successfully ($SIZE kb)" | tee $LOG
fi
if [ -f "$ARCHIVE"/db/aa-ac.sqlite ]; then
date +'* Start Compressing aa-ac.sqlite Data Base (%c)'
date +'* Start Compressing aa-ac.sqlite Data Base (%c)' | tee $LOG
rm -f $ARCHIVE/db/aa-ac.sqlite.bz2
pbzip2 -f "$ARCHIVE"/db/aa-ac.sqlite
if [ $? -ne "0" ]; then
echo " Creation of aa-ac.sqlite.bz2 failed"
echo " Creation of aa-ac.sqlite.bz2 failed" | tee $LOG
else
SIZE=`du -k $ARCHIVE/db/aa-ac.sqlite.bz2 | cut -f1`
echo " Created aa-ac.sqlite.bz2 successfully ($SIZE kb)"
echo " Created aa-ac.sqlite.bz2 successfully ($SIZE kb)" | tee $LOG
fi
fi
date +"* Start Creating full.sqlite Data Base (%c) using $SQLITE"
date +"* Start Creating full.sqlite Data Base (%c) using $SQLITE" | tee $LOG
# Update full database
rm -f $ARCHIVE/db/full.sqlite
find "$ARCHIVE"/data/ -name "*.sqlite" -exec "$CRAWLERHOME/scripts/merge_dbs.sh" "{}" "$ARCHIVE"/db/full.sqlite \; &> $LOGPREFIX-sqlite-full.log
if [ $? -ne "0" ]; then
echo " Creation of full.sqlite failed - see log file for details"
echo " Creation of full.sqlite failed - see log file for details" | tee $LOG
else
SIZE=`du -k $ARCHIVE/db/full.sqlite | cut -f1`
echo " Created full.sqlite successfully ($SIZE kb)"
echo " Created full.sqlite successfully ($SIZE kb)" | tee $LOG
fi
if [ -f "$ARCHIVE"/db/full.sqlite ]; then
date +'* Start Compressing full.sqlite Data Base (%c)'
date +'* Start Compressing full.sqlite Data Base (%c)' | tee $LOG
rm -f $ARCHIVE/db/full.sqlite.bz2
pbzip2 -f "$ARCHIVE"/db/full.sqlite
if [ $? -ne "0" ]; then
echo " Creation of full.sqlite.bz2 failed"
echo " Creation of full.sqlite.bz2 failed" | tee $LOG
else
SIZE=`du -k $ARCHIVE/db/full.sqlite.bz2 | cut -f1`
echo " Created full.sqlite.bz2 successfully ($SIZE kb)"
echo " Created full.sqlite.bz2 successfully ($SIZE kb)" | tee $LOG
fi
fi
@ -69,5 +70,5 @@ fi
# pbzip2 -f $f &> /dev/null
# done
date +'* Update Finished (%c)'
date +'* Update Finished (%c)' | tee $LOG