acb14jd
/
ExtensionCrawler
forked from BrowserSecurity/ExtensionCrawler
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Create tmp/build db to minimize the time in which no db is available.

This commit is contained in:
Achim D. Brucker 2017-08-10 09:06:09 +01:00
parent 5af2cff2d5
commit cd131cb4b8
1 changed files with 32 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
scripts/global_update_db.sh
View File

@ -1,8 +1,8 @@
#!/bin/bash
# m h dom mon dow command
# 01 02 * * * (cd ~/ExtensionCrawler; ((git fetch ; git checkout production; git pull) &> /dev/null))
# 15 01 * * * (cd ~/ExtensionCrawler; ((git fetch ; git checkout production; git pull) &> /dev/null))
# 33 01 * * * ~/ExtensionCrawler/scripts/global_update_db.sh
# 07 02 * * * ~/ExtensionCrawler/scripts/global_update.sh
export LD_LIBRARY_PATH=$HOME/local/lib:/usr/local/lib:$LD_LIBRARY_PATH
export PATH=$HOME/local/bin:/usr/local/bin:$PATH
@ -16,49 +16,54 @@ DBARCHIVE=`find $ARCHIVE/.snapshot -maxdepth 1 -mindepth 1 -name "D*" | sort -r
SQLITE=`which sqlite3`
date +"* Start Creating aa-ac.sqlite Data Base (%c) using $SQLITE (data: $DBARCHIVE/data)" | tee -a $LOG
date +"* Start Creating aa-ac.build.sqlite Data Base (%c) using $SQLITE (data: $DBARCHIVE/data)" | tee -a $LOG
# Update small database
rm -f $ARCHIVE/db/aa-ac.sqlite
(cd $CRAWLERHOME; (./scripts/generate_small_db.sh $DBARCHIVE/data $ARCHIVE/db/aa-ac.sqlite &> $LOGPREFIX-sqlite-aa-ac.log))
rm -f $ARCHIVE/db/aa-ac.build.*
(cd $CRAWLERHOME; (./scripts/generate_small_db.sh $DBARCHIVE/data $ARCHIVE/db/aa-ac.build.sqlite &> $LOGPREFIX-sqlite-aa-ac.log))
if [ $? -ne "0" ]; then
echo " Creation of aa-ac.sqlite failed - see log file for details" | tee -a $LOG
echo " Creation of aa-ac.build.sqlite failed - see log file for details" | tee -a $LOG
else
SIZE=`du -k $ARCHIVE/db/aa-ac.sqlite | cut -f1`
echo " Created aa-ac.sqlite successfully ($SIZE kb)" | tee -a $LOG
SIZE=`du -k $ARCHIVE/db/aa-ac.build.sqlite | cut -f1`
echo " Created aa-ac.build.sqlite successfully ($SIZE kb)" | tee -a $LOG
fi
if [ -f "$ARCHIVE"/db/aa-ac.sqlite ]; then
if [ -f "$ARCHIVE"/db/aa-ac.build.sqlite ]; then
date +'* Start Compressing aa-ac.sqlite Data Base (%c)' | tee -a $LOG
rm -f $ARCHIVE/db/aa-ac.sqlite.bz2
pbzip2 -f "$ARCHIVE"/db/aa-ac.sqlite
pbzip2 -f "$ARCHIVE"/db/aa-ac.build.sqlite
if [ $? -ne "0" ]; then
echo " Creation of aa-ac.sqlite.bz2 failed" | tee -a $LOG
else
SIZE=`du -k $ARCHIVE/db/aa-ac.sqlite.bz2 | cut -f1`
echo " Created aa-ac.sqlite.bz2 successfully ($SIZE kb)" | tee -a $LOG
echo " Creation of aa-ac.sqlite.build.bz2 failed" | tee -a $LOG
rm -f $ARCHIVE/db/aa-ac.build.*
else
rm -f $ARCHIVE/db/aa-ac.sqlite.bz2
mv $ARCHIVE/db/aa-ac.build.sqlite.bz2 $ARCHIVE/db/aa-ac.sqlite.bz2
SIZE=`du -k $ARCHIVE/db/aa-ac.bz2 | cut -f1`
echo " Created aa-ac.sqlite.bz2 successfully ($SIZE kb)" | tee -a $LOG
fi
fi
date +"* Start Creating full.sqlite Data Base (%c) using $SQLITE (data: $DBARCHIVE/data)" | tee -a $LOG
# Update full database
rm -f $ARCHIVE/db/full.sqlite
"$CRAWLERHOME/scripts/merge_dbs" "$DBARCHIVE/data" "$ARCHIVE/db/full.sqlite" &> $LOGPREFIX-sqlite-full.log
rm -f $ARCHIVE/db/full.build.*
"$CRAWLERHOME/scripts/merge_dbs" "$DBARCHIVE/data" "$ARCHIVE/db/full.build.sqlite" &> $LOGPREFIX-sqlite-full.log
if [ $? -ne "0" ]; then
echo " Creation of full.sqlite failed - see log file for details" | tee -a $LOG
echo " Creation of full.build.sqlite failed - see log file for details" | tee -a $LOG
else
SIZE=`du -k $ARCHIVE/db/full.sqlite | cut -f1`
echo " Created full.sqlite successfully ($SIZE kb)" | tee -a $LOG
SIZE=`du -k $ARCHIVE/db/full.build.sqlite | cut -f1`
echo " Created full.build.sqlite successfully ($SIZE kb)" | tee -a $LOG
fi
if [ -f "$ARCHIVE"/db/full.sqlite ]; then
date +'* Start Compressing full.sqlite Data Base (%c)' | tee -a $LOG
rm -f $ARCHIVE/db/full.sqlite.bz2
pbzip2 -f "$ARCHIVE"/db/full.sqlite
if [ -f "$ARCHIVE"/db/full.build.sqlite ]; then
date +'* Start Compressing full.build.sqlite Data Base (%c)' | tee -a $LOG
pbzip2 -f "$ARCHIVE"/db/full.build.sqlite
if [ $? -ne "0" ]; then
echo " Creation of full.sqlite.bz2 failed" | tee -a $LOG
rm -f $ARCHIVE/db/full.build.*
echo " Creation of full.sqlite.bz2 failed" | tee -a $LOG
else
SIZE=`du -k $ARCHIVE/db/full.sqlite.bz2 | cut -f1`
echo " Created full.sqlite.bz2 successfully ($SIZE kb)" | tee -a $LOG
rm -f $ARCHIVE/db/full.sqlite.bz2
mv $ARCHIVE/db/full.build.sqlite.bz2 $ARCHIVE/db/full.sqlite.bz2
SIZE=`du -k $ARCHIVE/db/full.sqlite.bz2 | cut -f1`
echo " Created full.sqlite.bz2 successfully ($SIZE kb)" | tee -a $LOG
fi
fi