BrowserSecurity
/
ExtensionCrawler
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Improved log output.

This commit is contained in:
Achim D. Brucker 2017-06-28 20:34:47 +01:00
parent 0a3d46f6da
commit 1f963ff4e4
1 changed files with 29 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
scripts/global_update.sh
View File

@ -5,25 +5,48 @@
ARCHIVE=${1:-/srv/Shared/BrowserExtensions/}
CRAWLERHOME=${2:-~/ExtensionCrawler}
date +'* Start Updating Code Repository (%c)'
# Update git repro
(cd $CRAWLERHOME; ((git fetch ; git checkout production; git pull) &> /dev/null))
date +'* Start Updating Extensions Archive (%c)'
# Update extensions
(cd $CRAWLERHOME; (./crawler -d -a $ARCHIVE > $ARCHIVE/log/`date --iso-8601=ns`.log))
date +'* Start Creating aa-ac.sqlite Data Base (%c)'
# Update small database
#rm -f $ARCHIVE/db/aa-ac.sqlite
(cd $CRAWLERHOME; (./scripts/generate_small_db.sh $ARCHIVE/data $ARCHIVE/db/aa-ac.sqlite > $ARCHIVE/log/`date --iso-8601=ns`-sqlite-aa-ac.log))
rm -f $ARCHIVE/db/aa-ac.sqlite
(cd $CRAWLERHOME; (./scripts/generate_small_db.sh $ARCHIVE/data $ARCHIVE/db/aa-ac.sqlite &> $ARCHIVE/log/`date --iso-8601=ns`-sqlite-aa-ac.log))
if [ $? -ne "0" ]; then
echo " Creation of aa-ac.sqlite failed - see log file for details"
else
echo " Created aa-ac.sqlite successfully"
fi
date +'* Start Creating full.sqlite Data Base (%c)'
# Update full database
rm -f $ARCHIVE/db/full.sqlite
(FIRSTDB=$(find "$ARCHIVE"/data/aa* -name "*.sqlite" | head -n 1);
sqlite3 "$FIRSTDB" .schema | sqlite3 "$ARCHIVE"/db/full.sqlite;
echo "Used $FIRSTDB for schema";
find "$ARCHIVE"/data/ -name "*.sqlite" -exec "$CRAWLERHOME/scripts/merge_dbs.sh" "{}" "$ARCHIVE"/db/full.sqlite \; ;
) > $ARCHIVE/log/`date --iso-8601=ns`-sqlite-full.log
if [ -f "$ARCHIVE"/db/full.sqlite ]; then
bzip2 "$ARCHIVE"/db/full.sqlite
) &> $ARCHIVE/log/`date --iso-8601=ns`-sqlite-full.log
if [ $? -ne "0" ]; then
echo " Creation of full.sqlite failed - see log file for details"
else
echo " Created full.sqlite successfully"
fi
if [ -f "$ARCHIVE"/db/full.sqlite ]; then
date +'* Start Compressing full.sqlite Data Base (%c)'
bzip2 "$ARCHIVE"/db/full.sqlite
if [ $? -ne "0" ]; then
echo " Creation of full.sqlite.bz2 failed"
else
echo " Created full.sqlite.bz2 successfully"
fi
fi
date +'* Update Finished (%c)'