2017-06-27 13:19:13 +00:00
|
|
|
#!/bin/bash
|
|
|
|
# m h dom mon dow command
|
2017-08-10 08:05:40 +00:00
|
|
|
# 15 01 * * * (cd ~/ExtensionCrawler; ((git fetch ; git checkout production; git pull) &> /dev/null))
|
2017-06-27 13:19:13 +00:00
|
|
|
# 07 02 * * * ~/ExtensionCrawler/scripts/global_update.sh
|
|
|
|
|
2017-09-08 20:10:40 +00:00
|
|
|
ARCHIVE=${1:-/srv/Shared/BrowserExtensions/archive}
|
2017-06-27 13:19:13 +00:00
|
|
|
CRAWLERHOME=${2:-~/ExtensionCrawler}
|
2017-06-29 18:45:21 +00:00
|
|
|
LOGPREFIX=$ARCHIVE/log/`date --iso-8601=ns`
|
2017-07-20 06:51:30 +00:00
|
|
|
LOG=$LOGPREFIX-global.log
|
2017-06-27 13:19:13 +00:00
|
|
|
|
2017-07-20 06:51:30 +00:00
|
|
|
date +'* Start Updating Extensions Archive (%c)' | tee $LOG
|
2017-06-28 19:34:47 +00:00
|
|
|
|
2017-06-27 13:19:13 +00:00
|
|
|
# Update extensions
|
2017-10-10 16:19:27 +00:00
|
|
|
(cd $CRAWLERHOME; (./crawler -p 32 -d -a $ARCHIVE > $LOGPREFIX.log))
|
2017-06-27 13:19:13 +00:00
|
|
|
|
2017-07-24 09:26:03 +00:00
|
|
|
date +'* Update Finished (%c)' | tee -a $LOG
|
2017-06-29 19:22:13 +00:00
|
|
|
|
2017-10-09 10:50:11 +00:00
|
|
|
ERRORS=`$LOGPREFIX.log | sort -k 5,5 -u | wc -l`
|
2017-10-09 07:45:06 +00:00
|
|
|
EXTENSIONS=`grep "Updating db" $LOGPREFIX.log | wc -l`
|
|
|
|
echo "ERROR LOG: $ERRORS (out of $EXTENSIONS)"
|
2017-10-07 18:31:43 +00:00
|
|
|
echo "=========="
|
2017-10-09 10:50:11 +00:00
|
|
|
grep ERROR $LOGPREFIX.log | sort -k 5,5 -u | sort -k 3,3
|
2017-10-07 18:31:43 +00:00
|
|
|
|