acb14jd
/
ExtensionCrawler
forked from BrowserSecurity/ExtensionCrawler
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fixed set -u.

This commit is contained in:
Michael Herzberg 2017-07-11 11:15:43 +01:00
parent 6d03e8da71
commit 885f580f2e
3 changed files with 6 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
scripts/grepper256.sge
View File

@ -11,8 +11,6 @@
# qsub grepper256.sge DBPATH EXTENSIONCRAWLERPATH
# OUTPATH out path
# EXTENSIONCRAWLERPATH ExtensionCrawler git repo
set -u
OUTPATH="$1"
EXTENSIONCRAWLERDIR="$2"
@ -21,4 +19,6 @@ source $EXTENSIONCRAWLERDIR/scripts/sge_common
load_python
override_vars
set -u
python "$EXTENSIONCRAWLERDIR/grepper" -a "$(get_last_snapshot_dir)" -p "$(task_id_to_letter_256 $SGE_TASK_ID)" "$OUTPATH" "amazon" "&tag="

4
scripts/merge_dbs.sge
View File

@ -7,8 +7,6 @@
# DBDIR base dir for the generated db files
# OUTDBPATH the generated full db
# EXTENSIONCRAWLERPATH ExtensionCrawler git repo
set -u
DBDIR="$1"
OUTDBPATH="$2"
EXTENSIONCRAWLERDIR="$3"
@ -16,5 +14,7 @@ EXTENSIONCRAWLERDIR="$3"
source $EXTENSIONCRAWLERDIR/scripts/sge_common
override_vars
set -u
find $DBDIR -name "*.sqlite" -exec bash -c "sqlite3 {} .schema | sqlite3 $OUTDBPATH" \; -quit
find $DBDIR -name "*.sqlite" -exec "$EXTENSIONCRAWLERDIR/scripts/merge_dbs.sh" "{}" "$OUTDBPATH" \;

4
scripts/sqlite256.sge
View File

@ -11,8 +11,6 @@
# qsub sqlite256.sge DBPATH EXTENSIONCRAWLERPATH
# DBPATH base dir for the generated db files
# EXTENSIONCRAWLERPATH ExtensionCrawler git repo
set -u
DBPATH="$1"
EXTENSIONCRAWLERDIR="$2"
@ -21,4 +19,6 @@ source $EXTENSIONCRAWLERDIR/scripts/sge_common
load_python
override_vars
set -u
python "$EXTENSIONCRAWLERDIR/create-db" -a "$(get_last_snapshot_dir)" -p "$(task_id_to_letter_256 $SGE_TASK_ID)" "$DBPATH"