diff --git a/scripts/grepper256.sge b/scripts/grepper256.sge index 178f4d2..7844017 100755 --- a/scripts/grepper256.sge +++ b/scripts/grepper256.sge @@ -11,8 +11,6 @@ # qsub grepper256.sge DBPATH EXTENSIONCRAWLERPATH # OUTPATH out path # EXTENSIONCRAWLERPATH ExtensionCrawler git repo -set -u - OUTPATH="$1" EXTENSIONCRAWLERDIR="$2" @@ -21,4 +19,6 @@ source $EXTENSIONCRAWLERDIR/scripts/sge_common load_python override_vars +set -u + python "$EXTENSIONCRAWLERDIR/grepper" -a "$(get_last_snapshot_dir)" -p "$(task_id_to_letter_256 $SGE_TASK_ID)" "$OUTPATH" "amazon" "&tag=" diff --git a/scripts/merge_dbs.sge b/scripts/merge_dbs.sge index 662c423..2a24243 100755 --- a/scripts/merge_dbs.sge +++ b/scripts/merge_dbs.sge @@ -7,8 +7,6 @@ # DBDIR base dir for the generated db files # OUTDBPATH the generated full db # EXTENSIONCRAWLERPATH ExtensionCrawler git repo -set -u - DBDIR="$1" OUTDBPATH="$2" EXTENSIONCRAWLERDIR="$3" @@ -16,5 +14,7 @@ EXTENSIONCRAWLERDIR="$3" source $EXTENSIONCRAWLERDIR/scripts/sge_common override_vars +set -u + find $DBDIR -name "*.sqlite" -exec bash -c "sqlite3 {} .schema | sqlite3 $OUTDBPATH" \; -quit find $DBDIR -name "*.sqlite" -exec "$EXTENSIONCRAWLERDIR/scripts/merge_dbs.sh" "{}" "$OUTDBPATH" \; diff --git a/scripts/sqlite256.sge b/scripts/sqlite256.sge index c4897a5..12abfb8 100755 --- a/scripts/sqlite256.sge +++ b/scripts/sqlite256.sge @@ -11,8 +11,6 @@ # qsub sqlite256.sge DBPATH EXTENSIONCRAWLERPATH # DBPATH base dir for the generated db files # EXTENSIONCRAWLERPATH ExtensionCrawler git repo -set -u - DBPATH="$1" EXTENSIONCRAWLERDIR="$2" @@ -21,4 +19,6 @@ source $EXTENSIONCRAWLERDIR/scripts/sge_common load_python override_vars +set -u + python "$EXTENSIONCRAWLERDIR/create-db" -a "$(get_last_snapshot_dir)" -p "$(task_id_to_letter_256 $SGE_TASK_ID)" "$DBPATH"