From c33e8204ea08157fcb0c05674eb4fb78ba3b0098 Mon Sep 17 00:00:00 2001
From: Michael Herzberg <mail@michael-herzberg.de>
Date: Sat, 2 Sep 2017 17:05:42 +0100
Subject: [PATCH] Cleaned up create-db sge script a bit.

---
 sge/create-db.sge |  2 +-
 sge/create-db.sh  | 19 +++++++++++++------
 2 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/sge/create-db.sge b/sge/create-db.sge
index 3d86849..93b6651 100755
--- a/sge/create-db.sge
+++ b/sge/create-db.sge
@@ -10,4 +10,4 @@ set -o nounset
 
 printenv
 
-"$BASEDIR/ExtensionCrawler/create-db" -t 1 -a "$ARCHIVE" -n $SGE_TASK_ID -N $MAX_SGE_TASK_ID "$BASEDIR/out"
+"$BASEDIR/ExtensionCrawler/create-db" -t 1 -a "$ARCHIVE" -n $SGE_TASK_ID -N $MAX_SGE_TASK_ID $EXTRAARGS
diff --git a/sge/create-db.sh b/sge/create-db.sh
index fcc1d04..1679b33 100755
--- a/sge/create-db.sh
+++ b/sge/create-db.sh
@@ -2,16 +2,22 @@
 set -o nounset
 set -o errexit
 
-ARCHIVE=${1:-$(ssh sharc.shef.ac.uk find /shared/brucker_research1/Shared/BrowserExtensions/.snapshot -maxdepth 1 -name \"D*\" | sort -r | head -n1)}
-echo "Using archive $ARCHIVE"
+EXTRAARGS=${EXTRAARGS:-}
+echo "Using extra args: $EXTRAARGS"
 
-TARGETDIR="${2:-/data/\$USER}/create-db-$(date +%Y%m%d-%H%M%S)"
+NRJOBS=${NRJOBS:-256}
+echo "Using $NRJOBS jobs"
+
+ARCHIVE=${ARCHIVE:-$(ssh sharc.shef.ac.uk find /shared/brucker_research1/Shared/BrowserExtensions/.snapshot -maxdepth 1 -name \"D*\" | sort -r | head -n1)}
+echo "Using archive: $ARCHIVE"
+
+TARGETDIR="${TARGETDIR:-/data/\$USER}/create-db-$(date +%Y%m%d-%H%M%S)"
+echo "Using target dir: $TARGETDIR"
 BASEDIR=$( cd $(dirname "$0"); cd ..; pwd -P )
 
 echo "Creating dirs ..."
 ssh sharc.shef.ac.uk mkdir -p $TARGETDIR/ExtensionCrawler
 ssh sharc.shef.ac.uk mkdir -p $TARGETDIR/logs
-ssh sharc.shef.ac.uk mkdir -p $TARGETDIR/out
 
 echo "Pushing $BASEDIR to sharc.shef.ac.uk:$TARGETDIR/ExtensionCrawler ..."
 rsync -zr --exclude "$BASEDIR/archive" "$BASEDIR/" sharc.shef.ac.uk:"$TARGETDIR/ExtensionCrawler"
@@ -20,10 +26,11 @@ echo "Starting job ..."
 ssh sharc.shef.ac.uk \
   ARCHIVE=\"$ARCHIVE\" \
   BASEDIR=\"$TARGETDIR\" \
-  MAX_SGE_TASK_ID=256 \
+  EXTRAARGS=\"$EXTRAARGS\" \
+  MAX_SGE_TASK_ID=\"$NRJOBS\" \
   qsub \
   -V \
-  -t 1-256 \
+  -t 1-$NRJOBS \
   -j yes \
   -o "$TARGETDIR/logs" \
   "$TARGETDIR/ExtensionCrawler/sge/create-db.sge"