From a81c53f9ca1670e8eedd57b296e1c8b13ac6671f Mon Sep 17 00:00:00 2001
From: Michael Herzberg <mail@michael-herzberg.de>
Date: Thu, 24 Aug 2017 08:56:02 +0100
Subject: [PATCH] Improved sge script.

---
 sge/create-db.sge | 2 +-
 sge/create-db.sh  | 5 ++---
 sge/grepper.sge   | 2 +-
 sge/grepper.sh    | 8 +++++++-
 4 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/sge/create-db.sge b/sge/create-db.sge
index 9f35615..1ccaff8 100755
--- a/sge/create-db.sge
+++ b/sge/create-db.sge
@@ -10,4 +10,4 @@ set -o nounset
 
 printenv
 
-"$BASEDIR/ExtensionCrawler/create-db" -a "$ARCHIVE" -n $SGE_TASK_ID -N $SGE_TASK_LAST "$BASEDIR/out"
+"$BASEDIR/ExtensionCrawler/create-db" -a "$ARCHIVE" -n $SGE_TASK_ID -N $MAX_SGE_TASK_ID "$BASEDIR/out"
diff --git a/sge/create-db.sh b/sge/create-db.sh
index 3496aad..d739bc5 100755
--- a/sge/create-db.sh
+++ b/sge/create-db.sh
@@ -3,11 +3,9 @@ set -o nounset
 set -o errexit
 
 ARCHIVE=${1:-$(ssh sharc.shef.ac.uk find /shared/brucker_research1/Shared/BrowserExtensions/.snapshot -maxdepth 1 -name \"D*\" | sort -r | head -n1)}
-TARGETBASEDIR=${2:-'/data/$USER'}
-
 echo "Using archive $ARCHIVE"
 
-TARGETDIR="${TARGETBASEDIR}/create-db-$(date +%Y%m%d-%H%M%S)"
+TARGETDIR="${2:-/data/\$USER}/create-db-$(date +%Y%m%d-%H%M%S)"
 BASEDIR=$( cd $(dirname "$0"); cd ..; pwd -P )
 
 echo "Creating dirs ..."
@@ -22,6 +20,7 @@ echo "Starting job ..."
 ssh sharc.shef.ac.uk \
   ARCHIVE=\"$ARCHIVE\" \
   BASEDIR=\"$TARGETDIR\" \
+  MAX_SGE_TASK_ID=256 \
   qsub \
   -V \
   -t 1-256 \
diff --git a/sge/grepper.sge b/sge/grepper.sge
index ec4da6e..9344b08 100755
--- a/sge/grepper.sge
+++ b/sge/grepper.sge
@@ -7,4 +7,4 @@ set -o nounset
 
 printenv
 
-"$BASEDIR/ExtensionCrawler/grepper" -a "/shared/brucker_research1/Shared/BrowserExtensions" -n $SGE_TASK_ID -N $SGE_TASK_LAST "$PATTERN" | bzip2 > "$BASEDIR/out/$SGE_TASK_ID.bz2"
+"$BASEDIR/ExtensionCrawler/grepper" -a "$ARCHIVE" -n $SGE_TASK_ID -N $MAX_SGE_TASK_ID "$PATTERN" | bzip2 > "$BASEDIR/out/$SGE_TASK_ID.bz2"
diff --git a/sge/grepper.sh b/sge/grepper.sh
index 1f0845a..5284718 100755
--- a/sge/grepper.sh
+++ b/sge/grepper.sh
@@ -3,8 +3,12 @@ set -o nounset
 set -o errexit
 
 PATTERN=$1
+
+ARCHIVE=${2:-$(ssh sharc.shef.ac.uk find /shared/brucker_research1/Shared/BrowserExtensions/.snapshot -maxdepth 1 -name \"D*\" | sort -r | head -n1)}
+echo "Using archive $ARCHIVE"
+
+TARGETDIR="${3:-/data/\$USER}/grepper-$(date +%Y%m%d-%H%M%S)"
 BASEDIR=$( cd $(dirname "$0"); cd ..; pwd -P )
-TARGETDIR='/data/$USER/grepper-'$(date +%Y%m%d-%H%M%S)
 
 echo "Creating dirs ..."
 ssh sharc.shef.ac.uk mkdir -p $TARGETDIR/ExtensionCrawler
@@ -16,8 +20,10 @@ rsync -zr "$BASEDIR/" sharc.shef.ac.uk:"$TARGETDIR/ExtensionCrawler"
 
 echo "Starting job ..."
 ssh sharc.shef.ac.uk \
+  ARCHIVE=\"$ARCHIVE\" \
   BASEDIR=\"$TARGETDIR\" \
   PATTERN=\"$PATTERN\" \
+  MAX_SGE_TASK_ID=256 \
   qsub \
   -V \
   -t 1-256 \