Fixed sge scripts.

sge/create-db.sge

@@ -8,4 +8,6 @@ export LD_LIBRARY_PATH=~/lib:$LD_LIBRARY_PATH
 
 set -o nounset
 
+printenv
+
 "$BASEDIR/ExtensionCrawler/create-db" -a "$ARCHIVE" -n $SGE_TASK_ID -N $SGE_TASK_LAST "$BASEDIR/out"

sge/create-db.sh

@@ -1,23 +1,30 @@
 #!/usr/bin/bash
 set -o nounset
+set -o errexit
+
+ARCHIVE=${1:-}
+if [ -z "$ARCHIVE" ]; then
+  ARCHIVE=$(ssh sharc.shef.ac.uk find /shared/brucker_research1/Shared/BrowserExtensions/.snapshot -maxdepth 1 -name \"D*\" | sort -r | head -n1)
+fi
+echo "Using archive $ARCHIVE"
 
-HOST=${1:-sharc.shef.ac.uk}
 BASEDIR=$( cd $(dirname "$0"); cd ..; pwd -P )
 TARGETDIR='/data/$USER/create-db-'$(date +%Y%m%d-%H%M%S)
 
 echo "Creating dirs ..."
-ssh "$HOST" mkdir -p $TARGETDIR/ExtensionCrawler
-ssh "$HOST" mkdir -p $TARGETDIR/logs
-ssh "$HOST" mkdir -p $TARGETDIR/out
+ssh sharc.shef.ac.uk mkdir -p $TARGETDIR/ExtensionCrawler
+ssh sharc.shef.ac.uk mkdir -p $TARGETDIR/logs
+ssh sharc.shef.ac.uk mkdir -p $TARGETDIR/out
 
-echo "Pushing $BASEDIR to $HOST:$TARGETDIR/ExtensionCrawler ..."
-rsync -zr "$BASEDIR/" $HOST:"$TARGETDIR/ExtensionCrawler"
+echo "Pushing $BASEDIR to sharc.shef.ac.uk:$TARGETDIR/ExtensionCrawler ..."
+rsync -zr "$BASEDIR/" sharc.shef.ac.uk:"$TARGETDIR/ExtensionCrawler"
 
 echo "Starting job ..."
-LAST_SNAPSHOT=$(ssh "$HOST" find /shared/brucker_research1/Shared/BrowserExtensions/.snapshot -maxdepth 1 -name \"D*\" | sort -r | head -n1)
-
-ssh "$HOST" qsub \
-  -v BASEDIR="$TARGETDIR",ARCHIVE=\'"$LAST_SNAPSHOT"\' \
+ssh sharc.shef.ac.uk \
+  ARCHIVE=\"$ARCHIVE\" \
+  BASEDIR=\"$TARGETDIR\" \
+  qsub \
+  -V \
   -t 1-256 \
   -j yes \
   -o "$TARGETDIR/logs" \

sge/grepper.sge

@@ -5,4 +5,6 @@ source activate mypython35
 
 set -o nounset
 
+printenv
+
 "$BASEDIR/ExtensionCrawler/grepper" -a "/shared/brucker_research1/Shared/BrowserExtensions" -n $SGE_TASK_ID -N $SGE_TASK_LAST "$PATTERN" | bzip2 > "$BASEDIR/out/$SGE_TASK_ID.bz2"

sge/grepper.sh

@@ -1,22 +1,25 @@
 #!/usr/bin/bash
 set -o nounset
+set -o errexit
 
 PATTERN=$1
-HOST=${2:-sharc.shef.ac.uk}
 BASEDIR=$( cd $(dirname "$0"); cd ..; pwd -P )
 TARGETDIR='/data/$USER/grepper-'$(date +%Y%m%d-%H%M%S)
 
 echo "Creating dirs ..."
-ssh "$HOST" mkdir -p $TARGETDIR/ExtensionCrawler
-ssh "$HOST" mkdir -p $TARGETDIR/logs
-ssh "$HOST" mkdir -p $TARGETDIR/out
+ssh sharc.shef.ac.uk mkdir -p $TARGETDIR/ExtensionCrawler
+ssh sharc.shef.ac.uk mkdir -p $TARGETDIR/logs
+ssh sharc.shef.ac.uk mkdir -p $TARGETDIR/out
 
-echo "Pushing $BASEDIR to $HOST:$TARGETDIR/ExtensionCrawler ..."
-rsync -zr "$BASEDIR/" $HOST:"$TARGETDIR/ExtensionCrawler"
+echo "Pushing $BASEDIR to sharc.shef.ac.uk:$TARGETDIR/ExtensionCrawler ..."
+rsync -zr "$BASEDIR/" sharc.shef.ac.uk:"$TARGETDIR/ExtensionCrawler"
 
 echo "Starting job ..."
-ssh "$HOST" qsub \
-  -v BASEDIR="$TARGETDIR",PATTERN="$PATTERN" \
+ssh sharc.shef.ac.uk \
+  BASEDIR=\"$TARGETDIR\" \
+  PATTERN=\"$PATTERN\" \
+  qsub \
+  -V \
   -t 1-256 \
   -j yes \
   -o "$TARGETDIR/logs" \