Added extension id file support and increased memory for create-db job to 4G.

sge/create-db.sge

@@ -1,5 +1,6 @@
 #!/bin/bash
 set -o nounset
+set -x
 
 SING_EXEC="singularity exec --pwd /opt/ExtensionCrawler -B $TMPDIR:/tmp $SING_IMG"
 
@@ -8,4 +9,11 @@ echo "The following parameter were passed: $*"
 echo "Printing the content of $ARCHIVE to force mounting:"
 ls "$ARCHIVE"
 
-/usr/bin/time $SING_EXEC ./create-db -t 1 -a "$ARCHIVE" -n $SGE_TASK_ID -N $MAX_SGE_TASK_ID $*
+if [ -f "$BASEDIR/ids" ]; then
+  EXT_SELECT="-e $BASEDIR/ids"
+else
+  EXT_SELECT=
+fi
+
+
+/usr/bin/time $SING_EXEC ./create-db -t 1 -a "$ARCHIVE" -n $SGE_TASK_ID -N $MAX_SGE_TASK_ID $EXT_SELECT $*

sge/create-db.sh

@@ -39,6 +39,11 @@ ssh sharc.shef.ac.uk cp "$SING_IMG_SRC" "$SING_IMG"
 echo "Pushing sge script ..."
 scp "$BASEDIR/sge/create-db.sge" sharc.shef.ac.uk:"$TARGETDIR/create-db.sge"
 
+if ! [ -z "${EXTIDLISTFILE:-}" ]; then
+  echo "Pushing list with extension ids ..."
+  scp "$EXTIDLISTFILE" sharc.shef.ac.uk:"$TARGETDIR/ids"
+fi
+
 echo "Starting job ..."
 ssh sharc.shef.ac.uk \
   SING_IMG=\"$SING_IMG\" \
@@ -48,6 +53,7 @@ ssh sharc.shef.ac.uk \
   qsub \
   -V \
   -m a \
+  -l rmem=4G
   -M "msherzberg1@sheffield.ac.uk" \
   -t $JOBRANGE \
   -j yes \