Merge branch 'master' of logicalhacking.com:BrowserSecurity/ExtensionCrawler

scripts/hpc-utilities/hpc-submit

@@ -0,0 +1,189 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+
+print_help()
+{
+    echo "Usage: $prog [OPTION] ... -- COMMAND ... %INPUT% ..."
+    echo ""
+    echo "Run ..."
+    echo ""
+    echo "  --help, -h              display this help message"
+    echo "  --jobs, -j  num         number of jobs (default: $jobs)"
+    echo "  --input, -i file        file with input data"
+    echo "  --prefix, -p prefix     prefix path for job directory (default: $prefix)"
+    echo "  --jobname, -n name      job name (default: $name)"
+    echo "  --wrapper, -w wrapper   exec wrapper (default: $wrapper)"
+    echo "  --copy-from, -f         copy command from direcotry (default: src)" 
+    echo "  --max-memory, -m mem    max mem (default: $mem)"
+    echo "  --max-time, -t          timelimit (default: $timelimit)"
+    echo "  --host, -s              remote host (default: $host)"
+    echo "  --srcdir, -d            src for copying binary (default: $srcdir)"
+    echo "                          assumed to be remote, if it starts with a \":\""
+    echo ""
+    echo " COMMAND is the command that should be executed on the HPC cluster, where"
+    echo " %INPUT% will be replaced with a file containing the job-specific input data."
+}
+
+
+
+mk_jobdir(){
+    echo "Creating temporary job directory in $workdir."
+    mkdir -p "$workdir"/bin
+    mkdir -p "$workdir"/cfg
+    mkdir -p "$workdir"/input
+    mkdir -p "$workdir"/output
+    mkdir -p "$workdir"/tmp
+}
+
+
+clean_jobdir(){
+   rm -rf "$workdir"
+}
+
+split_input(){
+    echo "Splitting input."
+    split -d -a 8 -e -n l/$jobs "$input" "$workdir/input/"
+}
+
+mk_hpc_script(){
+    local HOSTNAME=`hostname -f`
+    echo "Creating HPC script."
+    cat <<EOF > $workdir/job.sge
+
+#!/bin/bash
+## This script was generated by $prog (version: $version) 
+## on $timestamp 
+## by $USER@$HOSTNAME 
+## in $PWD 
+## using the following command:
+##   $invokation
+## 
+## SGE configuration:
+#$ -V
+#$ -t 1-$jobs
+#$ -l rmem=$mem
+#$ -l h_rt=$timelimit
+#$ -j yes
+#$ -o "$prefix"/"$name"/output
+
+export JOBINPUT="$prefix"/"$name"/input/\`printf %08d \$SGE_TASK_ID\`
+
+set -o nounset
+set -x
+
+/usr/bin/time -v $wrapper "$prefix"/"$name"/"bin"/$cmd
+echo "Execution successful."
+EOF
+}
+
+
+mk_remote_jobdir(){
+    echo "Create remote working directory ($host:$prefix)."
+    ssh $host mkdir -p $prefix
+}
+
+install_hpc_script(){
+    echo "Installing HPC Script"
+    scp -q -r "$workdir" "$host":"$prefix"/"$name"
+
+    if [[ $srcdir == ":"* ]]; then
+        echo "  Copying cmd from remote src."
+        ssh $host cp "${srcdir:1}"/"$srccmd" "$prefix"/"$name"/bin;
+    else
+        echo "  Copying cmd from local src."
+        scp $srccmd "$srcdir"/"$srccmd" "$host":"$prefix"/"$name"/bin;
+    fi
+}
+
+submit_job(){
+    echo "Submitting job."
+    ssh $host qsub "$prefix"/"$name"/job.sge
+}
+
+## global configuration
+version="0.0"
+prog=`echo $0 | sed 's|.*/||'`;
+invokation="$prog $(printf "%q " "$@")"
+timestamp=`date --utc --iso-8601=ns | sed -e 's/:/_/g' -e 's/,/./'`
+host=`hostname`
+workdir=`mktemp -d`
+echo $workdir
+
+## default values
+prefix="~/hpc/"
+jobs=1
+name="$host-$USER-$timestamp"
+input=""
+wrapper="singularity exec -B \$TMPDIR:$prefix/$name/tmp"
+cmd=""
+mem="2G"
+timelimit="01:00:00"
+local="false";
+host="sharc.shef.ac.uk"
+srcdir="."
+
+while [ $# -gt 0 ]
+do
+    case "$1" in
+	--jobs|-j)
+	    jobs="$2";
+            shift;;
+	--input|-i)
+	    input="$2";
+            shift;;
+	--jobname|-n)
+	    name="$2";
+            shift;;
+        --max-memory|-m)
+	    mem="$2";
+            shift;;
+        --max-time|-t)
+	    timelimit="$2";
+            shift;;
+        --host|-s)
+	    host="$2";
+            shift;;
+        --srcdir|-d)
+	    srcdir="$2";
+            shift;;
+        --wrapper|-w)
+	    wrapper="$2";
+            shift;;
+       --prefix|-p)
+	    prefix="$2";
+            shift;;
+	--help|-h)
+	    print_help
+	    exit 0;;
+        --) shift; break;;
+	*)  print_help
+            exit 1;;
+    esac
+    shift
+done
+cmd=`echo $(printf "%q " "$@") | sed -e 's/%INPUT%/\$JOBINPUT/'`
+cmdarray=("$@")
+srccmd=${cmdarray[0]}
+
+mk_jobdir;
+
+if [ -n "$input" ]; then
+    if [ ! -f "$input" ]; then
+        echo "Input file \"$input\" not found!"
+        exit 1
+    fi
+    split_input;
+fi
+
+mk_hpc_script;
+
+mk_remote_jobdir;
+
+install_hpc_script;
+
+clean_jobdir;
+
+submit_job;
+