Improved sge script.
This commit is contained in:
parent
9950984e27
commit
a81c53f9ca
|
@ -10,4 +10,4 @@ set -o nounset
|
||||||
|
|
||||||
printenv
|
printenv
|
||||||
|
|
||||||
"$BASEDIR/ExtensionCrawler/create-db" -a "$ARCHIVE" -n $SGE_TASK_ID -N $SGE_TASK_LAST "$BASEDIR/out"
|
"$BASEDIR/ExtensionCrawler/create-db" -a "$ARCHIVE" -n $SGE_TASK_ID -N $MAX_SGE_TASK_ID "$BASEDIR/out"
|
||||||
|
|
|
@ -3,11 +3,9 @@ set -o nounset
|
||||||
set -o errexit
|
set -o errexit
|
||||||
|
|
||||||
ARCHIVE=${1:-$(ssh sharc.shef.ac.uk find /shared/brucker_research1/Shared/BrowserExtensions/.snapshot -maxdepth 1 -name \"D*\" | sort -r | head -n1)}
|
ARCHIVE=${1:-$(ssh sharc.shef.ac.uk find /shared/brucker_research1/Shared/BrowserExtensions/.snapshot -maxdepth 1 -name \"D*\" | sort -r | head -n1)}
|
||||||
TARGETBASEDIR=${2:-'/data/$USER'}
|
|
||||||
|
|
||||||
echo "Using archive $ARCHIVE"
|
echo "Using archive $ARCHIVE"
|
||||||
|
|
||||||
TARGETDIR="${TARGETBASEDIR}/create-db-$(date +%Y%m%d-%H%M%S)"
|
TARGETDIR="${2:-/data/\$USER}/create-db-$(date +%Y%m%d-%H%M%S)"
|
||||||
BASEDIR=$( cd $(dirname "$0"); cd ..; pwd -P )
|
BASEDIR=$( cd $(dirname "$0"); cd ..; pwd -P )
|
||||||
|
|
||||||
echo "Creating dirs ..."
|
echo "Creating dirs ..."
|
||||||
|
@ -22,6 +20,7 @@ echo "Starting job ..."
|
||||||
ssh sharc.shef.ac.uk \
|
ssh sharc.shef.ac.uk \
|
||||||
ARCHIVE=\"$ARCHIVE\" \
|
ARCHIVE=\"$ARCHIVE\" \
|
||||||
BASEDIR=\"$TARGETDIR\" \
|
BASEDIR=\"$TARGETDIR\" \
|
||||||
|
MAX_SGE_TASK_ID=256 \
|
||||||
qsub \
|
qsub \
|
||||||
-V \
|
-V \
|
||||||
-t 1-256 \
|
-t 1-256 \
|
||||||
|
|
|
@ -7,4 +7,4 @@ set -o nounset
|
||||||
|
|
||||||
printenv
|
printenv
|
||||||
|
|
||||||
"$BASEDIR/ExtensionCrawler/grepper" -a "/shared/brucker_research1/Shared/BrowserExtensions" -n $SGE_TASK_ID -N $SGE_TASK_LAST "$PATTERN" | bzip2 > "$BASEDIR/out/$SGE_TASK_ID.bz2"
|
"$BASEDIR/ExtensionCrawler/grepper" -a "$ARCHIVE" -n $SGE_TASK_ID -N $MAX_SGE_TASK_ID "$PATTERN" | bzip2 > "$BASEDIR/out/$SGE_TASK_ID.bz2"
|
||||||
|
|
|
@ -3,8 +3,12 @@ set -o nounset
|
||||||
set -o errexit
|
set -o errexit
|
||||||
|
|
||||||
PATTERN=$1
|
PATTERN=$1
|
||||||
|
|
||||||
|
ARCHIVE=${2:-$(ssh sharc.shef.ac.uk find /shared/brucker_research1/Shared/BrowserExtensions/.snapshot -maxdepth 1 -name \"D*\" | sort -r | head -n1)}
|
||||||
|
echo "Using archive $ARCHIVE"
|
||||||
|
|
||||||
|
TARGETDIR="${3:-/data/\$USER}/grepper-$(date +%Y%m%d-%H%M%S)"
|
||||||
BASEDIR=$( cd $(dirname "$0"); cd ..; pwd -P )
|
BASEDIR=$( cd $(dirname "$0"); cd ..; pwd -P )
|
||||||
TARGETDIR='/data/$USER/grepper-'$(date +%Y%m%d-%H%M%S)
|
|
||||||
|
|
||||||
echo "Creating dirs ..."
|
echo "Creating dirs ..."
|
||||||
ssh sharc.shef.ac.uk mkdir -p $TARGETDIR/ExtensionCrawler
|
ssh sharc.shef.ac.uk mkdir -p $TARGETDIR/ExtensionCrawler
|
||||||
|
@ -16,8 +20,10 @@ rsync -zr "$BASEDIR/" sharc.shef.ac.uk:"$TARGETDIR/ExtensionCrawler"
|
||||||
|
|
||||||
echo "Starting job ..."
|
echo "Starting job ..."
|
||||||
ssh sharc.shef.ac.uk \
|
ssh sharc.shef.ac.uk \
|
||||||
|
ARCHIVE=\"$ARCHIVE\" \
|
||||||
BASEDIR=\"$TARGETDIR\" \
|
BASEDIR=\"$TARGETDIR\" \
|
||||||
PATTERN=\"$PATTERN\" \
|
PATTERN=\"$PATTERN\" \
|
||||||
|
MAX_SGE_TASK_ID=256 \
|
||||||
qsub \
|
qsub \
|
||||||
-V \
|
-V \
|
||||||
-t 1-256 \
|
-t 1-256 \
|
||||||
|
|
Loading…
Reference in New Issue