Move logs and out to separate dirs.
This commit is contained in:
parent
83c653dad9
commit
93daae78a6
|
@ -5,4 +5,4 @@ source activate mypython35
|
||||||
|
|
||||||
set -o nounset
|
set -o nounset
|
||||||
|
|
||||||
"$BASEDIR/ExtensionCrawler/grepper" -a "/shared/brucker_research1/Shared/BrowserExtensions" -n $SGE_TASK_ID -N $SGE_TASK_LAST "$PATTERN" | bzip2 > "$BASEDIR/$SGE_TASK_ID.bz2"
|
"$BASEDIR/ExtensionCrawler/grepper" -a "/shared/brucker_research1/Shared/BrowserExtensions" -n $SGE_TASK_ID -N $SGE_TASK_LAST "$PATTERN" | bzip2 > "$BASEDIR/out/$SGE_TASK_ID.bz2"
|
||||||
|
|
|
@ -8,8 +8,10 @@ TARGETDIR='/data/$USER/grepper-'$(date +%Y%m%d-%H%M%S)
|
||||||
|
|
||||||
SGEFILE="$BASEDIR/sge/grepper.sge"
|
SGEFILE="$BASEDIR/sge/grepper.sge"
|
||||||
|
|
||||||
echo "Creating $HOST:$TARGETDIR/ExtensionCrawler ..."
|
echo "Creating dirs ..."
|
||||||
ssh "$HOST" mkdir -p $TARGETDIR/ExtensionCrawler
|
ssh "$HOST" mkdir -p $TARGETDIR/ExtensionCrawler
|
||||||
|
ssh "$HOST" mkdir -p $TARGETDIR/logs
|
||||||
|
ssh "$HOST" mkdir -p $TARGETDIR/out
|
||||||
|
|
||||||
echo "Pushing $BASEDIR to $HOST:$TARGETDIR/ExtensionCrawler ..."
|
echo "Pushing $BASEDIR to $HOST:$TARGETDIR/ExtensionCrawler ..."
|
||||||
rsync -zr "$BASEDIR" $HOST:"$TARGETDIR/ExtensionCrawler"
|
rsync -zr "$BASEDIR" $HOST:"$TARGETDIR/ExtensionCrawler"
|
||||||
|
@ -22,5 +24,5 @@ ssh "$HOST" qsub \
|
||||||
-v BASEDIR="$TARGETDIR",PATTERN="$PATTERN" \
|
-v BASEDIR="$TARGETDIR",PATTERN="$PATTERN" \
|
||||||
-t 1-256 \
|
-t 1-256 \
|
||||||
-j yes \
|
-j yes \
|
||||||
-o "$TARGETDIR" \
|
-o "$TARGETDIR/logs" \
|
||||||
"$TARGETDIR/grepper.sge"
|
"$TARGETDIR/grepper.sge"
|
||||||
|
|
Loading…
Reference in New Issue