diff --git a/sge/create-db-cdnjs.sge b/sge/create-db-cdnjs.sge new file mode 100755 index 0000000..cf991c1 --- /dev/null +++ b/sge/create-db-cdnjs.sge @@ -0,0 +1,19 @@ +#!/bin/bash +#$ -V +#$ -l rmem=4G +#$ -t 1-10000 +#$ -j yes +#$ -o /shared/brucker_research1/Shared/BrowserExtensions/archive/filedb/log +set -o nounset +set -x + +SING_IMG=/shared/brucker_research1/Shared/BrowserExtensions/archive/filedb/ExtensionCrawler-cdnjs.img + +SING_EXEC="singularity exec -w --pwd /opt/ExtensionCrawler -B $TMPDIR:/tmp $SING_IMG" + +printenv +echo "The following parameter were passed: $*" +ls "$SING_IMG" > /dev/null + +/usr/bin/time $SING_EXEC ./cdnjs-git-miner -v -p 1 -i -a /opt/archive -n $SGE_TASK_ID -N 10000 $* +