Merge branch 'master' of logicalhacking.com:BrowserSecurity/ExtensionCrawler

This commit is contained in:
Achim D. Brucker 2019-05-23 09:01:29 +01:00
commit d8d2e41dc6
6 changed files with 55 additions and 549 deletions

3
.gitignore vendored
View File

@ -68,3 +68,6 @@ archive
.ropeproject
ExtensionCrawler.img
ExtensionCrawler-cdnjs.img
.idea
venv

View File

@ -31,6 +31,7 @@ import glob
import datetime
import hashlib
from jsmin import jsmin
import ast
def get_etag(ext_id, datepath, con):
@ -48,7 +49,7 @@ def get_etag(ext_id, datepath, con):
with open(headerpath) as f:
content = f.read()
try:
headers = eval(content)
headers = ast.literal_eval(content)
if "ETag" in headers:
return headers["ETag"]
except Exception:

View File

@ -25,6 +25,7 @@ import json
import sys
import csv
from jsmin import jsmin
import ast
from zipfile import ZipFile
@ -34,14 +35,12 @@ from ExtensionCrawler.js_mincer import mince_js
def get_etag(headers_content):
headers_content = headers_content.replace(
'"', '\\"').replace("'", '"')
headers_json = json.loads(headers_content)
if "ETag" in headers_json:
return headers_json["ETag"]
d = ast.literal_eval(headers_content)
if "ETag" in d:
return d["ETag"]
def get_name_and_version(overview_contents):
def get_metadata(overview_contents):
# Extract extension name
match = re.search("""<meta itemprop="name" content="(.*?)"\s*/>""",
overview_contents)
@ -51,8 +50,19 @@ def get_name_and_version(overview_contents):
match = re.search(
"""<meta itemprop="version" content="(.*?)"\s*/>""", overview_contents)
version = match.group(1) if match else None
# Extracts extension categories
match = re.search(
"""Attribute name="category">(.+?)</Attribute>""", overview_contents)
categories = match.group(1).split(",") if match else []
return name, version
# Extracts the number of downloads
match = re.search(
"""<meta itemprop="interactionCount" content="UserDownloads:((:?\d|,)+)""",
overview_contents)
downloads = int(match.group(1).replace(",", '')) if match else None
return name, version, categories, downloads
def handle_extid(conf, extid, permission_map, csvwriter):
@ -84,7 +94,7 @@ def handle_extid(conf, extid, permission_map, csvwriter):
crx_etags += [crx_etag]
if tarentry_filename == "overview.html":
name, version = get_name_and_version(tarfile.read().decode())
name, version, categories, downloads = get_metadata(tarfile.read().decode())
if tarentry_filename == "overview.html.status":
still_in_store = tarfile.read().decode().startswith("2")
@ -122,7 +132,7 @@ def handle_extid(conf, extid, permission_map, csvwriter):
used_permissions.add(str(permission))
if has_crx_file:
line = [date, crx_etag, name, version]
line = [date, crx_etag, name, version, "+".join(categories), downloads]
for permission in sorted(list(permission_map.keys())):
if permission in used_permissions:
if date_matches[permission]:
@ -155,10 +165,13 @@ def main(conf):
with open(conf.EXTID_FILE) as f:
csvwriter = csv.writer(sys.stdout, csv.unix_dialect)
csvwriter.writerow(["extid", "still_in_store", "most_recent_crx_etag", "date", "crx_etag", "name", "version"]
csvwriter.writerow(["extid", "still_in_store", "most_recent_crx_etag", "date", "crx_etag", "name", "version", "categories", "downloads"]
+ sorted(list(permission_map.keys())))
for extid in [l.strip() for l in f.readlines()]:
handle_extid(conf, extid, permission_map, csvwriter)
try:
handle_extid(conf, extid, permission_map, csvwriter)
except Exception as e:
logging.exception(f"Fatal error when handling extension '{extid}'")
def build_parser():

View File

@ -1,524 +0,0 @@
{
"mdns": [
"chrome.mdns.forceDiscovery"
],
"usb": [
"chrome.usb.getDevices",
"chrome.usb.getUserSelectedDevices",
"chrome.usb.getConfigurations",
"chrome.usb.requestAccess",
"chrome.usb.openDevice",
"chrome.usb.findDevices",
"chrome.usb.closeDevice",
"chrome.usb.setConfiguration",
"chrome.usb.getConfiguration",
"chrome.usb.listInterfaces",
"chrome.usb.claimInterface",
"chrome.usb.releaseInterface",
"chrome.usb.setInterfaceAlternateSetting",
"chrome.usb.controlTransfer",
"chrome.usb.bulkTransfer",
"chrome.usb.interruptTransfer",
"chrome.usb.isochronousTransfer",
"chrome.usb.resetDevice"
],
"certificateProvider": [
"chrome.certificateProvider.requestPin",
"chrome.certificateProvider.stopPinRequest"
],
"contentSettings": [
"chrome.contentSettings.cookies",
"chrome.contentSettings.images",
"chrome.contentSettings.javascript",
"chrome.contentSettings.location",
"chrome.contentSettings.plugins",
"chrome.contentSettings.popups",
"chrome.contentSettings.notifications",
"chrome.contentSettings.fullscreen",
"chrome.contentSettings.mouselock",
"chrome.contentSettings.microphone",
"chrome.contentSettings.camera",
"chrome.contentSettings.unsandboxedPlugins",
"chrome.contentSettings.automaticDownloads"
],
"idle": [
"chrome.idle.queryState",
"chrome.idle.setDetectionInterval",
"chrome.idle.getAutoLockDelay"
],
"proxy": [
"chrome.proxy"
],
"enterprise.deviceAttributes": [
"chrome.enterprise.deviceAttributes.getDirectoryDeviceId",
"chrome.enterprise.deviceAttributes.getDeviceSerialNumber",
"chrome.enterprise.deviceAttributes.getDeviceAssetId",
"chrome.enterprise.deviceAttributes.getDeviceAnnotatedLocation"
],
"printerProvider": [
"chrome.printerProvider.onGetPrintersRequested.addListener",
"chrome.printerProvider.onGetCapabilityRequested.addListener",
"chrome.printerProvider.onPrintRequested.addListener",
"chrome.printerProvider.onGetUsbPrinterInfoRequested.addListener"
],
"downloads": [
"chrome.downloads.download",
"chrome.downloads.search",
"chrome.downloads.pause",
"chrome.downloads.resume",
"chrome.downloads.cancel",
"chrome.downloads.getFileIcon",
"chrome.downloads.open",
"chrome.downloads.show",
"chrome.downloads.showDefaultFolder",
"chrome.downloads.erase",
"chrome.downloads.removeFile",
"chrome.downloads.acceptDanger",
"chrome.downloads.drag",
"chrome.downloads.setShelfEnabled"
],
"networking.config": [
"chrome.networking.config.setNetworkFilter",
"chrome.networking.config.finishAuthentication"
],
"mediaGalleries": [
"chrome.mediaGalleries.getMediaFileSystems",
"chrome.mediaGalleries.addUserSelectedFolder",
"chrome.mediaGalleries.dropPermissionForMediaFileSystem",
"chrome.mediaGalleries.startMediaScan",
"chrome.mediaGalleries.cancelMediaScan",
"chrome.mediaGalleries.addScanResults",
"chrome.mediaGalleries.getMediaFileSystemMetadata",
"chrome.mediaGalleries.getAllMediaFileSystemMetadata",
"chrome.mediaGalleries.getMetadata",
"chrome.mediaGalleries.addGalleryWatch",
"chrome.mediaGalleries.removeGalleryWatch",
"chrome.mediaGalleries.getAllGalleryWatch",
"chrome.mediaGalleries.removeAllGalleryWatch"
],
"system.memory": [
"chrome.system.memory.getInfo"
],
"history": [
"chrome.history.search",
"chrome.history.getVisits",
"chrome.history.addUrl",
"chrome.history.deleteUrl",
"chrome.history.deleteRange",
"chrome.history.deleteAll"
],
"ttsEngine": [
"chrome.ttsEngine.updateVoices"
],
"socket": [
"chrome.socket.create",
"chrome.socket.destroy",
"chrome.socket.connect",
"chrome.socket.bind",
"chrome.socket.disconnect",
"chrome.socket.read",
"chrome.socket.write",
"chrome.socket.recvFrom",
"chrome.socket.sendTo",
"chrome.socket.listen",
"chrome.socket.accept",
"chrome.socket.setKeepAlive",
"chrome.socket.setNoDelay",
"chrome.socket.getInfo",
"chrome.socket.getNetworkList",
"chrome.socket.joinGroup",
"chrome.socket.leaveGroup",
"chrome.socket.setMulticastTimeToLive",
"chrome.socket.setMulticastLoopbackMode",
"chrome.socket.getJoinedGroups",
"chrome.socket.secure"
],
"webview": [
"<webview>.getAudioState",
"<webview>.setAudioMuted",
"<webview>.isAudioMuted",
"<webview>.captureVisibleRegion",
"<webview>.addContentScripts",
"<webview>.back",
"<webview>.canGoBack",
"<webview>.canGoForward",
"<webview>.clearData",
"<webview>.executeScript",
"<webview>.find",
"<webview>.forward",
"<webview>.getProcessId",
"<webview>.getUserAgent",
"<webview>.getZoom",
"<webview>.getZoomMode",
"<webview>.go",
"<webview>.insertCSS",
"<webview>.isUserAgentOverridden",
"<webview>.print",
"<webview>.reload",
"<webview>.removeContentScripts",
"<webview>.setUserAgentOverride",
"<webview>.setZoom",
"<webview>.setZoomMode",
"<webview>.stop",
"<webview>.stopFinding",
"<webview>.loadDataWithBaseUrl",
"<webview>.setSpatialNavigationEnabled",
"<webview>.isSpatialNavigationEnabled",
"<webview>.terminate"
],
"system.network": [
"chrome.system.network.getNetworkInterfaces"
],
"hid": [
"chrome.hid.getDevices",
"chrome.hid.getUserSelectedDevices",
"chrome.hid.connect",
"chrome.hid.disconnect",
"chrome.hid.receive",
"chrome.hid.send",
"chrome.hid.receiveFeatureReport",
"chrome.hid.sendFeatureReport"
],
"enterprise.platformKeys": [
"chrome.enterprise.platformKeys.getTokens",
"chrome.enterprise.platformKeys.getCertificates",
"chrome.enterprise.platformKeys.importCertificate",
"chrome.enterprise.platformKeys.removeCertificate",
"chrome.enterprise.platformKeys.challengeMachineKey",
"chrome.enterprise.platformKeys.challengeUserKey"
],
"documentScan": [
"chrome.documentScan.scan"
],
"serial": [
"chrome.serial.getDevices",
"chrome.serial.connect",
"chrome.serial.update",
"chrome.serial.disconnect",
"chrome.serial.setPaused",
"chrome.serial.getInfo",
"chrome.serial.getConnections",
"chrome.serial.send",
"chrome.serial.flush",
"chrome.serial.getControlSignals",
" chrome.serial.setControlSignals",
"chrome.serial.setBreak",
"chrome.serial.clearBreak"
],
"pkcs11": [
"pkcs11.getModuleSlots",
"pkcs11.installModule",
"pkcs11.isModuleInstalled",
"pkcs11.uninstallModule"
],
"browsingData": [
"chrome.browsingData.settings",
"chrome.browsingData.remove",
"chrome.browsingData.removeAppcache",
"chrome.browsingData.removeCache",
"chrome.browsingData.removeCacheStorage",
"chrome.browsingData.removeCookies",
"chrome.browsingData.removeDownloads",
"chrome.browsingData.removeFileSystems",
"chrome.browsingData.removeFormData",
"chrome.browsingData.removeHistory",
"chrome.browsingData.removeIndexedDB",
"chrome.browsingData.removeLocalStorage",
"chrome.browsingData.removePluginData",
"chrome.browsingData.removePasswords",
"chrome.browsingData.removeServiceWorkers",
"chrome.browsingData.removeWebSQL"
],
"management": [
"chrome.management.getAll",
"chrome.management.get",
"chrome.management.getSelf",
"chrome.management.getPermissionWarningsById",
"chrome.management.getPermissionWarningsByManifest",
"chrome.management.setEnabled",
"chrome.management.uninstall",
"chrome.management.uninstallSelf",
"chrome.management.launchApp",
"chrome.management.createAppShortcut",
"chrome.management.setLaunchType",
"chrome.management.generateAppForLink"
],
"tabCapture": [
"chrome.tabCapture.capture",
"chrome.tabCapture.getCapturedTabs",
"chrome.tabCapture.captureOffscreenTab",
"chrome.tabCapture.getMediaStreamId"
],
"declarativeContent": [
"chrome.declarativeContent"
],
"privacy": [
"chrome.privacy"
],
"input": [
"chrome.input.ime.setComposition",
"chrome.input.ime.clearComposition",
"chrome.input.ime.commitText",
"chrome.input.ime.sendKeyEvents",
"chrome.input.ime.hideInputView",
"chrome.input.ime.setCandidateWindowProperties",
"chrome.input.ime.setCandidates",
"chrome.input.ime.setCursorPosition",
"chrome.input.ime.setMenuItems",
"chrome.input.ime.updateMenuItems",
"chrome.input.ime.deleteSurroundingText",
"chrome.input.ime.keyEventHandled",
"chrome.input.ime.createWindow",
"chrome.input.ime.showWindow",
"chrome.input.ime.hideWindow",
"chrome.input.ime.activate",
"chrome.input.ime.deactivate"
],
"fontSettings": [
"chrome.fontSettings.clearFont",
"chrome.fontSettings.getFont",
"chrome.fontSettings.setFont",
"chrome.fontSettings.getFontList",
"chrome.fontSettings.clearDefaultFontSize",
"chrome.fontSettings.getDefaultFontSize",
"chrome.fontSettings.setDefaultFontSize",
"chrome.fontSettings.clearDefaultFixedFontSize",
"chrome.fontSettings.getDefaultFixedFontSize",
"chrome.fontSettings.setDefaultFixedFontSize",
"chrome.fontSettings.clearMinimumFontSize",
"chrome.fontSettings.getMinimumFontSize",
"chrome.fontSettings.setMinimumFontSize"
],
"signedInDevices": [
"chrome.signedInDevices.get"
],
"clipboardRead": [
"document.execCommand(paste)"
],
"storage": [
"chrome.storage"
],
"vpnProvider": [
"chrome.vpnProvider.createConfig",
"chrome.vpnProvider.destroyConfig",
"chrome.vpnProvider.setParameters",
"chrome.vpnProvider.sendPacket",
"chrome.vpnProvider.notifyConnectionStateChanged"
],
"debugger": [
"chrome.debugger.attach",
"chrome.debugger.detach",
"chrome.debugger.sendCommand",
"chrome.debugger.getTargets"
],
"gcm": [
"chrome.gcm.register",
"chrome.gcm.unregister",
"chrome.gcm.send",
"chrome.instanceID.getID",
"chrome.instanceID.getCreationTime",
"chrome.instanceID.getToken",
"chrome.instanceID.deleteToken",
"chrome.instanceID.deleteID"
],
"webRequest": [
"chrome.webRequest.handlerBehaviorChanged",
"chrome.webRequest"
],
"tabs": [
"chrome.tabs.getCurrent",
"chrome.tabs.connect",
"chrome.tabs.query",
"chrome.tabs.sendMessage",
"chrome.tabs.get",
"chrome.tabs.sendRequest",
"chrome.tabs.getSelected",
"chrome.tabs.getAllInWindow",
"chrome.tabs.create",
"chrome.tabs.duplicate",
"chrome.tabs.highlight",
"chrome.tabs.update",
"chrome.tabs.move",
"chrome.tabs.reload",
"chrome.tabs.remove",
"chrome.tabs.detectLanguage",
"chrome.tabs.captureVisibleTab",
"chrome.tabs.executeScript",
"chrome.tabs.insertCSS",
"chrome.tabs.setZoom",
"chrome.tabs.getZoom",
"chrome.tabs.setZoomSettings",
"chrome.tabs.getZoomSettings",
"chrome.tabs.discard",
"chrome.tabs.goForward",
"chrome.tabs.goBack"
],
"syncFileSystem": [
"chrome.syncFileSystem.requestFileSystem",
"chrome.syncFileSystem.setConflictResolutionPolicy",
"hrome.syncFileSystem.getConflictResolutionPolicy",
"chrome.syncFileSystem.getUsageAndQuota",
"chrome.syncFileSystem.getFileStatus",
"chrome.syncFileSystem.getFileStatuses",
"chrome.syncFileSystem.getServiceStatus"
],
"virtualKeyboard": [
"chrome.virtualKeyboard.restrictFeatures"
],
"pageCapture": [
"chrome.pageCapture.saveAsMHTML"
],
"contextMenus": [
"chrome.contextMenus.create",
"chrome.contextMenus.update",
"chrome.contextMenus.remove",
"chrome.contextMenus.removeAll"
],
"cookies": [
"chrome.cookies.get",
"chrome.cookies.getAll",
"chrome.cookies.set",
"chrome.cookies.remove",
"chrome.cookies.getAllCookieStores"
],
"power": [
"chrome.power.requestKeepAwake",
"chrome.power.releaseKeepAwake"
],
"bookmarks": [
"chrome.bookmarks.get",
"chrome.bookmarks.getChildren",
"chrome.bookmarks.getRecent",
"chrome.bookmarks.getTree",
"chrome.bookmarks.getSubTree",
"chrome.bookmarks.search",
"chrome.bookmarks.create",
"chrome.bookmarks.move",
"chrome.bookmarks.update",
"chrome.bookmarks.remove",
"chrome.bookmarks.removeTree"
],
"fileSystemProvider": [
"chrome.fileSystemProvider.mount",
"chrome.fileSystemProvider.unmount",
"chrome.fileSystemProvider.getAll",
"chrome.fileSystemProvider.get",
"chrome.fileSystemProvider.notify"
],
"networking.onc": [
"chrome.networking.onc.getProperties",
"chrome.networking.onc.getManagedProperties",
"chrome.networking.onc.getState",
"chrome.networking.onc.setProperties",
"chrome.networking.onc.createNetwork",
"chrome.networking.onc.forgetNetwork",
"chrome.networking.onc.getNetworks",
"chrome.networking.onc.getDeviceStates",
"chrome.networking.onc.enableNetworkType",
"chrome.networking.onc.disableNetworkType",
"chrome.networking.onc.requestNetworkScan",
"chrome.networking.onc.startConnect",
"chrome.networking.onc.startDisconnect",
"chrome.networking.onc.getCaptivePortalStatus",
"chrome.networking.onc.getGlobalPolicy"
],
"fileBrowserHandler": [
"chrome.fileBrowserHandler.selectFile"
],
"webNavigation": [
"chrome.webNavigation.getFrame",
"chrome.webNavigation.getAllFrames"
],
"system.storage": [
"chrome.system.storage.getInfo",
"chrome.system.storage.ejectDevice",
"chrome.system.storage.getAvailableCapacity"
],
"system.cpu": [
"chrome.system.cpu.getInfo"
],
"accessibilityFeatures.read": [
"accessibilityFeatures.read"
],
"sessions": [
"chrome.sessions.getRecentlyClosed",
"chrome.sessions.getDevices",
"chrome.sessions.restore"
],
"pointerLock": [
"requestPointerLock"
],
"alarms": [
"chrome.alarms.create",
"chrome.alarms.get",
"chrome.alarms.getAll",
"chrome.alarms.clear",
"chrome.alarms.clearAll"
],
"wallpaper": [
"chrome.wallpaper.setWallpaper"
],
"tts": [
"chrome.tts.speak",
"chrome.tts.stop",
"chrome.tts.pause",
"chrome.tts.resume",
"chrome.tts.isSpeaking",
"chrome.tts.getVoices"
],
"topSites": [
"chrome.topSites.get"
],
"platformKeys": [
"chrome.platformKeys.selectClientCertificates",
"chrome.platformKeys.getKeyPair",
"chrome.platformKeys.subtleCrypto",
"chrome.platformKeys.verifyTLSServerCertificate"
],
"notifications": [
"chrome.notifications.create",
"chrome.notifications.update",
"chrome.notifications.clear",
"chrome.notifications.getAll",
"chrome.notifications.getPermissionLevel"
],
"audio": [
"chrome.audio.getDevices",
"chrome.audio.setActiveDevices",
"chrome.audio.setProperties",
"chrome.audio.getMute",
"chrome.audio.setMute"
],
"desktopCapture": [
"chrome.desktopCapture.chooseDesktopMedia",
"chrome.desktopCapture.cancelChooseDesktopMedia"
],
"identity": [
"chrome.identity.getAccounts",
"chrome.identity.getAuthToken",
"chrome.identity.getProfileUserInfo",
"chrome.identity.removeCachedAuthToken",
"chrome.identity.launchWebAuthFlow",
"chrome.identity.getRedirectURL"
],
"clipboardWrite": [
"document.execCommand(\"copy\")",
"document.execCommand(\"cut\")"
],
"system.display": [
"chrome.system.display.getInfo",
"chrome.system.display.getDisplayLayout",
"chrome.system.display.setDisplayProperties",
"chrome.system.display.setDisplayLayout",
"chrome.system.display.enableUnifiedDesktop",
"chrome.system.display.overscanCalibrationStart",
"chrome.system.display.overscanCalibrationAdjust",
"chrome.system.display.overscanCalibrationReset",
"chrome.system.display.overscanCalibrationComplete",
"chrome.system.display.showNativeTouchCalibration",
"chrome.system.display.startCustomTouchCalibration",
"chrome.system.display.completeCustomTouchCalibration",
"chrome.system.display.clearTouchCalibration",
"chrome.system.display.setMirrorMode"
]
}

View File

@ -26,6 +26,7 @@ import sys
import importlib.util
import csv
import math
import ast
from zipfile import ZipFile
@ -63,11 +64,9 @@ def import_regexs(path):
def get_etag(headers_content):
headers_content = headers_content.replace(
'"', '\\"').replace("'", '"')
headers_json = json.loads(headers_content)
if "ETag" in headers_json:
return headers_json["ETag"]
d = ast.literal_eval(headers_content)
if "ETag" in d:
return d["ETag"]
def get_name_and_version(overview_contents):

View File

@ -17,9 +17,10 @@ usage() {
echo " -s \"<args>\" (add qsub arguments, default: ${SGE_EXTRA_ARGS})"
echo " -p \"<args>\" (add python script arguments, default: ${PY_EXTRA_ARGS})"
echo " -e <path> (set path to extension id list, default: crawl from archive)"
echo " -l <N> (limit number of sharc tasks, default: number of extensions)"
}
while getopts ":a:t:s:p:m:e:" o; do
while getopts ":a:t:s:p:m:e:l:" o; do
case "${o}" in
a)
REMOTE_ARCHIVE=${OPTARG}
@ -39,6 +40,9 @@ while getopts ":a:t:s:p:m:e:" o; do
e)
EXTENSION_IDS="${OPTARG}"
;;
l)
MAX_TASKS="${OPTARG}"
;;
*)
usage
exit 1
@ -59,13 +63,19 @@ echo "Pushing sge script ..."
scp "$BASEDIR/sge/create-db.sge" sharc.shef.ac.uk:"$TARGETDIR/create-db.sge"
echo "Building image..."
if [ -f "$BASEDIR/singularity/create-db.img" ]; then
rm -f "$BASEDIR/singularity/create-db.img"
if [ -f "$BASEDIR/scripts/singularity/create-db.img" ]; then
rm -f "$BASEDIR/scripts/singularity/create-db.img"
fi
sudo singularity build "$BASEDIR/singularity/create-db.img" "$BASEDIR/singularity/ExtensionCrawler-dev.def"
(
cd "$BASEDIR/scripts/singularity"
if [[ "$(docker images -q singularitybuilder-arch 2> /dev/null)" == "" ]]; then
docker build --tag=singularitybuilder -f singularitybuilder-arch.Dockerfile .
fi
docker run -it -v "$(pwd):$(pwd)" -w "$(pwd)" --privileged singularitybuilder-arch:latest singularity build create-db.img ExtensionCrawler.def
)
echo "Pushing image..."
scp "$BASEDIR/singularity/create-db.img" sharc.shef.ac.uk:"$TARGETDIR/create-db.img"
scp "$BASEDIR/scripts/singularity/create-db.img" sharc.shef.ac.uk:"$TARGETDIR/create-db.img"
if [[ -z $EXTENSION_IDS ]]; then
@ -86,8 +96,12 @@ fi
echo "Pushing extension IDs..."
scp ${TEMP_FOLDER}/extension.ids sharc.shef.ac.uk:$TARGETDIR/
NO_BATCH_JOBS=$(((NO_IDS+1)/75000+1))
JOBS_PER_BATCH=$((NO_IDS/NO_BATCH_JOBS+1))
if [[ ! -v MAX_TASKS ]]; then
MAX_TASKS=NO_IDS
fi
NO_BATCH_JOBS=$(((MAX_TASKS+1)/75000+1))
JOBS_PER_BATCH=$((MAX_TASKS/NO_BATCH_JOBS+1))
for run_no in $(seq 1 $NO_BATCH_JOBS); do
FIRST_ID=$(((run_no-1) * $JOBS_PER_BATCH + 1))
@ -100,5 +114,5 @@ for run_no in $(seq 1 $NO_BATCH_JOBS); do
-wd "$TARGETDIR" \
-o "$TARGETDIR/logs" \
${SGE_EXTRA_ARGS} \
"$TARGETDIR/create-db.sge" -a "$REMOTE_ARCHIVE" -e "${TARGETDIR}/extension.ids" -N $NO_IDS ${PY_EXTRA_ARGS})
"$TARGETDIR/create-db.sge" -a "$REMOTE_ARCHIVE" -e "${TARGETDIR}/extension.ids" -N $MAX_TASKS ${PY_EXTRA_ARGS})
done