Update gradle-download-plugin and use some of its new features
We now download and verify checksums as a single task, rather than as two separate tasks. This simplifies other task dependencies, since we no longer have a checksum-verified "stamp" file separate from the download itself. Unfortunately the combined task now has a significant amount of repeated boilerplate. I'm hoping to refactor that all out into a custom task class, but haven't yet figured out the details: <https://github.com/michel-kraemer/gradle-download-task/issues/108>. We now also use ETags to be smarter about when a fresh download is or is not actually needed. I think there are still opportunities for improved caching here, but this is a step in the right direction.
This commit is contained in:
parent
17f42c3725
commit
ba455f4737
|
@ -6,7 +6,7 @@
|
|||
plugins {
|
||||
id 'com.diffplug.gradle.p2.asmaven' version '3.10.0'
|
||||
id 'com.github.hauner.jarTest' version '1.0.1' apply false
|
||||
id 'de.undercouch.download' apply false
|
||||
id 'de.undercouch.download'
|
||||
id 'nebula.lint' version '8.3.1'
|
||||
id "nebula.source-jar" version '7.0.1' apply false
|
||||
}
|
||||
|
|
|
@ -3,5 +3,5 @@ repositories {
|
|||
}
|
||||
|
||||
dependencies {
|
||||
compile 'de.undercouch:gradle-download-task:3.3.0'
|
||||
compile 'de.undercouch:gradle-download-task:3.4.1'
|
||||
}
|
||||
|
|
|
@ -1,12 +0,0 @@
|
|||
////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// helper task for checksum-verified downloads
|
||||
//
|
||||
|
||||
class VerifyWithStamp extends de.undercouch.gradle.tasks.download.Verify {
|
||||
VerifyWithStamp() {
|
||||
def stamp = new File(temporaryDir, 'stamp')
|
||||
outputs.file stamp
|
||||
doLast { stamp.text = '' }
|
||||
}
|
||||
}
|
|
@ -6,23 +6,26 @@ sourceSets.test.java.srcDirs = ['src']
|
|||
// download JLex
|
||||
//
|
||||
|
||||
def versionedArchive = 'ajaxslt-0.8.1'
|
||||
def packedArchive = "${versionedArchive}.tar.gz"
|
||||
|
||||
task downloadJLex(type: de.undercouch.gradle.tasks.download.Download) {
|
||||
task downloadJLex {
|
||||
outputs.file 'src/JLex/Main.java'
|
||||
doLast {
|
||||
download {
|
||||
src 'http://www.cs.princeton.edu/~appel/modern/java/JLex/current/Main.java'
|
||||
dest 'src/JLex/Main.java'
|
||||
overwrite false
|
||||
dest outputs.files.singleFile
|
||||
overwrite true
|
||||
onlyIfModified true
|
||||
useETag true
|
||||
}
|
||||
verifyChecksum {
|
||||
src outputs.files.singleFile
|
||||
checksum 'fe0cff5db3e2f0f5d67a153cf6c783af'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
task cleanDownloadJLex(type: Delete) {
|
||||
delete downloadJLex.dest.parent
|
||||
delete files(downloadJLex).singleFile.parent
|
||||
}
|
||||
|
||||
task verifyJLex(type: VerifyWithStamp, dependsOn: downloadJLex) {
|
||||
src downloadJLex.dest
|
||||
checksum 'fe0cff5db3e2f0f5d67a153cf6c783af'
|
||||
}
|
||||
|
||||
compileTestJava.dependsOn verifyJLex
|
||||
compileTestJava.dependsOn downloadJLex
|
||||
clean.dependsOn cleanDownloadJLex
|
||||
|
|
|
@ -10,19 +10,25 @@ dependencies {
|
|||
)
|
||||
}
|
||||
|
||||
task downloadNodeJS(type: de.undercouch.gradle.tasks.download.Download) {
|
||||
task downloadNodeJS {
|
||||
outputs.file "$temporaryDir/nodejs.zip"
|
||||
doLast {
|
||||
download {
|
||||
src 'https://api.github.com/repos/nodejs/node/zipball/0a604e92e258c5ee2752d763e50721e35053f135'
|
||||
dest new File(temporaryDir, 'nodejs.zip')
|
||||
overwrite false
|
||||
}
|
||||
|
||||
task verifyNodeJS(type: VerifyWithStamp, dependsOn: downloadNodeJS) {
|
||||
src files(downloadNodeJS).singleFile
|
||||
dest outputs.files.singleFile
|
||||
overwrite true
|
||||
onlyIfModified true
|
||||
useETag true
|
||||
}
|
||||
verifyChecksum {
|
||||
src outputs.files.singleFile
|
||||
checksum '33c5ba7a5d45644e70d268d8ad3e57df'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
task unpackNodeJSLib(type: Copy, dependsOn: verifyNodeJS) {
|
||||
from(zipTree(verifyNodeJS.src)) {
|
||||
task unpackNodeJSLib(type: Copy, dependsOn: downloadNodeJS) {
|
||||
from(zipTree(files(downloadNodeJS).singleFile)) {
|
||||
include 'nodejs-node-0a604e9/lib/*.js'
|
||||
eachFile {
|
||||
relativePath new RelativePath(!directory, relativePath.lastName)
|
||||
|
|
|
@ -1,22 +1,26 @@
|
|||
apply plugin: 'base'
|
||||
|
||||
task downloadAjaxslt(type: de.undercouch.gradle.tasks.download.Download) {
|
||||
task downloadAjaxslt {
|
||||
def version = '0.8.1'
|
||||
ext {
|
||||
versionedArchive = "ajaxslt-$version"
|
||||
def versionedArchive = "ajaxslt-${version}.tar.gz"
|
||||
outputs.file "$temporaryDir/$versionedArchive"
|
||||
doLast {
|
||||
download {
|
||||
src "https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/ajaxslt/$versionedArchive"
|
||||
dest outputs.files.singleFile
|
||||
overwrite true
|
||||
onlyIfModified true
|
||||
useETag true
|
||||
}
|
||||
src "https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/ajaxslt/${versionedArchive}.tar.gz"
|
||||
dest temporaryDir
|
||||
overwrite false
|
||||
}
|
||||
|
||||
task verifyAjaxslt(type: VerifyWithStamp, dependsOn: downloadAjaxslt) {
|
||||
src files(downloadAjaxslt).singleFile
|
||||
verifyChecksum {
|
||||
src outputs.files.singleFile
|
||||
checksum 'c995abe3310a401bb4db7f28a6409756'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
task unpackAjaxslt(type: Sync, dependsOn: verifyAjaxslt) {
|
||||
from(tarTree(verifyAjaxslt.src)) {
|
||||
task unpackAjaxslt(type: Sync, dependsOn: downloadAjaxslt) {
|
||||
from(tarTree(files(downloadAjaxslt).singleFile)) {
|
||||
eachFile {
|
||||
def newSegments = relativePath.segments[1 .. -1] as String[]
|
||||
relativePath new RelativePath(!directory, newSegments)
|
||||
|
|
|
@ -5,40 +5,42 @@ eclipse.project.natures 'org.eclipse.pde.PluginNature'
|
|||
sourceSets.test.java.srcDirs = ['src']
|
||||
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// download and/or create extra bundled jar archives
|
||||
//
|
||||
|
||||
import de.undercouch.gradle.tasks.download.*
|
||||
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
//
|
||||
// download and extract "bcel-5.2.jar"
|
||||
//
|
||||
|
||||
task downloadBcel(type: Download) {
|
||||
src 'http://archive.apache.org/dist/jakarta/bcel/binaries/bcel-5.2.tar.gz'
|
||||
dest temporaryDir
|
||||
overwrite false
|
||||
}
|
||||
|
||||
task verifyBcel(type: VerifyWithStamp, dependsOn: downloadBcel) {
|
||||
src files(downloadBcel).singleFile
|
||||
task downloadBcel {
|
||||
ext {
|
||||
version = '5.2'
|
||||
}
|
||||
def archive = "bcel-${version}.tar.gz"
|
||||
outputs.file "$temporaryDir/$archive"
|
||||
doLast {
|
||||
download {
|
||||
src "http://archive.apache.org/dist/jakarta/bcel/binaries/$archive"
|
||||
dest outputs.files.singleFile
|
||||
overwrite true
|
||||
onlyIfModified true
|
||||
useETag false // server does not provide ETags
|
||||
}
|
||||
verifyChecksum {
|
||||
src outputs.files.singleFile
|
||||
checksum '19bffd7f217b0eae415f1ef87af2f0bc'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
task extractBcel(type: Copy, dependsOn: verifyBcel) {
|
||||
from(tarTree(verifyBcel.src)) {
|
||||
include 'bcel-5.2/bcel-5.2.jar'
|
||||
task extractBcel(type: Copy, dependsOn: downloadBcel) {
|
||||
from(tarTree(files(downloadBcel).singleFile)) {
|
||||
include "bcel-${downloadBcel.version}/bcel-${downloadBcel.version}.jar"
|
||||
eachFile {
|
||||
relativePath new RelativePath(!directory, relativePath.lastName)
|
||||
}
|
||||
}
|
||||
into projectDir
|
||||
includeEmptyDirs false
|
||||
outputs.files 'bcel-5.2.jar'
|
||||
outputs.file "bcel-${downloadBcel.version}.jar"
|
||||
}
|
||||
|
||||
task cleanExtractBcel(type: Delete) {
|
||||
|
@ -54,21 +56,28 @@ clean.dependsOn cleanExtractBcel
|
|||
// download "java-cup-11a.jar"
|
||||
//
|
||||
|
||||
task downloadJavaCup(type: Download) {
|
||||
src 'http://www2.cs.tum.edu/projects/cup/java-cup-11a.jar'
|
||||
dest projectDir
|
||||
overwrite false
|
||||
task downloadJavaCup {
|
||||
def archive = 'java-cup-11a.jar'
|
||||
outputs.file "$projectDir/$archive"
|
||||
doLast {
|
||||
download {
|
||||
src "http://www2.cs.tum.edu/projects/cup/$archive"
|
||||
dest outputs.files.singleFile
|
||||
overwrite true
|
||||
onlyIfModified true
|
||||
useETag true
|
||||
}
|
||||
verifyChecksum {
|
||||
src outputs.files.singleFile
|
||||
checksum '2bda8c40abd0cbc295d3038643d6e4ec'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
task cleanDownloadJavaCup(type: Delete) {
|
||||
delete downloadJavaCup
|
||||
}
|
||||
|
||||
task verifyJavaCup(type: VerifyWithStamp, dependsOn: downloadJavaCup) {
|
||||
src files(downloadJavaCup).singleFile
|
||||
checksum '2bda8c40abd0cbc295d3038643d6e4ec'
|
||||
}
|
||||
|
||||
clean.dependsOn cleanDownloadJavaCup
|
||||
|
||||
|
||||
|
@ -97,23 +106,28 @@ clean.dependsOn cleanCollectJLex
|
|||
// generate "hello_hash.jar"
|
||||
//
|
||||
|
||||
task downloadOcamlJava(type: Download) {
|
||||
task downloadOcamlJava {
|
||||
def version = '2.0-alpha1'
|
||||
ext {
|
||||
basename = "ocamljava-$version"
|
||||
}
|
||||
ext.basename = "ocamljava-$version"
|
||||
def archive = "${basename}.tar.gz"
|
||||
outputs.file "$temporaryDir/$archive"
|
||||
doLast {
|
||||
download {
|
||||
src "http://www.ocamljava.org/downloads/download.php?version=$version-bin"
|
||||
dest new File(temporaryDir, "${basename}.tar.gz")
|
||||
overwrite false
|
||||
}
|
||||
|
||||
task verifyOcamlJava(type: VerifyWithStamp, dependsOn: downloadOcamlJava) {
|
||||
src files(downloadOcamlJava).singleFile
|
||||
dest outputs.files.singleFile
|
||||
overwrite true
|
||||
onlyIfModified true
|
||||
useETag true
|
||||
}
|
||||
verifyChecksum {
|
||||
src outputs.files.singleFile
|
||||
checksum '45feec6e3889f5073a39c2c4c84878d1'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
task unpackOcamlJava(type: Sync, dependsOn: verifyOcamlJava) {
|
||||
from tarTree(verifyOcamlJava.src)
|
||||
task unpackOcamlJava(type: Sync, dependsOn: downloadOcamlJava) {
|
||||
from tarTree(files(downloadOcamlJava).singleFile)
|
||||
into temporaryDir
|
||||
}
|
||||
|
||||
|
@ -191,8 +205,8 @@ afterEclipseBuildshipImport {
|
|||
collectJLex,
|
||||
collectTestData,
|
||||
collectTestDataA,
|
||||
downloadJavaCup,
|
||||
extractBcel,
|
||||
generateHelloHash,
|
||||
verifyJavaCup,
|
||||
)
|
||||
}
|
||||
|
|
|
@ -25,7 +25,6 @@ processTestResources {
|
|||
def testdata = project(':com.ibm.wala.core.testdata')
|
||||
dependsOn testdata.compileTestJava
|
||||
dependsOn testdata.extractBcel
|
||||
dependsOn testdata.verifyJavaCup
|
||||
|
||||
from testdata.collectJLex
|
||||
from testdata.collectTestData
|
||||
|
|
|
@ -6,8 +6,6 @@ sourceSets.test {
|
|||
]
|
||||
}
|
||||
|
||||
import de.undercouch.gradle.tasks.download.Download
|
||||
|
||||
task cloneDroidBench(type: Exec) {
|
||||
// TODO: download to somewhere private to this build tree; update code that looks in "/tmp" accordingly
|
||||
def destDir = file('/tmp/DroidBench')
|
||||
|
@ -18,88 +16,108 @@ task cloneDroidBench(type: Exec) {
|
|||
commandLine 'git', 'clone', "--branch=$inputs.properties.branch", '--config=advice.detachedHead=false', '--depth=1', '--quiet', inputs.properties.repository, destDir
|
||||
}
|
||||
|
||||
task downloadAndroidSdk(type: Download) {
|
||||
task downloadAndroidSdk {
|
||||
def sdkOs
|
||||
def sdkChecksum
|
||||
switch (System.getProperty('os.name')) {
|
||||
case ~/Linux/:
|
||||
sdkOs = 'linux'
|
||||
ext.checksum = '444e22ce8ca0f67353bda4b85175ed3731cae3ffa695ca18119cbacef1c1bea0'
|
||||
sdkChecksum = '444e22ce8ca0f67353bda4b85175ed3731cae3ffa695ca18119cbacef1c1bea0'
|
||||
break
|
||||
case ~/Mac OS X/:
|
||||
sdkOs = 'darwin'
|
||||
ext.checksum = '4a81754a760fce88cba74d69c364b05b31c53d57b26f9f82355c61d5fe4b9df9'
|
||||
sdkChecksum = '4a81754a760fce88cba74d69c364b05b31c53d57b26f9f82355c61d5fe4b9df9'
|
||||
break
|
||||
case ~/Windows.*/:
|
||||
sdkOs = 'windows'
|
||||
ext.checksum = '7f6037d3a7d6789b4fdc06ee7af041e071e9860c51f66f7a4eb5913df9871fd2'
|
||||
sdkChecksum = '7f6037d3a7d6789b4fdc06ee7af041e071e9860c51f66f7a4eb5913df9871fd2'
|
||||
break
|
||||
}
|
||||
src "https://dl.google.com/android/repository/sdk-tools-$sdkOs-3859397.zip"
|
||||
dest temporaryDir
|
||||
overwrite false
|
||||
}
|
||||
|
||||
task verifyAndroidSdk(type: VerifyWithStamp, dependsOn: downloadAndroidSdk) {
|
||||
src files(downloadAndroidSdk).singleFile
|
||||
def archive = "sdk-tools-$sdkOs-3859397.zip"
|
||||
outputs.file "$temporaryDir/$archive"
|
||||
doLast {
|
||||
download {
|
||||
src "https://dl.google.com/android/repository/$archive"
|
||||
dest outputs.files.singleFile
|
||||
overwrite true
|
||||
onlyIfModified true
|
||||
useETag true
|
||||
}
|
||||
verifyChecksum {
|
||||
src outputs.files.singleFile
|
||||
algorithm 'SHA-256'
|
||||
checksum downloadAndroidSdk.checksum
|
||||
checksum sdkChecksum
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
task unpackAndroidSdk(type: Sync, dependsOn: verifyAndroidSdk) {
|
||||
from zipTree(verifyAndroidSdk.src)
|
||||
task unpackAndroidSdk(type: Sync, dependsOn: downloadAndroidSdk) {
|
||||
from zipTree(files(downloadAndroidSdk).singleFile)
|
||||
into temporaryDir
|
||||
}
|
||||
|
||||
// TODO: factor out common code in installAndroidBuildTools and installAndroidPlatforms
|
||||
task installAndroidBuildTools(type: Exec, dependsOn: unpackAndroidSdk) {
|
||||
def manager = "${unpackAndroidSdk.outputs.files.singleFile}/tools/bin/sdkmanager"
|
||||
def androidSdk = files(unpackAndroidSdk).singleFile
|
||||
def manager = "$androidSdk/tools/bin/sdkmanager"
|
||||
ext.version = '26.0.2'
|
||||
commandLine 'sh', '-ceu', "yes 2>/dev/null | $manager build-tools\\;$version >/dev/null"
|
||||
def androidSdk = unpackAndroidSdk.outputs.files.singleFile
|
||||
inputs.dir "$androidSdk/tools"
|
||||
outputs.dir "$androidSdk/build-tools"
|
||||
}
|
||||
|
||||
task copyDxJar(type: Sync, dependsOn: installAndroidBuildTools) {
|
||||
from "${installAndroidBuildTools.outputs.files.singleFile}/${installAndroidBuildTools.version}/lib/dx.jar"
|
||||
from "${files(installAndroidBuildTools).singleFile}/${installAndroidBuildTools.version}/lib/dx.jar"
|
||||
into 'lib'
|
||||
}
|
||||
|
||||
// TODO: factor out common code in installAndroidBuildTools and installAndroidPlatforms
|
||||
task installAndroidPlatforms(type: Exec, dependsOn: unpackAndroidSdk) {
|
||||
def manager = "${unpackAndroidSdk.outputs.files.singleFile}/tools/bin/sdkmanager"
|
||||
def manager = "${files(unpackAndroidSdk).singleFile}/tools/bin/sdkmanager"
|
||||
ext.version = "android-${installAndroidBuildTools.version.tokenize('.')[0]}"
|
||||
commandLine 'sh', '-ceu', "yes 2>/dev/null | $manager platforms\\;$version >/dev/null"
|
||||
def androidSdk = unpackAndroidSdk.outputs.files.singleFile
|
||||
def androidSdk = files(unpackAndroidSdk).singleFile
|
||||
inputs.dir "$androidSdk/tools"
|
||||
outputs.dir "$androidSdk/platforms"
|
||||
}
|
||||
|
||||
task copyAndroidJar(type: Sync, dependsOn: installAndroidPlatforms) {
|
||||
from "${installAndroidPlatforms.outputs.files.singleFile}/${installAndroidPlatforms.version}/android.jar"
|
||||
from "${files(installAndroidPlatforms).singleFile}/${installAndroidPlatforms.version}/android.jar"
|
||||
into temporaryDir
|
||||
}
|
||||
|
||||
task downloadSampleCup(type: Download) {
|
||||
task downloadSampleCup {
|
||||
outputs.file 'data/sample.cup'
|
||||
doLast {
|
||||
download {
|
||||
src 'http://www.cc.gatech.edu/gvu/people/faculty/hudson/java_cup/classes.v0.9e/java_cup/parser.cup'
|
||||
dest 'data/sample.cup'
|
||||
overwrite false
|
||||
}
|
||||
|
||||
task verifySampleCup(type: VerifyWithStamp, dependsOn: downloadSampleCup) {
|
||||
src files(downloadSampleCup).singleFile
|
||||
dest outputs.files.singleFile
|
||||
overwrite true
|
||||
onlyIfModified true
|
||||
useETag true
|
||||
}
|
||||
verifyChecksum {
|
||||
src outputs.files.singleFile
|
||||
checksum '76b549e7c6e802b811a374248175ecf4'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
task downloadSampleLex(type: Download) {
|
||||
task downloadSampleLex {
|
||||
outputs.file 'data/sample.lex'
|
||||
doLast {
|
||||
download {
|
||||
src 'https://www.cs.princeton.edu/~appel/modern/java/JLex/current/sample.lex'
|
||||
dest 'data'
|
||||
overwrite false
|
||||
}
|
||||
|
||||
task verifySampleLex(type: VerifyWithStamp, dependsOn: downloadSampleLex) {
|
||||
src files(downloadSampleLex).singleFile
|
||||
dest outputs.files.singleFile
|
||||
overwrite true
|
||||
onlyIfModified true
|
||||
useETag true
|
||||
}
|
||||
verifyChecksum {
|
||||
src outputs.files.singleFile
|
||||
checksum 'ae887758b2657981d023a72a165da830'
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
clean.dependsOn cleanCopyDxJar
|
||||
|
@ -110,29 +128,24 @@ dependencies {
|
|||
testCompile(
|
||||
'junit:junit:4.11',
|
||||
'org.osgi:org.osgi.core:4.2.0',
|
||||
files("${copyDxJar.outputs.files.singleFile}/dx.jar"),
|
||||
files("${files(copyDxJar).singleFile}/dx.jar"),
|
||||
project(':com.ibm.wala.core'),
|
||||
project(':com.ibm.wala.dalvik'),
|
||||
project(':com.ibm.wala.shrike'),
|
||||
project(':com.ibm.wala.util'),
|
||||
project(configuration: 'testArchives', path: ':com.ibm.wala.core.tests'),
|
||||
)
|
||||
testRuntime files("${copyAndroidJar.outputs.files.singleFile}/android.jar")
|
||||
testRuntime files("${files(copyAndroidJar).singleFile}/android.jar")
|
||||
}
|
||||
|
||||
processTestResources {
|
||||
dependsOn cloneDroidBench
|
||||
|
||||
dependsOn verifySampleCup
|
||||
from copyAndroidJar
|
||||
from downloadSampleCup
|
||||
|
||||
dependsOn verifySampleLex
|
||||
from downloadSampleLex
|
||||
|
||||
from copyAndroidJar
|
||||
|
||||
def testdata = project(':com.ibm.wala.core.testdata')
|
||||
dependsOn testdata.verifyJavaCup
|
||||
from testdata.collectJLex
|
||||
from testdata.collectTestDataA
|
||||
from testdata.downloadJavaCup
|
||||
|
|
Loading…
Reference in New Issue