Update gradle-download-plugin and use some of its new features

We now download and verify checksums as a single task, rather than as
two separate tasks.  This simplifies other task dependencies, since we
no longer have a checksum-verified "stamp" file separate from the
download itself.  Unfortunately the combined task now has a
significant amount of repeated boilerplate.  I'm hoping to refactor
that all out into a custom task class, but haven't yet figured out the
details:
<https://github.com/michel-kraemer/gradle-download-task/issues/108>.

We now also use ETags to be smarter about when a fresh download is or
is not actually needed.  I think there are still opportunities for
improved caching here, but this is a step in the right direction.

build.gradle

@@ -6,7 +6,7 @@
 plugins {
 	id 'com.diffplug.gradle.p2.asmaven' version '3.10.0'
 	id 'com.github.hauner.jarTest' version '1.0.1' apply false
-	id 'de.undercouch.download' apply false
+	id 'de.undercouch.download'
 	id 'nebula.lint' version '8.3.1'
 	id "nebula.source-jar" version '7.0.1' apply false
 }

buildSrc/build.gradle

@@ -3,5 +3,5 @@ repositories {
 }
 
 dependencies {
-	compile 'de.undercouch:gradle-download-task:3.3.0'
+	compile 'de.undercouch:gradle-download-task:3.4.1'
 }

buildSrc/src/main/groovy/stamp.groovy

@@ -1,12 +0,0 @@
-////////////////////////////////////////////////////////////////////////
-//
-//  helper task for checksum-verified downloads
-//
-
-class VerifyWithStamp extends de.undercouch.gradle.tasks.download.Verify {
-	VerifyWithStamp() {
-		def stamp = new File(temporaryDir, 'stamp')
-		outputs.file stamp
-		doLast { stamp.text = '' }
-	}
-}

com.ibm.wala.cast.java.test.data/build.gradle

@@ -6,23 +6,26 @@ sourceSets.test.java.srcDirs = ['src']
 //  download JLex
 //
 
-def versionedArchive = 'ajaxslt-0.8.1'
+task downloadJLex {
-def packedArchive = "${versionedArchive}.tar.gz"
+	outputs.file 'src/JLex/Main.java'
-
+	doLast {
-task downloadJLex(type: de.undercouch.gradle.tasks.download.Download) {
+		download {
-	src 'http://www.cs.princeton.edu/~appel/modern/java/JLex/current/Main.java'
+			src 'http://www.cs.princeton.edu/~appel/modern/java/JLex/current/Main.java'
-	dest 'src/JLex/Main.java'
+			dest outputs.files.singleFile
-	overwrite false
+			overwrite true
+			onlyIfModified true
+			useETag true
+		}
+		verifyChecksum {
+			src outputs.files.singleFile
+			checksum 'fe0cff5db3e2f0f5d67a153cf6c783af'
+		}
+	}
 }
 
 task cleanDownloadJLex(type: Delete) {
-	delete downloadJLex.dest.parent
+	delete files(downloadJLex).singleFile.parent
 }
 
-task verifyJLex(type: VerifyWithStamp, dependsOn: downloadJLex) {
+compileTestJava.dependsOn downloadJLex
-	src downloadJLex.dest
-	checksum 'fe0cff5db3e2f0f5d67a153cf6c783af'
-}
-
-compileTestJava.dependsOn verifyJLex
 clean.dependsOn cleanDownloadJLex

com.ibm.wala.cast.js.nodejs/build.gradle

@@ -10,19 +10,25 @@ dependencies {
 		)
 }
 
-task downloadNodeJS(type: de.undercouch.gradle.tasks.download.Download) {
+task downloadNodeJS {
-	src 'https://api.github.com/repos/nodejs/node/zipball/0a604e92e258c5ee2752d763e50721e35053f135'
+	outputs.file "$temporaryDir/nodejs.zip"
-	dest new File(temporaryDir, 'nodejs.zip')
+	doLast {
-	overwrite false
+		download {
+			src 'https://api.github.com/repos/nodejs/node/zipball/0a604e92e258c5ee2752d763e50721e35053f135'
+			dest outputs.files.singleFile
+			overwrite true
+			onlyIfModified true
+			useETag true
+		}
+		verifyChecksum {
+			src outputs.files.singleFile
+			checksum '33c5ba7a5d45644e70d268d8ad3e57df'
+		}
+	}
 }
 
-task verifyNodeJS(type: VerifyWithStamp, dependsOn: downloadNodeJS) {
+task unpackNodeJSLib(type: Copy, dependsOn: downloadNodeJS) {
-	src files(downloadNodeJS).singleFile
+	from(zipTree(files(downloadNodeJS).singleFile)) {
-	checksum '33c5ba7a5d45644e70d268d8ad3e57df'
-}
-
-task unpackNodeJSLib(type: Copy, dependsOn: verifyNodeJS) {
-	from(zipTree(verifyNodeJS.src)) {
 		include 'nodejs-node-0a604e9/lib/*.js'
 		eachFile {
 			relativePath new RelativePath(!directory, relativePath.lastName)

com.ibm.wala.cast.js.test.data/build.gradle

@@ -1,22 +1,26 @@
 apply plugin: 'base'
 
-task downloadAjaxslt(type: de.undercouch.gradle.tasks.download.Download) {
+task downloadAjaxslt {
 	def version = '0.8.1'
-	ext {
+	def versionedArchive = "ajaxslt-${version}.tar.gz"
-		versionedArchive = "ajaxslt-$version"
+	outputs.file "$temporaryDir/$versionedArchive"
+	doLast {
+		download {
+			src "https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/ajaxslt/$versionedArchive"
+			dest outputs.files.singleFile
+			overwrite true
+			onlyIfModified true
+			useETag true
+		}
+		verifyChecksum {
+			src outputs.files.singleFile
+			checksum 'c995abe3310a401bb4db7f28a6409756'
+		}
 	}
-	src "https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/ajaxslt/${versionedArchive}.tar.gz"
-	dest temporaryDir
-	overwrite false
 }
 
-task verifyAjaxslt(type: VerifyWithStamp, dependsOn: downloadAjaxslt) {
+task unpackAjaxslt(type: Sync, dependsOn: downloadAjaxslt) {
-	src files(downloadAjaxslt).singleFile
+	from(tarTree(files(downloadAjaxslt).singleFile)) {
-	checksum 'c995abe3310a401bb4db7f28a6409756'
-}
-
-task unpackAjaxslt(type: Sync, dependsOn: verifyAjaxslt) {
-	from(tarTree(verifyAjaxslt.src)) {
 		eachFile {
 			def newSegments = relativePath.segments[1 .. -1] as String[]
 			relativePath new RelativePath(!directory, newSegments)

com.ibm.wala.core.testdata/build.gradle

@@ -5,40 +5,42 @@ eclipse.project.natures 'org.eclipse.pde.PluginNature'
 sourceSets.test.java.srcDirs = ['src']
 
 
-////////////////////////////////////////////////////////////////////////
-//
-//  download and/or create extra bundled jar archives
-//
-
-import de.undercouch.gradle.tasks.download.*
-
-
 ////////////////////////////////////////////////////////////////////////
 //
 //  download and extract "bcel-5.2.jar"
 //
 
-task downloadBcel(type: Download) {
+task downloadBcel {
-	src 'http://archive.apache.org/dist/jakarta/bcel/binaries/bcel-5.2.tar.gz'
+	ext {
-	dest temporaryDir
+		version = '5.2'
-	overwrite false
+	}
+	def archive = "bcel-${version}.tar.gz"
+	outputs.file "$temporaryDir/$archive"
+	doLast {
+		download {
+			src "http://archive.apache.org/dist/jakarta/bcel/binaries/$archive"
+			dest outputs.files.singleFile
+			overwrite true
+			onlyIfModified true
+			useETag false // server does not provide ETags
+		}
+		verifyChecksum {
+			src outputs.files.singleFile
+			checksum '19bffd7f217b0eae415f1ef87af2f0bc'
+		}
+	}
 }
 
-task verifyBcel(type: VerifyWithStamp, dependsOn: downloadBcel) {
+task extractBcel(type: Copy, dependsOn: downloadBcel) {
-	src files(downloadBcel).singleFile
+	from(tarTree(files(downloadBcel).singleFile)) {
-	checksum '19bffd7f217b0eae415f1ef87af2f0bc'
+		include "bcel-${downloadBcel.version}/bcel-${downloadBcel.version}.jar"
-}
-
-task extractBcel(type: Copy, dependsOn: verifyBcel) {
-	from(tarTree(verifyBcel.src)) {
-		include 'bcel-5.2/bcel-5.2.jar'
 		eachFile {
 			relativePath new RelativePath(!directory, relativePath.lastName)
 		}
 	}
 	into projectDir
 	includeEmptyDirs false
-	outputs.files 'bcel-5.2.jar'
+	outputs.file "bcel-${downloadBcel.version}.jar"
 }
 
 task cleanExtractBcel(type: Delete) {
@@ -54,21 +56,28 @@ clean.dependsOn cleanExtractBcel
 //  download "java-cup-11a.jar"
 //
 
-task downloadJavaCup(type: Download) {
+task downloadJavaCup {
-	src 'http://www2.cs.tum.edu/projects/cup/java-cup-11a.jar'
+	def archive = 'java-cup-11a.jar'
-	dest projectDir
+	outputs.file "$projectDir/$archive"
-	overwrite false
+	doLast {
+		download {
+			src "http://www2.cs.tum.edu/projects/cup/$archive"
+			dest outputs.files.singleFile
+			overwrite true
+			onlyIfModified true
+			useETag true
+		}
+		verifyChecksum {
+			src outputs.files.singleFile
+			checksum '2bda8c40abd0cbc295d3038643d6e4ec'
+		}
+	}
 }
 
 task cleanDownloadJavaCup(type: Delete) {
 	delete downloadJavaCup
 }
 
-task verifyJavaCup(type: VerifyWithStamp, dependsOn: downloadJavaCup) {
-	src files(downloadJavaCup).singleFile
-	checksum '2bda8c40abd0cbc295d3038643d6e4ec'
-}
-
 clean.dependsOn cleanDownloadJavaCup
 
 
@@ -97,23 +106,28 @@ clean.dependsOn cleanCollectJLex
 //  generate "hello_hash.jar"
 //
 
-task downloadOcamlJava(type: Download) {
+task downloadOcamlJava {
 	def version = '2.0-alpha1'
-	ext {
+	ext.basename = "ocamljava-$version"
-		basename = "ocamljava-$version"
+	def archive = "${basename}.tar.gz"
+	outputs.file "$temporaryDir/$archive"
+	doLast {
+		download {
+			src "http://www.ocamljava.org/downloads/download.php?version=$version-bin"
+			dest outputs.files.singleFile
+			overwrite true
+			onlyIfModified true
+			useETag true
+		}
+		verifyChecksum {
+			src outputs.files.singleFile
+			checksum '45feec6e3889f5073a39c2c4c84878d1'
+		}
 	}
-	src "http://www.ocamljava.org/downloads/download.php?version=$version-bin"
-	dest new File(temporaryDir, "${basename}.tar.gz")
-	overwrite false
 }
 
-task verifyOcamlJava(type: VerifyWithStamp, dependsOn: downloadOcamlJava) {
+task unpackOcamlJava(type: Sync, dependsOn: downloadOcamlJava) {
-	src files(downloadOcamlJava).singleFile
+	from tarTree(files(downloadOcamlJava).singleFile)
-	checksum '45feec6e3889f5073a39c2c4c84878d1'
-}
-
-task unpackOcamlJava(type: Sync, dependsOn: verifyOcamlJava) {
-	from tarTree(verifyOcamlJava.src)
 	into temporaryDir
 }
 
@@ -191,8 +205,8 @@ afterEclipseBuildshipImport {
 		collectJLex,
 		collectTestData,
 		collectTestDataA,
+		downloadJavaCup,
 		extractBcel,
 		generateHelloHash,
-		verifyJavaCup,
 		)
 }

com.ibm.wala.core.tests/build.gradle

@@ -25,7 +25,6 @@ processTestResources {
 	def testdata = project(':com.ibm.wala.core.testdata')
 	dependsOn testdata.compileTestJava
 	dependsOn testdata.extractBcel
-	dependsOn testdata.verifyJavaCup
 
 	from testdata.collectJLex
 	from testdata.collectTestData

com.ibm.wala.dalvik.test/build.gradle

@@ -6,8 +6,6 @@ sourceSets.test {
 		]
 }
 
-import de.undercouch.gradle.tasks.download.Download
-
 task cloneDroidBench(type: Exec) {
 	// TODO: download to somewhere private to this build tree; update code that looks in "/tmp" accordingly
 	def destDir = file('/tmp/DroidBench')
@@ -18,88 +16,108 @@ task cloneDroidBench(type: Exec) {
 	commandLine 'git', 'clone', "--branch=$inputs.properties.branch", '--config=advice.detachedHead=false', '--depth=1', '--quiet', inputs.properties.repository, destDir
 }
 
-task downloadAndroidSdk(type: Download) {
+task downloadAndroidSdk {
 	def sdkOs
+	def sdkChecksum
 	switch (System.getProperty('os.name')) {
 		case ~/Linux/:
 			sdkOs = 'linux'
-			ext.checksum = '444e22ce8ca0f67353bda4b85175ed3731cae3ffa695ca18119cbacef1c1bea0'
+			sdkChecksum = '444e22ce8ca0f67353bda4b85175ed3731cae3ffa695ca18119cbacef1c1bea0'
 			break
 		case ~/Mac OS X/:
 			sdkOs = 'darwin'
-			ext.checksum = '4a81754a760fce88cba74d69c364b05b31c53d57b26f9f82355c61d5fe4b9df9'
+			sdkChecksum = '4a81754a760fce88cba74d69c364b05b31c53d57b26f9f82355c61d5fe4b9df9'
 			break
 		case ~/Windows.*/:
 			sdkOs = 'windows'
-			ext.checksum = '7f6037d3a7d6789b4fdc06ee7af041e071e9860c51f66f7a4eb5913df9871fd2'
+			sdkChecksum = '7f6037d3a7d6789b4fdc06ee7af041e071e9860c51f66f7a4eb5913df9871fd2'
 			break
 	}
-	src "https://dl.google.com/android/repository/sdk-tools-$sdkOs-3859397.zip"
+	def archive = "sdk-tools-$sdkOs-3859397.zip"
-	dest temporaryDir
+	outputs.file "$temporaryDir/$archive"
-	overwrite false
+	doLast {
+		download {
+			src "https://dl.google.com/android/repository/$archive"
+			dest outputs.files.singleFile
+			overwrite true
+			onlyIfModified true
+			useETag true
+		}
+		verifyChecksum {
+			src outputs.files.singleFile
+			algorithm 'SHA-256'
+			checksum sdkChecksum
+		}
+	}
 }
 
-task verifyAndroidSdk(type: VerifyWithStamp, dependsOn: downloadAndroidSdk) {
+task unpackAndroidSdk(type: Sync, dependsOn: downloadAndroidSdk) {
-	src files(downloadAndroidSdk).singleFile
+	from zipTree(files(downloadAndroidSdk).singleFile)
-	algorithm 'SHA-256'
-	checksum downloadAndroidSdk.checksum
-}
-
-task unpackAndroidSdk(type: Sync, dependsOn: verifyAndroidSdk) {
-	from zipTree(verifyAndroidSdk.src)
 	into temporaryDir
 }
 
 // TODO: factor out common code in installAndroidBuildTools and installAndroidPlatforms
 task installAndroidBuildTools(type: Exec, dependsOn: unpackAndroidSdk) {
-	def manager = "${unpackAndroidSdk.outputs.files.singleFile}/tools/bin/sdkmanager"
+	def androidSdk = files(unpackAndroidSdk).singleFile
+	def manager = "$androidSdk/tools/bin/sdkmanager"
 	ext.version = '26.0.2'
 	commandLine 'sh', '-ceu', "yes 2>/dev/null | $manager build-tools\\;$version >/dev/null"
-	def androidSdk = unpackAndroidSdk.outputs.files.singleFile
 	inputs.dir "$androidSdk/tools"
 	outputs.dir "$androidSdk/build-tools"
 }
 
 task copyDxJar(type: Sync, dependsOn: installAndroidBuildTools) {
-	from "${installAndroidBuildTools.outputs.files.singleFile}/${installAndroidBuildTools.version}/lib/dx.jar"
+	from "${files(installAndroidBuildTools).singleFile}/${installAndroidBuildTools.version}/lib/dx.jar"
 	into 'lib'
 }
 
 // TODO: factor out common code in installAndroidBuildTools and installAndroidPlatforms
 task installAndroidPlatforms(type: Exec, dependsOn: unpackAndroidSdk) {
-	def manager = "${unpackAndroidSdk.outputs.files.singleFile}/tools/bin/sdkmanager"
+	def manager = "${files(unpackAndroidSdk).singleFile}/tools/bin/sdkmanager"
 	ext.version = "android-${installAndroidBuildTools.version.tokenize('.')[0]}"
 	commandLine 'sh', '-ceu', "yes 2>/dev/null | $manager platforms\\;$version >/dev/null"
-	def androidSdk = unpackAndroidSdk.outputs.files.singleFile
+	def androidSdk = files(unpackAndroidSdk).singleFile
 	inputs.dir "$androidSdk/tools"
 	outputs.dir "$androidSdk/platforms"
 }
 
 task copyAndroidJar(type: Sync, dependsOn: installAndroidPlatforms) {
-	from "${installAndroidPlatforms.outputs.files.singleFile}/${installAndroidPlatforms.version}/android.jar"
+	from "${files(installAndroidPlatforms).singleFile}/${installAndroidPlatforms.version}/android.jar"
 	into temporaryDir
 }
 
-task downloadSampleCup(type: Download) {
+task downloadSampleCup {
-	src 'http://www.cc.gatech.edu/gvu/people/faculty/hudson/java_cup/classes.v0.9e/java_cup/parser.cup'
+	outputs.file 'data/sample.cup'
-	dest 'data/sample.cup'
+	doLast {
-	overwrite false
+		download {
+			src 'http://www.cc.gatech.edu/gvu/people/faculty/hudson/java_cup/classes.v0.9e/java_cup/parser.cup'
+			dest outputs.files.singleFile
+			overwrite true
+			onlyIfModified true
+			useETag true
+		}
+		verifyChecksum {
+			src outputs.files.singleFile
+			checksum '76b549e7c6e802b811a374248175ecf4'
+		}
+	}
 }
 
-task verifySampleCup(type: VerifyWithStamp, dependsOn: downloadSampleCup) {
+task downloadSampleLex {
-	src files(downloadSampleCup).singleFile
+	outputs.file 'data/sample.lex'
-	checksum '76b549e7c6e802b811a374248175ecf4'
+	doLast {
-}
+		download {
-
+			src 'https://www.cs.princeton.edu/~appel/modern/java/JLex/current/sample.lex'
-task downloadSampleLex(type: Download) {
+			dest outputs.files.singleFile
-	src 'https://www.cs.princeton.edu/~appel/modern/java/JLex/current/sample.lex'
+			overwrite true
-	dest 'data'
+			onlyIfModified true
-	overwrite false
+			useETag true
-}
+		}
-
+		verifyChecksum {
-task verifySampleLex(type: VerifyWithStamp, dependsOn: downloadSampleLex) {
+			src outputs.files.singleFile
-	src files(downloadSampleLex).singleFile
+			checksum 'ae887758b2657981d023a72a165da830'
-	checksum 'ae887758b2657981d023a72a165da830'
+		}
+	}
 }
 
 clean.dependsOn cleanCopyDxJar
@@ -110,29 +128,24 @@ dependencies {
 	testCompile(
 		'junit:junit:4.11',
 		'org.osgi:org.osgi.core:4.2.0',
-		files("${copyDxJar.outputs.files.singleFile}/dx.jar"),
+		files("${files(copyDxJar).singleFile}/dx.jar"),
 		project(':com.ibm.wala.core'),
 		project(':com.ibm.wala.dalvik'),
 		project(':com.ibm.wala.shrike'),
 		project(':com.ibm.wala.util'),
 		project(configuration: 'testArchives', path: ':com.ibm.wala.core.tests'),
 		)
-	testRuntime files("${copyAndroidJar.outputs.files.singleFile}/android.jar")
+	testRuntime files("${files(copyAndroidJar).singleFile}/android.jar")
 }
 
 processTestResources {
 	dependsOn cloneDroidBench
 
-	dependsOn verifySampleCup
+	from copyAndroidJar
 	from downloadSampleCup
-
-	dependsOn verifySampleLex
 	from downloadSampleLex
 
-	from copyAndroidJar
-
 	def testdata = project(':com.ibm.wala.core.testdata')
-	dependsOn testdata.verifyJavaCup
 	from testdata.collectJLex
 	from testdata.collectTestDataA
 	from testdata.downloadJavaCup