DASCA
/
WALA
2
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Update gradle-download-plugin and use some of its new features 

We now download and verify checksums as a single task, rather than as
two separate tasks.  This simplifies other task dependencies, since we
no longer have a checksum-verified "stamp" file separate from the
download itself.  Unfortunately the combined task now has a
significant amount of repeated boilerplate.  I'm hoping to refactor
that all out into a custom task class, but haven't yet figured out the
details:
<https://github.com/michel-kraemer/gradle-download-task/issues/108>.

We now also use ETags to be smarter about when a fresh download is or
is not actually needed.  I think there are still opportunities for
improved caching here, but this is a step in the right direction.
This commit is contained in:
Ben Liblit 2018-03-05 12:33:29 -06:00
parent 17f42c3725
commit ba455f4737
9 changed files with 173 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
build.gradle
View File

@ -6,7 +6,7 @@
plugins {
id 'com.diffplug.gradle.p2.asmaven' version '3.10.0'
id 'com.github.hauner.jarTest' version '1.0.1' apply false
id 'de.undercouch.download' apply false
id 'de.undercouch.download'
id 'nebula.lint' version '8.3.1'
id "nebula.source-jar" version '7.0.1' apply false
}

2
buildSrc/build.gradle
View File

@ -3,5 +3,5 @@ repositories {
}
dependencies {
compile 'de.undercouch:gradle-download-task:3.3.0'
compile 'de.undercouch:gradle-download-task:3.4.1'
}

12
buildSrc/src/main/groovy/stamp.groovy
View File

@ -1,12 +0,0 @@
////////////////////////////////////////////////////////////////////////
//
// helper task for checksum-verified downloads
//
class VerifyWithStamp extends de.undercouch.gradle.tasks.download.Verify {
VerifyWithStamp() {
def stamp = new File(temporaryDir, 'stamp')
outputs.file stamp
doLast { stamp.text = '' }
}
}

31
com.ibm.wala.cast.java.test.data/build.gradle
View File

@ -6,23 +6,26 @@ sourceSets.test.java.srcDirs = ['src']
// download JLex
//
def versionedArchive = 'ajaxslt-0.8.1'
def packedArchive = "${versionedArchive}.tar.gz"
task downloadJLex(type: de.undercouch.gradle.tasks.download.Download) {
src 'http://www.cs.princeton.edu/~appel/modern/java/JLex/current/Main.java'
dest 'src/JLex/Main.java'
overwrite false
task downloadJLex {
outputs.file 'src/JLex/Main.java'
doLast {
download {
src 'http://www.cs.princeton.edu/~appel/modern/java/JLex/current/Main.java'
dest outputs.files.singleFile
overwrite true
onlyIfModified true
useETag true
}
verifyChecksum {
src outputs.files.singleFile
checksum 'fe0cff5db3e2f0f5d67a153cf6c783af'
}
}
}
task cleanDownloadJLex(type: Delete) {
delete downloadJLex.dest.parent
delete files(downloadJLex).singleFile.parent
}
task verifyJLex(type: VerifyWithStamp, dependsOn: downloadJLex) {
src downloadJLex.dest
checksum 'fe0cff5db3e2f0f5d67a153cf6c783af'
}
compileTestJava.dependsOn verifyJLex
compileTestJava.dependsOn downloadJLex
clean.dependsOn cleanDownloadJLex

28
com.ibm.wala.cast.js.nodejs/build.gradle
View File

@ -10,19 +10,25 @@ dependencies {
)
}
task downloadNodeJS(type: de.undercouch.gradle.tasks.download.Download) {
src 'https://api.github.com/repos/nodejs/node/zipball/0a604e92e258c5ee2752d763e50721e35053f135'
dest new File(temporaryDir, 'nodejs.zip')
overwrite false
task downloadNodeJS {
outputs.file "$temporaryDir/nodejs.zip"
doLast {
download {
src 'https://api.github.com/repos/nodejs/node/zipball/0a604e92e258c5ee2752d763e50721e35053f135'
dest outputs.files.singleFile
overwrite true
onlyIfModified true
useETag true
}
verifyChecksum {
src outputs.files.singleFile
checksum '33c5ba7a5d45644e70d268d8ad3e57df'
}
}
}
task verifyNodeJS(type: VerifyWithStamp, dependsOn: downloadNodeJS) {
src files(downloadNodeJS).singleFile
checksum '33c5ba7a5d45644e70d268d8ad3e57df'
}
task unpackNodeJSLib(type: Copy, dependsOn: verifyNodeJS) {
from(zipTree(verifyNodeJS.src)) {
task unpackNodeJSLib(type: Copy, dependsOn: downloadNodeJS) {
from(zipTree(files(downloadNodeJS).singleFile)) {
include 'nodejs-node-0a604e9/lib/*.js'
eachFile {
relativePath new RelativePath(!directory, relativePath.lastName)

30
com.ibm.wala.cast.js.test.data/build.gradle
View File

@ -1,22 +1,26 @@
apply plugin: 'base'
task downloadAjaxslt(type: de.undercouch.gradle.tasks.download.Download) {
task downloadAjaxslt {
def version = '0.8.1'
ext {
versionedArchive = "ajaxslt-$version"
def versionedArchive = "ajaxslt-${version}.tar.gz"
outputs.file "$temporaryDir/$versionedArchive"
doLast {
download {
src "https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/ajaxslt/$versionedArchive"
dest outputs.files.singleFile
overwrite true
onlyIfModified true
useETag true
}
verifyChecksum {
src outputs.files.singleFile
checksum 'c995abe3310a401bb4db7f28a6409756'
}
}
src "https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/ajaxslt/${versionedArchive}.tar.gz"
dest temporaryDir
overwrite false
}
task verifyAjaxslt(type: VerifyWithStamp, dependsOn: downloadAjaxslt) {
src files(downloadAjaxslt).singleFile
checksum 'c995abe3310a401bb4db7f28a6409756'
}
task unpackAjaxslt(type: Sync, dependsOn: verifyAjaxslt) {
from(tarTree(verifyAjaxslt.src)) {
task unpackAjaxslt(type: Sync, dependsOn: downloadAjaxslt) {
from(tarTree(files(downloadAjaxslt).singleFile)) {
eachFile {
def newSegments = relativePath.segments[1 .. -1] as String[]
relativePath new RelativePath(!directory, newSegments)

102
com.ibm.wala.core.testdata/build.gradle
View File

@ -5,40 +5,42 @@ eclipse.project.natures 'org.eclipse.pde.PluginNature'
sourceSets.test.java.srcDirs = ['src']
////////////////////////////////////////////////////////////////////////
//
// download and/or create extra bundled jar archives
//
import de.undercouch.gradle.tasks.download.*
////////////////////////////////////////////////////////////////////////
//
// download and extract "bcel-5.2.jar"
//
task downloadBcel(type: Download) {
src 'http://archive.apache.org/dist/jakarta/bcel/binaries/bcel-5.2.tar.gz'
dest temporaryDir
overwrite false
task downloadBcel {
ext {
version = '5.2'
}
def archive = "bcel-${version}.tar.gz"
outputs.file "$temporaryDir/$archive"
doLast {
download {
src "http://archive.apache.org/dist/jakarta/bcel/binaries/$archive"
dest outputs.files.singleFile
overwrite true
onlyIfModified true
useETag false // server does not provide ETags
}
verifyChecksum {
src outputs.files.singleFile
checksum '19bffd7f217b0eae415f1ef87af2f0bc'
}
}
}
task verifyBcel(type: VerifyWithStamp, dependsOn: downloadBcel) {
src files(downloadBcel).singleFile
checksum '19bffd7f217b0eae415f1ef87af2f0bc'
}
task extractBcel(type: Copy, dependsOn: verifyBcel) {
from(tarTree(verifyBcel.src)) {
include 'bcel-5.2/bcel-5.2.jar'
task extractBcel(type: Copy, dependsOn: downloadBcel) {
from(tarTree(files(downloadBcel).singleFile)) {
include "bcel-${downloadBcel.version}/bcel-${downloadBcel.version}.jar"
eachFile {
relativePath new RelativePath(!directory, relativePath.lastName)
}
}
into projectDir
includeEmptyDirs false
outputs.files 'bcel-5.2.jar'
outputs.file "bcel-${downloadBcel.version}.jar"
}
task cleanExtractBcel(type: Delete) {
@ -54,21 +56,28 @@ clean.dependsOn cleanExtractBcel
// download "java-cup-11a.jar"
//
task downloadJavaCup(type: Download) {
src 'http://www2.cs.tum.edu/projects/cup/java-cup-11a.jar'
dest projectDir
overwrite false
task downloadJavaCup {
def archive = 'java-cup-11a.jar'
outputs.file "$projectDir/$archive"
doLast {
download {
src "http://www2.cs.tum.edu/projects/cup/$archive"
dest outputs.files.singleFile
overwrite true
onlyIfModified true
useETag true
}
verifyChecksum {
src outputs.files.singleFile
checksum '2bda8c40abd0cbc295d3038643d6e4ec'
}
}
}
task cleanDownloadJavaCup(type: Delete) {
delete downloadJavaCup
}
task verifyJavaCup(type: VerifyWithStamp, dependsOn: downloadJavaCup) {
src files(downloadJavaCup).singleFile
checksum '2bda8c40abd0cbc295d3038643d6e4ec'
}
clean.dependsOn cleanDownloadJavaCup
@ -97,23 +106,28 @@ clean.dependsOn cleanCollectJLex
// generate "hello_hash.jar"
//
task downloadOcamlJava(type: Download) {
task downloadOcamlJava {
def version = '2.0-alpha1'
ext {
basename = "ocamljava-$version"
ext.basename = "ocamljava-$version"
def archive = "${basename}.tar.gz"
outputs.file "$temporaryDir/$archive"
doLast {
download {
src "http://www.ocamljava.org/downloads/download.php?version=$version-bin"
dest outputs.files.singleFile
overwrite true
onlyIfModified true
useETag true
}
verifyChecksum {
src outputs.files.singleFile
checksum '45feec6e3889f5073a39c2c4c84878d1'
}
}
src "http://www.ocamljava.org/downloads/download.php?version=$version-bin"
dest new File(temporaryDir, "${basename}.tar.gz")
overwrite false
}
task verifyOcamlJava(type: VerifyWithStamp, dependsOn: downloadOcamlJava) {
src files(downloadOcamlJava).singleFile
checksum '45feec6e3889f5073a39c2c4c84878d1'
}
task unpackOcamlJava(type: Sync, dependsOn: verifyOcamlJava) {
from tarTree(verifyOcamlJava.src)
task unpackOcamlJava(type: Sync, dependsOn: downloadOcamlJava) {
from tarTree(files(downloadOcamlJava).singleFile)
into temporaryDir
}
@ -191,8 +205,8 @@ afterEclipseBuildshipImport {
collectJLex,
collectTestData,
collectTestDataA,
downloadJavaCup,
extractBcel,
generateHelloHash,
verifyJavaCup,
)
}

1
com.ibm.wala.core.tests/build.gradle
View File

@ -25,7 +25,6 @@ processTestResources {
def testdata = project(':com.ibm.wala.core.testdata')
dependsOn testdata.compileTestJava
dependsOn testdata.extractBcel
dependsOn testdata.verifyJavaCup
from testdata.collectJLex
from testdata.collectTestData

111
com.ibm.wala.dalvik.test/build.gradle
View File

@ -6,8 +6,6 @@ sourceSets.test {
]
}
import de.undercouch.gradle.tasks.download.Download
task cloneDroidBench(type: Exec) {
// TODO: download to somewhere private to this build tree; update code that looks in "/tmp" accordingly
def destDir = file('/tmp/DroidBench')
@ -18,88 +16,108 @@ task cloneDroidBench(type: Exec) {
commandLine 'git', 'clone', "--branch=$inputs.properties.branch", '--config=advice.detachedHead=false', '--depth=1', '--quiet', inputs.properties.repository, destDir
}
task downloadAndroidSdk(type: Download) {
task downloadAndroidSdk {
def sdkOs
def sdkChecksum
switch (System.getProperty('os.name')) {
case ~/Linux/:
sdkOs = 'linux'
ext.checksum = '444e22ce8ca0f67353bda4b85175ed3731cae3ffa695ca18119cbacef1c1bea0'
sdkChecksum = '444e22ce8ca0f67353bda4b85175ed3731cae3ffa695ca18119cbacef1c1bea0'
break
case ~/Mac OS X/:
sdkOs = 'darwin'
ext.checksum = '4a81754a760fce88cba74d69c364b05b31c53d57b26f9f82355c61d5fe4b9df9'
sdkChecksum = '4a81754a760fce88cba74d69c364b05b31c53d57b26f9f82355c61d5fe4b9df9'
break
case ~/Windows.*/:
sdkOs = 'windows'
ext.checksum = '7f6037d3a7d6789b4fdc06ee7af041e071e9860c51f66f7a4eb5913df9871fd2'
sdkChecksum = '7f6037d3a7d6789b4fdc06ee7af041e071e9860c51f66f7a4eb5913df9871fd2'
break
}
src "https://dl.google.com/android/repository/sdk-tools-$sdkOs-3859397.zip"
dest temporaryDir
overwrite false
def archive = "sdk-tools-$sdkOs-3859397.zip"
outputs.file "$temporaryDir/$archive"
doLast {
download {
src "https://dl.google.com/android/repository/$archive"
dest outputs.files.singleFile
overwrite true
onlyIfModified true
useETag true
}
verifyChecksum {
src outputs.files.singleFile
algorithm 'SHA-256'
checksum sdkChecksum
}
}
}
task verifyAndroidSdk(type: VerifyWithStamp, dependsOn: downloadAndroidSdk) {
src files(downloadAndroidSdk).singleFile
algorithm 'SHA-256'
checksum downloadAndroidSdk.checksum
}
task unpackAndroidSdk(type: Sync, dependsOn: verifyAndroidSdk) {
from zipTree(verifyAndroidSdk.src)
task unpackAndroidSdk(type: Sync, dependsOn: downloadAndroidSdk) {
from zipTree(files(downloadAndroidSdk).singleFile)
into temporaryDir
}
// TODO: factor out common code in installAndroidBuildTools and installAndroidPlatforms
task installAndroidBuildTools(type: Exec, dependsOn: unpackAndroidSdk) {
def manager = "${unpackAndroidSdk.outputs.files.singleFile}/tools/bin/sdkmanager"
def androidSdk = files(unpackAndroidSdk).singleFile
def manager = "$androidSdk/tools/bin/sdkmanager"
ext.version = '26.0.2'
commandLine 'sh', '-ceu', "yes 2>/dev/null | $manager build-tools\\;$version >/dev/null"
def androidSdk = unpackAndroidSdk.outputs.files.singleFile
inputs.dir "$androidSdk/tools"
outputs.dir "$androidSdk/build-tools"
}
task copyDxJar(type: Sync, dependsOn: installAndroidBuildTools) {
from "${installAndroidBuildTools.outputs.files.singleFile}/${installAndroidBuildTools.version}/lib/dx.jar"
from "${files(installAndroidBuildTools).singleFile}/${installAndroidBuildTools.version}/lib/dx.jar"
into 'lib'
}
// TODO: factor out common code in installAndroidBuildTools and installAndroidPlatforms
task installAndroidPlatforms(type: Exec, dependsOn: unpackAndroidSdk) {
def manager = "${unpackAndroidSdk.outputs.files.singleFile}/tools/bin/sdkmanager"
def manager = "${files(unpackAndroidSdk).singleFile}/tools/bin/sdkmanager"
ext.version = "android-${installAndroidBuildTools.version.tokenize('.')[0]}"
commandLine 'sh', '-ceu', "yes 2>/dev/null | $manager platforms\\;$version >/dev/null"
def androidSdk = unpackAndroidSdk.outputs.files.singleFile
def androidSdk = files(unpackAndroidSdk).singleFile
inputs.dir "$androidSdk/tools"
outputs.dir "$androidSdk/platforms"
}
task copyAndroidJar(type: Sync, dependsOn: installAndroidPlatforms) {
from "${installAndroidPlatforms.outputs.files.singleFile}/${installAndroidPlatforms.version}/android.jar"
from "${files(installAndroidPlatforms).singleFile}/${installAndroidPlatforms.version}/android.jar"
into temporaryDir
}
task downloadSampleCup(type: Download) {
src 'http://www.cc.gatech.edu/gvu/people/faculty/hudson/java_cup/classes.v0.9e/java_cup/parser.cup'
dest 'data/sample.cup'
overwrite false
task downloadSampleCup {
outputs.file 'data/sample.cup'
doLast {
download {
src 'http://www.cc.gatech.edu/gvu/people/faculty/hudson/java_cup/classes.v0.9e/java_cup/parser.cup'
dest outputs.files.singleFile
overwrite true
onlyIfModified true
useETag true
}
verifyChecksum {
src outputs.files.singleFile
checksum '76b549e7c6e802b811a374248175ecf4'
}
}
}
task verifySampleCup(type: VerifyWithStamp, dependsOn: downloadSampleCup) {
src files(downloadSampleCup).singleFile
checksum '76b549e7c6e802b811a374248175ecf4'
}
task downloadSampleLex(type: Download) {
src 'https://www.cs.princeton.edu/~appel/modern/java/JLex/current/sample.lex'
dest 'data'
overwrite false
}
task verifySampleLex(type: VerifyWithStamp, dependsOn: downloadSampleLex) {
src files(downloadSampleLex).singleFile
checksum 'ae887758b2657981d023a72a165da830'
task downloadSampleLex {
outputs.file 'data/sample.lex'
doLast {
download {
src 'https://www.cs.princeton.edu/~appel/modern/java/JLex/current/sample.lex'
dest outputs.files.singleFile
overwrite true
onlyIfModified true
useETag true
}
verifyChecksum {
src outputs.files.singleFile
checksum 'ae887758b2657981d023a72a165da830'
}
}
}
clean.dependsOn cleanCopyDxJar
@ -110,29 +128,24 @@ dependencies {
testCompile(
'junit:junit:4.11',
'org.osgi:org.osgi.core:4.2.0',
files("${copyDxJar.outputs.files.singleFile}/dx.jar"),
files("${files(copyDxJar).singleFile}/dx.jar"),
project(':com.ibm.wala.core'),
project(':com.ibm.wala.dalvik'),
project(':com.ibm.wala.shrike'),
project(':com.ibm.wala.util'),
project(configuration: 'testArchives', path: ':com.ibm.wala.core.tests'),
)
testRuntime files("${copyAndroidJar.outputs.files.singleFile}/android.jar")
testRuntime files("${files(copyAndroidJar).singleFile}/android.jar")
}
processTestResources {
dependsOn cloneDroidBench
dependsOn verifySampleCup
from copyAndroidJar
from downloadSampleCup
dependsOn verifySampleLex
from downloadSampleLex
from copyAndroidJar
def testdata = project(':com.ibm.wala.core.testdata')
dependsOn testdata.verifyJavaCup
from testdata.collectJLex
from testdata.collectTestDataA
from testdata.downloadJavaCup