From ba455f4737dc04846abc66394d536f3400e98179 Mon Sep 17 00:00:00 2001 From: Ben Liblit Date: Mon, 5 Mar 2018 12:33:29 -0600 Subject: [PATCH] Update gradle-download-plugin and use some of its new features We now download and verify checksums as a single task, rather than as two separate tasks. This simplifies other task dependencies, since we no longer have a checksum-verified "stamp" file separate from the download itself. Unfortunately the combined task now has a significant amount of repeated boilerplate. I'm hoping to refactor that all out into a custom task class, but haven't yet figured out the details: . We now also use ETags to be smarter about when a fresh download is or is not actually needed. I think there are still opportunities for improved caching here, but this is a step in the right direction. --- build.gradle | 2 +- buildSrc/build.gradle | 2 +- buildSrc/src/main/groovy/stamp.groovy | 12 -- com.ibm.wala.cast.java.test.data/build.gradle | 31 ++--- com.ibm.wala.cast.js.nodejs/build.gradle | 28 +++-- com.ibm.wala.cast.js.test.data/build.gradle | 30 +++-- com.ibm.wala.core.testdata/build.gradle | 102 +++++++++------- com.ibm.wala.core.tests/build.gradle | 1 - com.ibm.wala.dalvik.test/build.gradle | 111 ++++++++++-------- 9 files changed, 173 insertions(+), 146 deletions(-) delete mode 100644 buildSrc/src/main/groovy/stamp.groovy diff --git a/build.gradle b/build.gradle index 480c190de..3429a45c9 100644 --- a/build.gradle +++ b/build.gradle @@ -6,7 +6,7 @@ plugins { id 'com.diffplug.gradle.p2.asmaven' version '3.10.0' id 'com.github.hauner.jarTest' version '1.0.1' apply false - id 'de.undercouch.download' apply false + id 'de.undercouch.download' id 'nebula.lint' version '8.3.1' id "nebula.source-jar" version '7.0.1' apply false } diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 77cd72bc3..f038c25f3 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -3,5 +3,5 @@ repositories { } dependencies { - compile 'de.undercouch:gradle-download-task:3.3.0' + compile 'de.undercouch:gradle-download-task:3.4.1' } diff --git a/buildSrc/src/main/groovy/stamp.groovy b/buildSrc/src/main/groovy/stamp.groovy deleted file mode 100644 index 5d2142265..000000000 --- a/buildSrc/src/main/groovy/stamp.groovy +++ /dev/null @@ -1,12 +0,0 @@ -//////////////////////////////////////////////////////////////////////// -// -// helper task for checksum-verified downloads -// - -class VerifyWithStamp extends de.undercouch.gradle.tasks.download.Verify { - VerifyWithStamp() { - def stamp = new File(temporaryDir, 'stamp') - outputs.file stamp - doLast { stamp.text = '' } - } -} diff --git a/com.ibm.wala.cast.java.test.data/build.gradle b/com.ibm.wala.cast.java.test.data/build.gradle index 75fdf2b8a..840d92fd0 100644 --- a/com.ibm.wala.cast.java.test.data/build.gradle +++ b/com.ibm.wala.cast.java.test.data/build.gradle @@ -6,23 +6,26 @@ sourceSets.test.java.srcDirs = ['src'] // download JLex // -def versionedArchive = 'ajaxslt-0.8.1' -def packedArchive = "${versionedArchive}.tar.gz" - -task downloadJLex(type: de.undercouch.gradle.tasks.download.Download) { - src 'http://www.cs.princeton.edu/~appel/modern/java/JLex/current/Main.java' - dest 'src/JLex/Main.java' - overwrite false +task downloadJLex { + outputs.file 'src/JLex/Main.java' + doLast { + download { + src 'http://www.cs.princeton.edu/~appel/modern/java/JLex/current/Main.java' + dest outputs.files.singleFile + overwrite true + onlyIfModified true + useETag true + } + verifyChecksum { + src outputs.files.singleFile + checksum 'fe0cff5db3e2f0f5d67a153cf6c783af' + } + } } task cleanDownloadJLex(type: Delete) { - delete downloadJLex.dest.parent + delete files(downloadJLex).singleFile.parent } -task verifyJLex(type: VerifyWithStamp, dependsOn: downloadJLex) { - src downloadJLex.dest - checksum 'fe0cff5db3e2f0f5d67a153cf6c783af' -} - -compileTestJava.dependsOn verifyJLex +compileTestJava.dependsOn downloadJLex clean.dependsOn cleanDownloadJLex diff --git a/com.ibm.wala.cast.js.nodejs/build.gradle b/com.ibm.wala.cast.js.nodejs/build.gradle index 28a2a9f15..24a603a04 100644 --- a/com.ibm.wala.cast.js.nodejs/build.gradle +++ b/com.ibm.wala.cast.js.nodejs/build.gradle @@ -10,19 +10,25 @@ dependencies { ) } -task downloadNodeJS(type: de.undercouch.gradle.tasks.download.Download) { - src 'https://api.github.com/repos/nodejs/node/zipball/0a604e92e258c5ee2752d763e50721e35053f135' - dest new File(temporaryDir, 'nodejs.zip') - overwrite false +task downloadNodeJS { + outputs.file "$temporaryDir/nodejs.zip" + doLast { + download { + src 'https://api.github.com/repos/nodejs/node/zipball/0a604e92e258c5ee2752d763e50721e35053f135' + dest outputs.files.singleFile + overwrite true + onlyIfModified true + useETag true + } + verifyChecksum { + src outputs.files.singleFile + checksum '33c5ba7a5d45644e70d268d8ad3e57df' + } + } } -task verifyNodeJS(type: VerifyWithStamp, dependsOn: downloadNodeJS) { - src files(downloadNodeJS).singleFile - checksum '33c5ba7a5d45644e70d268d8ad3e57df' -} - -task unpackNodeJSLib(type: Copy, dependsOn: verifyNodeJS) { - from(zipTree(verifyNodeJS.src)) { +task unpackNodeJSLib(type: Copy, dependsOn: downloadNodeJS) { + from(zipTree(files(downloadNodeJS).singleFile)) { include 'nodejs-node-0a604e9/lib/*.js' eachFile { relativePath new RelativePath(!directory, relativePath.lastName) diff --git a/com.ibm.wala.cast.js.test.data/build.gradle b/com.ibm.wala.cast.js.test.data/build.gradle index e658884cb..acb178841 100644 --- a/com.ibm.wala.cast.js.test.data/build.gradle +++ b/com.ibm.wala.cast.js.test.data/build.gradle @@ -1,22 +1,26 @@ apply plugin: 'base' -task downloadAjaxslt(type: de.undercouch.gradle.tasks.download.Download) { +task downloadAjaxslt { def version = '0.8.1' - ext { - versionedArchive = "ajaxslt-$version" + def versionedArchive = "ajaxslt-${version}.tar.gz" + outputs.file "$temporaryDir/$versionedArchive" + doLast { + download { + src "https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/ajaxslt/$versionedArchive" + dest outputs.files.singleFile + overwrite true + onlyIfModified true + useETag true + } + verifyChecksum { + src outputs.files.singleFile + checksum 'c995abe3310a401bb4db7f28a6409756' + } } - src "https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/ajaxslt/${versionedArchive}.tar.gz" - dest temporaryDir - overwrite false } -task verifyAjaxslt(type: VerifyWithStamp, dependsOn: downloadAjaxslt) { - src files(downloadAjaxslt).singleFile - checksum 'c995abe3310a401bb4db7f28a6409756' -} - -task unpackAjaxslt(type: Sync, dependsOn: verifyAjaxslt) { - from(tarTree(verifyAjaxslt.src)) { +task unpackAjaxslt(type: Sync, dependsOn: downloadAjaxslt) { + from(tarTree(files(downloadAjaxslt).singleFile)) { eachFile { def newSegments = relativePath.segments[1 .. -1] as String[] relativePath new RelativePath(!directory, newSegments) diff --git a/com.ibm.wala.core.testdata/build.gradle b/com.ibm.wala.core.testdata/build.gradle index 6639f7e48..d8d34ee3d 100644 --- a/com.ibm.wala.core.testdata/build.gradle +++ b/com.ibm.wala.core.testdata/build.gradle @@ -5,40 +5,42 @@ eclipse.project.natures 'org.eclipse.pde.PluginNature' sourceSets.test.java.srcDirs = ['src'] -//////////////////////////////////////////////////////////////////////// -// -// download and/or create extra bundled jar archives -// - -import de.undercouch.gradle.tasks.download.* - - //////////////////////////////////////////////////////////////////////// // // download and extract "bcel-5.2.jar" // -task downloadBcel(type: Download) { - src 'http://archive.apache.org/dist/jakarta/bcel/binaries/bcel-5.2.tar.gz' - dest temporaryDir - overwrite false +task downloadBcel { + ext { + version = '5.2' + } + def archive = "bcel-${version}.tar.gz" + outputs.file "$temporaryDir/$archive" + doLast { + download { + src "http://archive.apache.org/dist/jakarta/bcel/binaries/$archive" + dest outputs.files.singleFile + overwrite true + onlyIfModified true + useETag false // server does not provide ETags + } + verifyChecksum { + src outputs.files.singleFile + checksum '19bffd7f217b0eae415f1ef87af2f0bc' + } + } } -task verifyBcel(type: VerifyWithStamp, dependsOn: downloadBcel) { - src files(downloadBcel).singleFile - checksum '19bffd7f217b0eae415f1ef87af2f0bc' -} - -task extractBcel(type: Copy, dependsOn: verifyBcel) { - from(tarTree(verifyBcel.src)) { - include 'bcel-5.2/bcel-5.2.jar' +task extractBcel(type: Copy, dependsOn: downloadBcel) { + from(tarTree(files(downloadBcel).singleFile)) { + include "bcel-${downloadBcel.version}/bcel-${downloadBcel.version}.jar" eachFile { relativePath new RelativePath(!directory, relativePath.lastName) } } into projectDir includeEmptyDirs false - outputs.files 'bcel-5.2.jar' + outputs.file "bcel-${downloadBcel.version}.jar" } task cleanExtractBcel(type: Delete) { @@ -54,21 +56,28 @@ clean.dependsOn cleanExtractBcel // download "java-cup-11a.jar" // -task downloadJavaCup(type: Download) { - src 'http://www2.cs.tum.edu/projects/cup/java-cup-11a.jar' - dest projectDir - overwrite false +task downloadJavaCup { + def archive = 'java-cup-11a.jar' + outputs.file "$projectDir/$archive" + doLast { + download { + src "http://www2.cs.tum.edu/projects/cup/$archive" + dest outputs.files.singleFile + overwrite true + onlyIfModified true + useETag true + } + verifyChecksum { + src outputs.files.singleFile + checksum '2bda8c40abd0cbc295d3038643d6e4ec' + } + } } task cleanDownloadJavaCup(type: Delete) { delete downloadJavaCup } -task verifyJavaCup(type: VerifyWithStamp, dependsOn: downloadJavaCup) { - src files(downloadJavaCup).singleFile - checksum '2bda8c40abd0cbc295d3038643d6e4ec' -} - clean.dependsOn cleanDownloadJavaCup @@ -97,23 +106,28 @@ clean.dependsOn cleanCollectJLex // generate "hello_hash.jar" // -task downloadOcamlJava(type: Download) { +task downloadOcamlJava { def version = '2.0-alpha1' - ext { - basename = "ocamljava-$version" + ext.basename = "ocamljava-$version" + def archive = "${basename}.tar.gz" + outputs.file "$temporaryDir/$archive" + doLast { + download { + src "http://www.ocamljava.org/downloads/download.php?version=$version-bin" + dest outputs.files.singleFile + overwrite true + onlyIfModified true + useETag true + } + verifyChecksum { + src outputs.files.singleFile + checksum '45feec6e3889f5073a39c2c4c84878d1' + } } - src "http://www.ocamljava.org/downloads/download.php?version=$version-bin" - dest new File(temporaryDir, "${basename}.tar.gz") - overwrite false } -task verifyOcamlJava(type: VerifyWithStamp, dependsOn: downloadOcamlJava) { - src files(downloadOcamlJava).singleFile - checksum '45feec6e3889f5073a39c2c4c84878d1' -} - -task unpackOcamlJava(type: Sync, dependsOn: verifyOcamlJava) { - from tarTree(verifyOcamlJava.src) +task unpackOcamlJava(type: Sync, dependsOn: downloadOcamlJava) { + from tarTree(files(downloadOcamlJava).singleFile) into temporaryDir } @@ -191,8 +205,8 @@ afterEclipseBuildshipImport { collectJLex, collectTestData, collectTestDataA, + downloadJavaCup, extractBcel, generateHelloHash, - verifyJavaCup, ) } diff --git a/com.ibm.wala.core.tests/build.gradle b/com.ibm.wala.core.tests/build.gradle index 076a2bbba..aef898df9 100644 --- a/com.ibm.wala.core.tests/build.gradle +++ b/com.ibm.wala.core.tests/build.gradle @@ -25,7 +25,6 @@ processTestResources { def testdata = project(':com.ibm.wala.core.testdata') dependsOn testdata.compileTestJava dependsOn testdata.extractBcel - dependsOn testdata.verifyJavaCup from testdata.collectJLex from testdata.collectTestData diff --git a/com.ibm.wala.dalvik.test/build.gradle b/com.ibm.wala.dalvik.test/build.gradle index f3b5bff0a..682e3f7ef 100644 --- a/com.ibm.wala.dalvik.test/build.gradle +++ b/com.ibm.wala.dalvik.test/build.gradle @@ -6,8 +6,6 @@ sourceSets.test { ] } -import de.undercouch.gradle.tasks.download.Download - task cloneDroidBench(type: Exec) { // TODO: download to somewhere private to this build tree; update code that looks in "/tmp" accordingly def destDir = file('/tmp/DroidBench') @@ -18,88 +16,108 @@ task cloneDroidBench(type: Exec) { commandLine 'git', 'clone', "--branch=$inputs.properties.branch", '--config=advice.detachedHead=false', '--depth=1', '--quiet', inputs.properties.repository, destDir } -task downloadAndroidSdk(type: Download) { +task downloadAndroidSdk { def sdkOs + def sdkChecksum switch (System.getProperty('os.name')) { case ~/Linux/: sdkOs = 'linux' - ext.checksum = '444e22ce8ca0f67353bda4b85175ed3731cae3ffa695ca18119cbacef1c1bea0' + sdkChecksum = '444e22ce8ca0f67353bda4b85175ed3731cae3ffa695ca18119cbacef1c1bea0' break case ~/Mac OS X/: sdkOs = 'darwin' - ext.checksum = '4a81754a760fce88cba74d69c364b05b31c53d57b26f9f82355c61d5fe4b9df9' + sdkChecksum = '4a81754a760fce88cba74d69c364b05b31c53d57b26f9f82355c61d5fe4b9df9' break case ~/Windows.*/: sdkOs = 'windows' - ext.checksum = '7f6037d3a7d6789b4fdc06ee7af041e071e9860c51f66f7a4eb5913df9871fd2' + sdkChecksum = '7f6037d3a7d6789b4fdc06ee7af041e071e9860c51f66f7a4eb5913df9871fd2' break } - src "https://dl.google.com/android/repository/sdk-tools-$sdkOs-3859397.zip" - dest temporaryDir - overwrite false + def archive = "sdk-tools-$sdkOs-3859397.zip" + outputs.file "$temporaryDir/$archive" + doLast { + download { + src "https://dl.google.com/android/repository/$archive" + dest outputs.files.singleFile + overwrite true + onlyIfModified true + useETag true + } + verifyChecksum { + src outputs.files.singleFile + algorithm 'SHA-256' + checksum sdkChecksum + } + } } -task verifyAndroidSdk(type: VerifyWithStamp, dependsOn: downloadAndroidSdk) { - src files(downloadAndroidSdk).singleFile - algorithm 'SHA-256' - checksum downloadAndroidSdk.checksum -} - -task unpackAndroidSdk(type: Sync, dependsOn: verifyAndroidSdk) { - from zipTree(verifyAndroidSdk.src) +task unpackAndroidSdk(type: Sync, dependsOn: downloadAndroidSdk) { + from zipTree(files(downloadAndroidSdk).singleFile) into temporaryDir } // TODO: factor out common code in installAndroidBuildTools and installAndroidPlatforms task installAndroidBuildTools(type: Exec, dependsOn: unpackAndroidSdk) { - def manager = "${unpackAndroidSdk.outputs.files.singleFile}/tools/bin/sdkmanager" + def androidSdk = files(unpackAndroidSdk).singleFile + def manager = "$androidSdk/tools/bin/sdkmanager" ext.version = '26.0.2' commandLine 'sh', '-ceu', "yes 2>/dev/null | $manager build-tools\\;$version >/dev/null" - def androidSdk = unpackAndroidSdk.outputs.files.singleFile inputs.dir "$androidSdk/tools" outputs.dir "$androidSdk/build-tools" } task copyDxJar(type: Sync, dependsOn: installAndroidBuildTools) { - from "${installAndroidBuildTools.outputs.files.singleFile}/${installAndroidBuildTools.version}/lib/dx.jar" + from "${files(installAndroidBuildTools).singleFile}/${installAndroidBuildTools.version}/lib/dx.jar" into 'lib' } // TODO: factor out common code in installAndroidBuildTools and installAndroidPlatforms task installAndroidPlatforms(type: Exec, dependsOn: unpackAndroidSdk) { - def manager = "${unpackAndroidSdk.outputs.files.singleFile}/tools/bin/sdkmanager" + def manager = "${files(unpackAndroidSdk).singleFile}/tools/bin/sdkmanager" ext.version = "android-${installAndroidBuildTools.version.tokenize('.')[0]}" commandLine 'sh', '-ceu', "yes 2>/dev/null | $manager platforms\\;$version >/dev/null" - def androidSdk = unpackAndroidSdk.outputs.files.singleFile + def androidSdk = files(unpackAndroidSdk).singleFile inputs.dir "$androidSdk/tools" outputs.dir "$androidSdk/platforms" } task copyAndroidJar(type: Sync, dependsOn: installAndroidPlatforms) { - from "${installAndroidPlatforms.outputs.files.singleFile}/${installAndroidPlatforms.version}/android.jar" + from "${files(installAndroidPlatforms).singleFile}/${installAndroidPlatforms.version}/android.jar" into temporaryDir } -task downloadSampleCup(type: Download) { - src 'http://www.cc.gatech.edu/gvu/people/faculty/hudson/java_cup/classes.v0.9e/java_cup/parser.cup' - dest 'data/sample.cup' - overwrite false +task downloadSampleCup { + outputs.file 'data/sample.cup' + doLast { + download { + src 'http://www.cc.gatech.edu/gvu/people/faculty/hudson/java_cup/classes.v0.9e/java_cup/parser.cup' + dest outputs.files.singleFile + overwrite true + onlyIfModified true + useETag true + } + verifyChecksum { + src outputs.files.singleFile + checksum '76b549e7c6e802b811a374248175ecf4' + } + } } -task verifySampleCup(type: VerifyWithStamp, dependsOn: downloadSampleCup) { - src files(downloadSampleCup).singleFile - checksum '76b549e7c6e802b811a374248175ecf4' -} - -task downloadSampleLex(type: Download) { - src 'https://www.cs.princeton.edu/~appel/modern/java/JLex/current/sample.lex' - dest 'data' - overwrite false -} - -task verifySampleLex(type: VerifyWithStamp, dependsOn: downloadSampleLex) { - src files(downloadSampleLex).singleFile - checksum 'ae887758b2657981d023a72a165da830' +task downloadSampleLex { + outputs.file 'data/sample.lex' + doLast { + download { + src 'https://www.cs.princeton.edu/~appel/modern/java/JLex/current/sample.lex' + dest outputs.files.singleFile + overwrite true + onlyIfModified true + useETag true + } + verifyChecksum { + src outputs.files.singleFile + checksum 'ae887758b2657981d023a72a165da830' + } + } } clean.dependsOn cleanCopyDxJar @@ -110,29 +128,24 @@ dependencies { testCompile( 'junit:junit:4.11', 'org.osgi:org.osgi.core:4.2.0', - files("${copyDxJar.outputs.files.singleFile}/dx.jar"), + files("${files(copyDxJar).singleFile}/dx.jar"), project(':com.ibm.wala.core'), project(':com.ibm.wala.dalvik'), project(':com.ibm.wala.shrike'), project(':com.ibm.wala.util'), project(configuration: 'testArchives', path: ':com.ibm.wala.core.tests'), ) - testRuntime files("${copyAndroidJar.outputs.files.singleFile}/android.jar") + testRuntime files("${files(copyAndroidJar).singleFile}/android.jar") } processTestResources { dependsOn cloneDroidBench - dependsOn verifySampleCup + from copyAndroidJar from downloadSampleCup - - dependsOn verifySampleLex from downloadSampleLex - from copyAndroidJar - def testdata = project(':com.ibm.wala.core.testdata') - dependsOn testdata.verifyJavaCup from testdata.collectJLex from testdata.collectTestDataA from testdata.downloadJavaCup