From a7cef8e77abc91bab47d557ac4485fe4801fa175 Mon Sep 17 00:00:00 2001 From: Ben Liblit Date: Sat, 4 Aug 2018 04:08:12 -0500 Subject: [PATCH 1/4] Declare a task's outputs, enabling incremental build and caching This specific task runs an external command, and we consider the task successful if that command exits without error. We don't actually examine the stdout or stderr of the command being run. However, it is still useful to log the stdout and stderr to a file, and to declare that file to be the output of the task. Otherwise, the task has no declared outputs at all. A task with no outputs is ineligible for caching and is always considered to be out-of-date. squash! Declare a task's outputs, enabling incremental build and caching --- com.ibm.wala.cast.test/build.gradle | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/com.ibm.wala.cast.test/build.gradle b/com.ibm.wala.cast.test/build.gradle index 33469953a..e9d0ceb2f 100644 --- a/com.ibm.wala.cast.test/build.gradle +++ b/com.ibm.wala.cast.test/build.gradle @@ -111,6 +111,15 @@ model { // all combined as a colon-delimited path list args pathElements.join(':') + + // log output to file, although we don't validate it + final def outFile = file("$temporaryDir/stdout-and-stderr.log") + outputs.file outFile + doFirst { + final def fileStream = new FileOutputStream(outFile) + standardOutput fileStream + errorOutput fileStream + } } check.dependsOn checkSmoke_main From 8324a9cb566dffb418781c07e98c76c108b6f833 Mon Sep 17 00:00:00 2001 From: Ben Liblit Date: Fri, 3 Aug 2018 16:35:32 -0500 Subject: [PATCH 2/4] Ignore downloaded "kawa-chess" subdirectory This is not our source code. It's just a clone of someone else's Git repository. Maven or Gradle will recreate it when asked to do so. --- com.ibm.wala.core.testdata/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 com.ibm.wala.core.testdata/.gitignore diff --git a/com.ibm.wala.core.testdata/.gitignore b/com.ibm.wala.core.testdata/.gitignore new file mode 100644 index 000000000..598c83f99 --- /dev/null +++ b/com.ibm.wala.core.testdata/.gitignore @@ -0,0 +1 @@ +/kawa-chess/ From febf20145e5fe495764c9e476b6ef0c803fe32fe Mon Sep 17 00:00:00 2001 From: Ben Liblit Date: Sat, 4 Aug 2018 04:37:41 -0500 Subject: [PATCH 3/4] Rework Kawa download and build tasks to be more Gradle'y All Kawa-related downloads now use our existing VerifiedDownload task class. This gives us Gradle-integrated progress reporting, incremental build support, build caching, correct dependencies, etc. Using Kawa to compile Scheme into bytecode now also has proper dependency management, incremental build support, and build caching. Same goes for bundling these compiled bytecode files into jar archives for later use in regression tests. Also, when downloading kawa-chess, grab a specific commit hash rather than whatever is the most recent master commit. If this project changes in the future, we don't want our tests to break unexpectedly. Perhaps we'd want to pick up any new kawa-chess commits; perhaps not. Either way, that should be a conscious decision rather than something that can happen behind our backs. --- buildSrc/build.gradle | 2 +- com.ibm.wala.core.testdata/build.gradle | 143 ++++++++++-------------- 2 files changed, 61 insertions(+), 84 deletions(-) diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index f038c25f3..9471c529c 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -3,5 +3,5 @@ repositories { } dependencies { - compile 'de.undercouch:gradle-download-task:3.4.1' + compile 'de.undercouch:gradle-download-task:3.4.3' } diff --git a/com.ibm.wala.core.testdata/build.gradle b/com.ibm.wala.core.testdata/build.gradle index afacc1833..ce75e9afb 100644 --- a/com.ibm.wala.core.testdata/build.gradle +++ b/com.ibm.wala.core.testdata/build.gradle @@ -1,8 +1,3 @@ -buildscript { - repositories { mavenCentral() } - dependencies { classpath 'org.ajoberstar:gradle-git:0.2.3' } -} - plugins { id 'eclipse' } @@ -11,94 +6,90 @@ eclipse.project.natures 'org.eclipse.pde.PluginNature' sourceSets.test.java.srcDirs = ['src'] + //////////////////////////////////////////////////////////////////////// // // download and extract kawa 3.0 "kawa.jar" // -task downloadKawa(type: Exec) { - commandLine "wget", "ftp://ftp.gnu.org/pub/gnu/kawa/kawa-3.0.zip" - workingDir "$temporaryDir" - def destination = file("$temporaryDir/kawa-3.0.zip") - enabled = !destination.exists() //to clone only once +task downloadKawa(type: VerifiedDownload) { + ext.version = '3.0' + final def archive = "kawa-${version}.zip" + src "https://ftp.gnu.org/pub/gnu/kawa/$archive" + dest "$temporaryDir/$archive" + checksum '2713e6dfb939274ba3b1d36daea68436' } -task extractKawa(type: Copy, dependsOn: downloadKawa) { - from(zipTree("${downloadKawa.temporaryDir}/kawa-3.0.zip")) { - include "kawa-3.0/lib/kawa.jar" +task extractKawa(type: Sync, dependsOn: downloadKawa) { + from(zipTree(downloadKawa.dest)) { + include "kawa-${downloadKawa.version}/lib/kawa.jar" eachFile { relativePath new RelativePath(!directory, relativePath.lastName) } } - into projectDir + into temporaryDir includeEmptyDirs false - outputs.file 'kawa.jar' - def destination = file('kawa.jar') - enabled = !destination.exists() //to clone only once + outputs.file "$temporaryDir/kawa.jar" } -task cleanExtractKawa(type: Delete) { - delete files(extractKawa)[0] -} - -clean.dependsOn cleanExtractKawa //////////////////////////////////////////////////////////////////////// // -// build kawa chess +// specialized task for Kawa compilation into jar archive // -import org.ajoberstar.gradle.git.tasks.* -task getKawaChess(type: GitClone) { - def destination = file("kawa-chess") - uri = "https://github.com/ttu-fpclub/kawa-chess" - destinationPath = destination - bare = false - enabled = !destination.exists() //to clone only once +@CacheableTask +class CompileKawaJar extends Jar { + + @Nested final def compile = project.task("compileKawaInto${name.capitalize()}", type: JavaExec) { + dependsOn(project.extractKawa) + final def kawaJar = project.files(project.extractKawa)[0] + classpath kawaJar + main 'kawa.repl' + + args '-d', temporaryDir + outputs.dir temporaryDir + + logging.captureStandardError LogLevel.INFO + args '--main', '-C' + } + + CompileKawaJar() { + compile.dependsOn(dependsOn) + from compile + version null + } + + void setArgs(Object... args) { + compile.args args + } } -task compileChessFiles(type: JavaExec) { - def kawaImg = file('kawa-chess/img.scm') - def kawaPos = file('kawa-chess/pos.scm') - def kawaChess = file('kawa-chess/chess.scm') - def kawaGui = file('kawa-chess/gui.scm') - inputs.file kawaImg - inputs.file kawaPos - inputs.file kawaChess - inputs.file kawaGui - workingDir = file('kawa-chess') +//////////////////////////////////////////////////////////////////////// +// +// download, unpack, and build kawa chess +// - def kawaJar = new File("kawa.jar") - inputs.file kawaJar - classpath kawaJar - - main 'kawa.repl' - args '-C', 'img.scm', 'pos.scm', 'chess.scm', 'gui.scm' +task downloadKawaChess(type: VerifiedDownload) { + ext.commitHash = 'f1d2dcc707a1ef19dc159e2eaee5aecc8a41d7a8' + src "https://github.com/ttu-fpclub/kawa-chess/archive/${commitHash}.zip" + dest "$temporaryDir/kawa-chess.zip" + checksum 'cf29613d2be5f476a475ee28b4df9d9e' } -compileChessFiles.dependsOn extractKawa -compileChessFiles.dependsOn getKawaChess - -task compileChessMain(type: JavaExec, dependsOn: compileChessFiles) { - def kawaJar = new File("kawa.jar") - inputs.file kawaJar - classpath kawaJar - - workingDir = file("kawa-chess") - - def kawaMain = file('kawa-chess/main.scm') - inputs.file kawaMain - - main 'kawa.repl' - args '--main', '-C', 'main.scm' +task unpackKawaChess(type: Sync, dependsOn: downloadKawaChess) { + from zipTree(downloadKawaChess.dest) + into temporaryDir + ext.top = "$destinationDir/kawa-chess-${downloadKawaChess.commitHash}" } -task buildChessJar(type: Jar, dependsOn: compileChessMain) { - from file('kawa-chess') +task buildChessJar(type: CompileKawaJar, dependsOn: unpackKawaChess) { + final def schemePath = { "$unpackKawaChess.top/${it}.scm" } + final def schemeFiles = files(['chess', 'gui', 'img', 'main', 'pos'].collect(schemePath)) + args schemePath('main') + inputs.files schemeFiles baseName 'kawachess' - version null - destinationDir projectDir } @@ -107,25 +98,11 @@ task buildChessJar(type: Jar, dependsOn: compileChessMain) { // build the kawa test jar // -task compileKawaTestMain(type: JavaExec, dependsOn: extractKawa) { - def kawaJar = new File("kawa.jar") - inputs.file kawaJar - classpath kawaJar - - workingDir file('kawasrc') - - def kawaMain = file('kawasrc/test.scm') - inputs.file kawaMain - - main 'kawa.repl' - args '--main', '-C', 'test.scm' -} - -task buildKawaTestJar(type: Jar, dependsOn: compileKawaTestMain) { - from file('kawasrc') +task buildKawaTestJar(type: CompileKawaJar) { + final def schemeFile = file('kawasrc/test.scm') + args schemeFile + inputs.files schemeFile baseName 'kawatest' - version null - destinationDir projectDir } From e4bb8f22468bcc255c579a011c52efe84d0e4f2e Mon Sep 17 00:00:00 2001 From: Ben Liblit Date: Sat, 4 Aug 2018 05:15:08 -0500 Subject: [PATCH 4/4] Remove redundant "afterEclipseBuildshipImport" dependencies Every dependency task listed here is already a dependency of at least one subproject's "processTestResources" task, and each "processTestResources" task already depends on the corresponding "afterEclipseBuildshipImport" task. So listing these tasks here too is unnecessary. --- com.ibm.wala.core.testdata/build.gradle | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/com.ibm.wala.core.testdata/build.gradle b/com.ibm.wala.core.testdata/build.gradle index ce75e9afb..df80b911d 100644 --- a/com.ibm.wala.core.testdata/build.gradle +++ b/com.ibm.wala.core.testdata/build.gradle @@ -270,22 +270,3 @@ task cleanColllectTestDataA(type: Delete) { } clean.dependsOn cleanCollectTestDataA - - -//////////////////////////////////////////////////////////////////////// -// -// help Eclipse build these extra jars when needed -// - -afterEclipseBuildshipImport { - dependsOn( - collectJLex, - collectTestData, - collectTestDataA, - downloadJavaCup, - extractBcel, - generateHelloHashJar, - buildChessJar, - buildKawaTestJar, - ) -}