/* * (C) Copyright 2010-2015 SAP SE. * * All rights reserved. This program and the accompanying materials * are made available under the terms of the Eclipse Public License v1.0 * which accompanies this distribution, and is available at * http://www.eclipse.org/legal/epl-v10.html * */ package com.logicalhacking.dasca.crosslanguage.builder import java.io.File import java.io.FileNotFoundException import java.lang.ref.ReferenceQueue.Null import scala.collection.JavaConverters._ import scala.collection.immutable.StringOps import scala.xml.XML import org.apache.commons.io.FileUtils import org.apache.commons.io.filefilter.IOFileFilter import org.apache.commons.io.filefilter.TrueFileFilter import com.ibm.wala.classLoader.IClass import com.ibm.wala.classLoader.IMethod import com.ibm.wala.dalvik.classLoader.DexIRFactory import com.ibm.wala.dalvik.ipa.callgraph.impl.AndroidEntryPoint import com.ibm.wala.dalvik.ipa.callgraph.impl.DexEntryPoint import com.ibm.wala.dalvik.util.AndroidEntryPointLocator import com.ibm.wala.dalvik.util.AndroidEntryPointLocator.LocatorFlags import com.ibm.wala.ipa.callgraph.AnalysisCache import com.ibm.wala.ipa.callgraph.AnalysisCacheImpl import com.ibm.wala.ipa.callgraph.AnalysisOptions import com.ibm.wala.ipa.callgraph.AnalysisOptions.ReflectionOptions import com.ibm.wala.ipa.callgraph.AnalysisScope import com.ibm.wala.ipa.callgraph.CallGraph import com.ibm.wala.ipa.callgraph.Entrypoint import com.logicalhacking.dasca.crosslanguage.util.Util import com.ibm.wala.ipa.cha.ClassHierarchyFactory import com.ibm.wala.ipa.cha.ClassHierarchy import brut.androlib.ApkDecoder import spray.json._ import spray.json.DefaultJsonProtocol._ import com.ibm.wala.cast.js.html.WebPageLoaderFactory import com.ibm.wala.cast.js.translator.CAstRhinoTranslatorFactory import com.ibm.wala.cast.js.loader.JavaScriptLoaderFactory import java.net.URL import com.ibm.wala.cast.js.loader.JavaScriptLoader import com.ibm.wala.cast.js.html.WebUtil import com.ibm.wala.classLoader.SourceURLModule import com.ibm.wala.classLoader.SourceModule import com.ibm.wala.classLoader.SourceFileModule import com.ibm.wala.classLoader.ModuleEntry import com.ibm.wala.classLoader.Module import scala.concurrent._ import ExecutionContext.Implicits.global import scala.concurrent.duration._ import com.typesafe.scalalogging.Logger import org.slf4j.LoggerFactory import scala.util.Random import com.logicalhacking.dasca.crosslanguage.util.Util import scala.collection.mutable.ListBuffer import com.ibm.wala.cast.js.html.jericho.JerichoHtmlParser import com.ibm.wala.cast.js.html.IdentityUrlResolver import com.ibm.wala.cast.js.html.DefaultSourceExtractor import scala.xml.Elem import org.apache.commons.io.input.BOMInputStream import org.apache.commons.io.ByteOrderMark import scala.util.matching.Regex import java.io.BufferedInputStream import java.io.BufferedReader import java.io.InputStreamReader import com.logicalhacking.dasca.crosslanguage.builder.CordovaCGBuilder._ import com.ibm.wala.ipa.callgraph.impl.{ Util => WalaUtil } import com.logicalhacking.dasca.crosslanguage.util.FileMappingStore import com.logicalhacking.dasca.crosslanguage.util.FileMapRecorder import com.ibm.wala.util.io.TemporaryFile import org.apache.commons.io.IOUtils import com.ibm.wala.util.io.FileUtil import java.nio.file.Files import com.ibm.wala.cast.ir.ssa.AstIRFactory import com.ibm.wala.cast.ipa.callgraph.CAstAnalysisScope import com.ibm.wala.ipa.callgraph.propagation.cfa.ZeroXInstanceKeys import com.ibm.wala.cast.js.ipa.callgraph.JSCallGraphUtil import com.ibm.wala.cast.js.ipa.callgraph.JSZeroOrOneXCFABuilder import java.util.Collections import com.ibm.wala.cast.js.callgraph.fieldbased.OptimisticCallgraphBuilder import com.ibm.wala.util.WalaException import com.ibm.wala.dalvik.util.AndroidAnalysisScope import java.util.jar.JarFile import com.ibm.wala.types.ClassLoaderReference import com.ibm.wala.classLoader.JarFileModule object CordovaCGBuilder { val ExecuteSuffix = "walaexec" val ExclusionFile = new File("jsexclusions.txt") def apply(apk: File): CordovaCGBuilder = { val tmpApkDir = new File(System.getProperty("java.io.tmpdir"), s"${apk.getName}-${Random.alphanumeric.take(16).mkString}") tmpApkDir.deleteOnExit() CordovaCGBuilder(apk, tmpApkDir) } def apply(apk: File, tmpApkDir: File): CordovaCGBuilder = new CordovaCGBuilder(apk, tmpApkDir) } class CordovaCGBuilder(val apk: File, val apkUnzipDir: File) { val PluginsRegex = """(?s).*\.exports\s*=\s*(.+]).*""".r implicit val logger = Logger(LoggerFactory.getLogger(getClass.toString)) var mockCordovaExec = false var filterJavaCallSites = false var replacePluginDefinesAndRequires = false var filterJSFrameworks = false var preciseJS = false var runBuildersInParallel = false def waitForFutures(fut1: Future[Any], fut2: Future[Any]) = { while (fut1.value.isEmpty || fut2.value.isEmpty) { for (value <- fut1.value if value.isFailure) { value.get } for (value <- fut2.value if value.isFailure) { value.get } Thread.sleep(500) } } def createCallGraph: MergedCallGraph = { val mcg = if (runBuildersInParallel) { val javaFuture = Future { Util.time("Creation of Java Call Graph", { createJavaCallGraph }) } val jsFuture = Future { Util.time("Creation of JavaScript Call Graph", { createJavaScriptCallGraph }) } waitForFutures(javaFuture, jsFuture) val javaCG = javaFuture.value.get.get val (jsCG, xml) = jsFuture.value.get.get new MergedCallGraph(javaCG, jsCG, xml) val mcgFuture = for { javaCG <- javaFuture jsCG <- jsFuture } yield new MergedCallGraph(javaCG, jsCG._1, jsCG._2) Await.result(mcgFuture, Duration.Inf) } else { val javaCG = Util.time("Creation of Java Call Graph", { createJavaCallGraph }) val (jsCG, xml) = Util.time("Creation of JavaScript Call Graph", { createJavaScriptCallGraph }) new MergedCallGraph(javaCG, jsCG, xml) } logger.info(s"The Java Call Graph has ${mcg.javaCG.getNumberOfNodes} nodes") logger.info(s"The JavaScript Call Graph has ${mcg.jsCG.getNumberOfNodes} nodes") DalvikLineNumberCalculator.setLineNumbers(apkUnzipDir, mcg.javaCG) logger.info("Connecting cross language calls...") mcg.connect(filterJavaCallSites, filterJSFrameworks) Util.time("Connecting the cross calls", { mcg.connect })(logger) logger.info("The following calls have been found:") for (line <- Util.prettyPrintCrossTargets(mcg.getAllCrossTargets)) logger.info(line) logger.info("End of cross calls") mcg } private def createJavaCallGraph = { val tmpAndroidJar = File.createTempFile("android-dasca", ".jar") tmpAndroidJar.deleteOnExit() if (null != getClass.getClassLoader.getResource("android-dasca.jar")){ TemporaryFile.urlToFile(tmpAndroidJar, getClass.getClassLoader.getResource("android-dasca.jar")) }else{ val msg = "Please install android-dasca.jar in resources directory." logger.error(msg); throw new FileNotFoundException(msg); } val scope = AndroidAnalysisScope.setUpAndroidAnalysisScope(apk.toURI(), getClass.getClassLoader.getResource("javaexclusions.txt").getFile, CordovaCGBuilder.getClass.getClassLoader()) scope.addToScope(ClassLoaderReference.Primordial, new JarFileModule(new JarFile(tmpAndroidJar))) val cha = ClassHierarchyFactory.make(scope) val cache = new AnalysisCacheImpl(new DexIRFactory()) val eps = new AndroidEntryPointLocator(Set( LocatorFlags.INCLUDE_CALLBACKS, LocatorFlags.EP_HEURISTIC, LocatorFlags.CB_HEURISTIC).asJava) val pluginEntryPoints = getPluginEntryPoints(cha) for (ep <- pluginEntryPoints) logger.info(s"Found cordova plugin entry point: $ep") val entryPoints = eps.getEntryPoints(cha).asScala ++ pluginEntryPoints val options = new AnalysisOptions(scope, entryPoints.asJava); options.setReflectionOptions(ReflectionOptions.FULL); val cgb = WalaUtil.makeZeroCFABuilder(com.ibm.wala.classLoader.Language.JAVA, options, cache, cha, scope); cgb.makeCallGraph(options, null); } private def getPluginEntryPoints(cha: ClassHierarchy): List[Entrypoint] = for ( cls <- cha.iterator.asScala.toList; m <- cls.getDeclaredMethods.asScala if !cls.isInterface() && cls.getClassLoader.getName == AnalysisScope.APPLICATION && Util.derivesFromCordovaPlugin(cls.getSuperclass); if m.getName.toString.equals("execute") ) yield new DexEntryPoint(m, cha) private def createJavaScriptCallGraph: (CallGraph, Elem) = { decodeApk(apk); val configXml = XML.loadFile(new File(apkUnzipDir, "/res/xml/config.xml")) val entrypoint = getEntryPoint(configXml) if (!entrypoint.exists()) { logger.error(s"Could not find entrypoint $entrypoint, using an empty call graph ...") return (new EmptyCallGraph(), configXml) } val pluginInfos = getPluginInfos val loaders = new WebPageLoaderFactory(new CAstRhinoTranslatorFactory()) val scripts = makeHtmlScope(entrypoint.toURI().toURL(), loaders, pluginInfos); preprocessApkDir(scripts, pluginInfos) val cache = new AnalysisCacheImpl(AstIRFactory.makeDefaultFactory()) val scope = new CAstAnalysisScope((scripts:List[Module]).toArray, loaders, Collections.singleton(JavaScriptLoader.JS)) val cha = ClassHierarchyFactory.make(scope, loaders, JavaScriptLoader.JS) val roots = JSCallGraphUtil.makeScriptRoots(cha) val options = JSCallGraphUtil.makeOptions(scope, cha, roots) try { com.ibm.wala.cast.util.Util.checkForFrontEndErrors(cha); } catch { case e: WalaException => logger.warn("JavaScript front end error:", e) } val cg = if (preciseJS) { val builder = new JSZeroOrOneXCFABuilder(cha, options, cache, null, null, ZeroXInstanceKeys.ALLOCATIONS, true) builder.makeCallGraph(options) } else { val builder = new OptimisticCallgraphBuilder(cha, JSCallGraphUtil.makeOptions(scope, cha, roots), cache, false) builder.buildCallGraph(roots, null).fst } (cg, configXml) } private def makeHtmlScope(url: URL, loaders: JavaScriptLoaderFactory, pluginInfos: List[PluginInfo]): List[SourceModule] = { JavaScriptLoader.addBootstrapFile(WebUtil.preamble); val lb = new ListBuffer[SourceModule]() lb += new SourceURLModule(CordovaCGBuilder.getClass.getClassLoader.getResource("prologue.js")) lb += new SourceURLModule(CordovaCGBuilder.getClass.getClassLoader.getResource("preamble.js")) for (info <- pluginInfos) lb += new SourceURLModule(info.file.toURI().toURL()) val extractor = new AllScriptsExtractor(apkUnzipDir) lb ++= extractor.extractSources(url, new JerichoHtmlParser, new IdentityUrlResolver).asScala lb ++= extractor.extractedScripts filter { !excludeJS(_) } map { toSourceModule } logger.debug("The following scripts are in the JavaScript scope:") for (scriptUrl <- lb.map { _.getURL }) logger.debug(scriptUrl.toString()) lb.toList } private def toSourceModule(src: URL) = new SourceURLModule(src) { override def getInputStream = new BOMInputStream(src.openConnection().getInputStream(), false, ByteOrderMark.UTF_8, ByteOrderMark.UTF_16LE, ByteOrderMark.UTF_16BE, ByteOrderMark.UTF_32LE, ByteOrderMark.UTF_32BE) } private def decodeApk(apk: File) = { val decoder = new ApkDecoder(); //decoder.setFrameworkDir(new File(System.getProperty("java.io.tmpdir"), s"apktoolframework${Random.alphanumeric.take(16).mkString}").getCanonicalPath) decoder.setApkFile(apk); if (apkUnzipDir.exists()) FileUtils.deleteDirectory(apkUnzipDir); decoder.setOutDir(apkUnzipDir); decoder.decode(); } private def getPluginInfos: List[PluginInfo] = { val lb = new ListBuffer[PluginInfo]() val files = Util.walkTree(apkUnzipDir).filter({ file => file.getName == "cordova_plugins.js" }).toList if (files.isEmpty) { logger.warn("Expected a cordova_plugins.js file, but found none. Assuming there are no plugins.") return lb.toList } val cordovaPluginJs = if (files.size > 1) { logger.warn(s"Found ${files.size} cordova_plugin.js files! Trying to take one in a folder called 'www'.") files.find { Util.hasSomeParentWithName("www", _) } getOrElse (files(0)) } else { files(0) } val jsonArray = FileUtils.readFileToString(cordovaPluginJs) match { case PluginsRegex(jsonArray) => jsonArray case _ => throw new IllegalArgumentException("Could not parse plugin section of cordova_plugins.js!") } val list = jsonArray.parseJson.convertTo[List[Map[String, JsValue]]] for (info <- list) { val fileName = info.get("file").get.convertTo[String] val file = new File(cordovaPluginJs.getParentFile, fileName) val id = info.get("id").get.convertTo[String] val clobbers = if (info.contains("clobbers")) { info.get("clobbers").get.convertTo[Vector[JsValue]].map({ _.convertTo[String] }).toList } else { List[String]() } lb += new PluginInfo(fileName, id, clobbers, file) } lb.toList } private def preprocessApkDir(scripts: List[SourceModule], plugins: List[PluginInfo]) = { var counter = 0 for ( script <- scripts; file = new File(script.getURL.getFile) if script.getURL.getProtocol == "file" ) { val recorder = new FileMapRecorder(file) // Fixes WALA bug when encountering "new Function('');" statements for (m <- """new Function\(""\)""".r.findAllMatchIn(recorder.content)) { recorder.replace(m.start, m.end, """new Function("wala123")""") } for (m <- """new Function\(''\)""".r.findAllMatchIn(recorder.content)) { recorder.replace(m.start, m.end, """new Function('wala123')""") } for (plugin <- plugins.find { _.file == file } if replacePluginDefinesAndRequires) { for (m <- """cordova\.define\(".+?",\s*function\(.*?\)\s*\{""".r.findFirstMatchIn(recorder.content)) { recorder.replace(m.start, m.end, s"(function() {\n${plugin.getGlobalVar} = function() {};\n") } val lastSemi = recorder.content.lastIndexOf(";") recorder.replace(lastSemi, lastSemi, "()") for (m <- """module\.exports""".r.findAllMatchIn(recorder.content)) { recorder.replace(m.start, m.end, plugin.getGlobalVar) } // for (m <- """exports\s*=""".r.findAllMatchIn(recorder.content)) { // recorder.replace(m.start, m.end, plugin.getGlobalVar + " =") // } for (clobber <- plugin.clobbers if !clobber.contains("-")) { recorder.replace(recorder.content.size, recorder.content.size, s"\n$clobber = ${plugin.getGlobalVar};") } for (m <- """require\((?:'|")(.+?)(?:'|")\)""".r.findAllMatchIn(recorder.content)) { recorder.replace(m.start, m.end, replaceRequire(plugin, plugins, m)) } } if (mockCordovaExec) { var counterAtBeginning = counter for (m <- """cordova\s*\.\s*exec\s*\(""".r.findAllMatchIn(recorder.content)) { counter += 1 recorder.replace(recorder.content.size, recorder.content.size, s"\nfunction fun${counter}$ExecuteSuffix(s, f){s();f();};") recorder.replace(m.start, m.end, s"fun${counter}$ExecuteSuffix(") } val VarRegex = for (m <- """([a-zA-Z0-9_]+)\s*=\s*[a-zA-Z0-9_]+\s*\('cordova/exec'\)""".r.unanchored.findFirstMatchIn(recorder.content)) { val name = m.group(1) for (m2 <- ("""([\s\};])""" + name + """\(""").r.findAllMatchIn(recorder.content)) { counter += 1 recorder.replace(recorder.content.size, recorder.content.size, s"\nfunction fun${counter}$ExecuteSuffix(s, f){s();f();};") recorder.replace(m2.start, m2.end, m2.group(1) + s"fun${counter}$ExecuteSuffix(") } } logger.debug(s"Replaced ${counter - counterAtBeginning} cordova exec calls in file $file.") } recorder.writeToFile FileMappingStore.map += ((file, recorder)) } } val LocalRequireRegex = """\./([a-zA-Z]+)""".r private def replaceRequire(plugin: PluginInfo, plugins: List[PluginInfo], m: Regex.Match): String = m.group(1) match { case LocalRequireRegex(name) => plugins.find({ p => p.fileName.substring(0, p.fileName.lastIndexOf('/')) == plugin.fileName.substring(0, plugin.fileName.lastIndexOf('/')) && p.fileName.endsWith(s"/$name.js") }).get.getGlobalVar case _ => m.group(0) } def setOptions(options: CrossBuilderOption*) = for (option <- options) option match { case MockCordovaExec => mockCordovaExec = true case ReplacePluginDefinesAndRequires => replacePluginDefinesAndRequires = true case FilterJavaCallSites => filterJavaCallSites = true case FilterJSFrameworks => filterJSFrameworks = true case PreciseJS => preciseJS = true case RunBuildersInParallel => runBuildersInParallel = true } def getEntryPoint(configXml: Elem) = { val RemotePathRegex = """(https?://.+)""".r val entryPoint = configXml \ "content" \ "@src" text match { case RemotePathRegex(urlString) => { // val url = new URL(urlString) // val dest = new File(new File(apkUnzipDir, "downloaded"), "entrypoint.html") // logger.debug(s"Downloading $urlString to $dest") // dest.getParentFile.mkdirs() // val stream = url.openStream() // FileUtils.copyInputStreamToFile(stream, dest) // IOUtils.closeQuietly(stream) // dest new File(apkUnzipDir, "assets/www/index.html") } case "" => new File(apkUnzipDir, "assets/www/index.html") case text => new File(new File(apkUnzipDir, "assets/www"), text) } if (!entryPoint.getCanonicalPath.startsWith(apkUnzipDir.getCanonicalPath)) throw new RuntimeException("Entrypoint seems to be outside of the unzip dir") entryPoint } lazy val jsExclusions = if (ExclusionFile.isFile() && ExclusionFile.canRead()) { FileUtils.readLines(ExclusionFile).asScala.map { _.r.unanchored }.toList } else { List.empty } def excludeJS(url: URL) = jsExclusions.exists { _.unapplySeq(url.toString()).isDefined } }