Browse Source

Added the modules and everything

pull/1/head
standash 4 years ago
parent
commit
b4f1ccff9b
  1. 7
      .gitignore
  2. 136
      molerat/pom.xml
  3. 267
      molerat/src/main/java/it/unitn/molerat/cmd/Main.java
  4. 16
      molerat/src/main/java/it/unitn/molerat/cmd/experiments/AbstractExperiment.java
  5. 61
      molerat/src/main/java/it/unitn/molerat/cmd/experiments/CalculateAllChangesExperiment.java
  6. 53
      molerat/src/main/java/it/unitn/molerat/cmd/experiments/ChangeEvdPerformanceExperiment.java
  7. 45
      molerat/src/main/java/it/unitn/molerat/cmd/experiments/VulnEvdPerformanceExperiment.java
  8. 68
      molerat/src/main/java/it/unitn/molerat/data/csv/AbstractDataPoint.java
  9. 25
      molerat/src/main/java/it/unitn/molerat/data/csv/ChangeStatsDataPoint.java
  10. 21
      molerat/src/main/java/it/unitn/molerat/data/csv/FixStatsDataPoint.java
  11. 23
      molerat/src/main/java/it/unitn/molerat/data/csv/InputDataPoint.java
  12. 26
      molerat/src/main/java/it/unitn/molerat/data/csv/OutputDataPoint.java
  13. 21
      molerat/src/main/java/it/unitn/molerat/data/csv/PerfOutputDataPoint.java
  14. 22
      molerat/src/main/java/it/unitn/molerat/data/csv/VulnEvidenceDataPoint.java
  15. 245
      molerat/src/main/java/it/unitn/molerat/data/db/MongoWrapper.java
  16. 74
      molerat/src/main/java/it/unitn/molerat/data/memory/AnalysisEntry.java
  17. 46
      molerat/src/main/java/it/unitn/molerat/data/printers/VulnerabilityEvidencePrinter.java
  18. 45
      molerat/src/main/java/it/unitn/molerat/evidence/ChangeEvidence.java
  19. 93
      molerat/src/main/java/it/unitn/molerat/evidence/Changes.java
  20. 30
      molerat/src/main/java/it/unitn/molerat/evidence/GenericEvidence.java
  21. 32
      molerat/src/main/java/it/unitn/molerat/evidence/VulnerabilityEvidence.java
  22. 56
      molerat/src/main/java/it/unitn/molerat/repos/trackers/AbstractEvidenceTracker.java
  23. 108
      molerat/src/main/java/it/unitn/molerat/repos/trackers/changes/ChangeEvidenceTracker.java
  24. 13
      molerat/src/main/java/it/unitn/molerat/repos/trackers/changes/FullChangeEvidenceTracker.java
  25. 92
      molerat/src/main/java/it/unitn/molerat/repos/trackers/vuln/DeletionVulnerabilityEvidenceTracker.java
  26. 72
      molerat/src/main/java/it/unitn/molerat/repos/trackers/vuln/EnhancedDeletionVulnerabilityEvidenceTracker.java
  27. 71
      molerat/src/main/java/it/unitn/molerat/repos/trackers/vuln/FixStatisticsVulnerabilityEvidenceTracker.java
  28. 148
      molerat/src/main/java/it/unitn/molerat/repos/trackers/vuln/PatchedMethodBodyTracker.java
  29. 85
      molerat/src/main/java/it/unitn/molerat/repos/trackers/vuln/SliceDecayVulnerabilityEvidenceTracker.java
  30. 139
      molerat/src/main/java/it/unitn/molerat/repos/trackers/vuln/SliceVulnerabilityEvidenceTracker.java
  31. 234
      molerat/src/main/java/it/unitn/molerat/repos/trackers/vuln/VulnerabilityEvidenceTracker.java
  32. 73
      molerat/src/main/java/it/unitn/molerat/repos/trackers/vuln/VulnerabilityEvidenceTrackerFactory.java
  33. 98
      molerat/src/main/java/it/unitn/molerat/repos/utils/CommitMetrics.java
  34. 150
      molerat/src/main/java/it/unitn/molerat/repos/utils/IORoutines.java
  35. 233
      molerat/src/main/java/it/unitn/molerat/repos/utils/SignatureExtractor.java
  36. 204
      molerat/src/main/java/it/unitn/molerat/repos/wrappers/GitRepoWrapper.java
  37. 161
      molerat/src/main/java/it/unitn/molerat/repos/wrappers/RepoWrapper.java
  38. 185
      molerat/src/main/java/it/unitn/molerat/repos/wrappers/SvnRepoWrapper.java
  39. 20
      pom.xml
  40. 69
      repoman/pom.xml
  41. 87
      repoman/src/main/java/it/unitn/repoman/cmd/Main.java
  42. 9
      repoman/src/main/java/it/unitn/repoman/core/exceptions/InvalidLineException.java
  43. 59
      repoman/src/main/java/it/unitn/repoman/core/lang/LanguageFactory.java
  44. 221
      repoman/src/main/java/it/unitn/repoman/core/lang/parsers/c/C.tokens
  45. 1048
      repoman/src/main/java/it/unitn/repoman/core/lang/parsers/c/CBaseListener.java
  46. 618
      repoman/src/main/java/it/unitn/repoman/core/lang/parsers/c/CLexer.java
  47. 221
      repoman/src/main/java/it/unitn/repoman/core/lang/parsers/c/CLexer.tokens
  48. 852
      repoman/src/main/java/it/unitn/repoman/core/lang/parsers/c/CListener.java
  49. 7491
      repoman/src/main/java/it/unitn/repoman/core/lang/parsers/c/CParser.java
  50. 201
      repoman/src/main/java/it/unitn/repoman/core/lang/parsers/java/Java.tokens
  51. 1252
      repoman/src/main/java/it/unitn/repoman/core/lang/parsers/java/JavaBaseListener.java
  52. 545
      repoman/src/main/java/it/unitn/repoman/core/lang/parsers/java/JavaLexer.java
  53. 201
      repoman/src/main/java/it/unitn/repoman/core/lang/parsers/java/JavaLexer.tokens
  54. 1021
      repoman/src/main/java/it/unitn/repoman/core/lang/parsers/java/JavaListener.java
  55. 8421
      repoman/src/main/java/it/unitn/repoman/core/lang/parsers/java/JavaParser.java
  56. 110
      repoman/src/main/java/it/unitn/repoman/core/lang/traversals/c/CMethodCallTraversal.java
  57. 56
      repoman/src/main/java/it/unitn/repoman/core/lang/traversals/generic/AssignmentSymbolExtractTraversal.java
  58. 25
      repoman/src/main/java/it/unitn/repoman/core/lang/traversals/generic/ConditionTraversal.java
  59. 58
      repoman/src/main/java/it/unitn/repoman/core/lang/traversals/generic/DFSTraversal.java
  60. 51
      repoman/src/main/java/it/unitn/repoman/core/lang/traversals/generic/DotSymbolTraversal.java
  61. 52
      repoman/src/main/java/it/unitn/repoman/core/lang/traversals/generic/FormalParameterTraversal.java
  62. 64
      repoman/src/main/java/it/unitn/repoman/core/lang/traversals/generic/MethodCallTraversal.java
  63. 69
      repoman/src/main/java/it/unitn/repoman/core/lang/traversals/generic/MethodDeclarationTraversal.java
  64. 27
      repoman/src/main/java/it/unitn/repoman/core/lang/traversals/generic/NodeContainmentTraversal.java
  65. 56
      repoman/src/main/java/it/unitn/repoman/core/lang/traversals/generic/ScopesTraversal.java
  66. 39
      repoman/src/main/java/it/unitn/repoman/core/lang/traversals/generic/SeedTraversal.java
  67. 31
      repoman/src/main/java/it/unitn/repoman/core/lang/traversals/generic/StatementMappingTraversal.java
  68. 42
      repoman/src/main/java/it/unitn/repoman/core/lang/traversals/generic/SymbolExtractTraversal.java
  69. 9
      repoman/src/main/java/it/unitn/repoman/core/lang/traversals/generic/Traversal.java
  70. 125
      repoman/src/main/java/it/unitn/repoman/core/lang/traversals/java/JavaAPISignatureTraversal.java
  71. 114
      repoman/src/main/java/it/unitn/repoman/core/lang/traversals/java/JavaMethodCallTraversal.java
  72. 103
      repoman/src/main/java/it/unitn/repoman/core/lang/wrappers/c/CWrapper.java
  73. 137
      repoman/src/main/java/it/unitn/repoman/core/lang/wrappers/generic/Wrapper.java
  74. 114
      repoman/src/main/java/it/unitn/repoman/core/lang/wrappers/java/JavaWrapper.java
  75. 124
      repoman/src/main/java/it/unitn/repoman/core/slicers/LightweightSlice.java
  76. 35
      repoman/src/main/java/it/unitn/repoman/core/slicers/LightweightSlicePessimist.java
  77. 73
      repoman/src/main/java/it/unitn/repoman/core/slicers/tainters/AbstractTainter.java
  78. 75
      repoman/src/main/java/it/unitn/repoman/core/slicers/tainters/BackwardTainter.java
  79. 104
      repoman/src/main/java/it/unitn/repoman/core/slicers/tainters/BackwardTainterPessimist.java
  80. 140
      repoman/src/main/java/it/unitn/repoman/core/slicers/tainters/ForwardTainter.java
  81. 119
      repoman/src/main/java/it/unitn/repoman/core/slicers/tainters/ForwardTainterPessimist.java
  82. 38
      repoman/src/main/java/it/unitn/repoman/core/slicers/tainters/MethodCallTainter.java
  83. 29
      repoman/src/main/java/it/unitn/repoman/core/slicers/tainters/TaintedVariable.java
  84. 95
      repoman/src/main/java/it/unitn/repoman/core/slicers/tainters/TaintedVariableSet.java
  85. 102
      repoman/src/main/java/it/unitn/repoman/core/utils/printers/ConsolePrinterListener.java
  86. 18
      repoman/src/main/java/it/unitn/repoman/core/utils/printers/HTMLPrinter.java
  87. 73
      repoman/src/main/java/it/unitn/repoman/core/utils/printers/TextPrinterListener.java

7
.gitignore

@ -0,0 +1,7 @@
.classpath
.project
*.iml
*.ipr
*.iws
.idea/
target/

136
molerat/pom.xml

@ -0,0 +1,136 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<!-- parent project -->
<parent>
<groupId>it.unitn</groupId>
<artifactId>foss-vuln-tracker</artifactId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>it.unitn.molerat</groupId>
<artifactId>molerat</artifactId>
<version>1.0-SNAPSHOT</version>
<packaging>jar</packaging>
<!-- declare dependencies -->
<dependencies>
<dependency>
<groupId>org.antlr</groupId>
<artifactId>antlr4-runtime</artifactId>
<version>4.7</version>
</dependency>
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>bson</artifactId>
<version>3.4.2</version>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
<version>1.4</version>
</dependency>
<dependency>
<groupId>it.unitn.repoman</groupId>
<artifactId>repoman</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.5</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.4</version>
</dependency>
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongodb-driver</artifactId>
<version>3.4.2</version>
</dependency>
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongodb-driver-core</artifactId>
<version>3.4.2</version>
</dependency>
<dependency>
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit</artifactId>
<version>4.2.0.201601211800-r</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.25</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-nop</artifactId>
<version>1.7.25</version>
</dependency>
<dependency>
<groupId>org.tmatesoft.svnkit</groupId>
<artifactId>svnkit-cli</artifactId>
<version>1.8.13</version>
</dependency>
<dependency>
<groupId>org.tmatesoft.svnkit</groupId>
<artifactId>svnkit</artifactId>
<version>1.8.13</version>
</dependency>
<dependency>
<groupId>org.tmatesoft.svnkit</groupId>
<artifactId>svnkit-javahl16</artifactId>
<version>1.8.13</version>
</dependency>
<!--
<dependency>
<groupId>org.tmatesoft.svnkit</groupId>
<artifactId>svnkit-javahl</artifactId>
<version>1.3.5</version>
</dependency>
-->
</dependencies>
<!-- build plugins -->
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>3.0.1</version>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>prepare-package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>
${project.build.directory}/libs
</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>3.0.2</version>
<configuration>
<archive>
<manifest>
<addClasspath>true</addClasspath>
<classpathPrefix>libs/</classpathPrefix>
<mainClass>it.unitn.molerat.cmd.Main</mainClass>
</manifest>
</archive>
</configuration>
</plugin>
</plugins>
</build>
</project>

267
molerat/src/main/java/it/unitn/molerat/cmd/Main.java

@ -0,0 +1,267 @@
package it.unitn.molerat.cmd;
import it.unitn.molerat.data.csv.InputDataPoint;
import it.unitn.molerat.data.csv.VulnEvidenceDataPoint;
import it.unitn.molerat.data.db.MongoWrapper;
import it.unitn.molerat.data.memory.AnalysisEntry;
import it.unitn.molerat.evidence.VulnerabilityEvidence;
import it.unitn.molerat.repos.trackers.vuln.VulnerabilityEvidenceTracker;
import it.unitn.molerat.repos.trackers.vuln.VulnerabilityEvidenceTrackerFactory;
import it.unitn.molerat.repos.utils.IORoutines;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
import org.apache.commons.cli.*;
import org.bson.types.ObjectId;
public class Main {
private static MongoWrapper db = new MongoWrapper("molerat");
public static void main(String[] args) {
Options opts = new Options();
Option listTrackersOpt = Option.builder()
.longOpt("list-trackers")
.desc("List the available trackers for vulnerability molerat.evidence")
.build();
opts.addOption(listTrackersOpt);
Option projectNameOpt = Option.builder()
.longOpt("project-name")
.desc("The name of the project")
.hasArg()
.build();
opts.addOption(projectNameOpt);
Option repoTypeOpt = Option.builder()
.longOpt("repo-type")
.desc("The type of the source code repository ('git' or 'svn')")
.hasArg()
.build();
opts.addOption(repoTypeOpt);
Option repoPathOpt = Option.builder()
.longOpt("repo-path")
.desc("The path of the working copy of the source code repository")
.hasArg()
.build();
opts.addOption(repoPathOpt);
Option cveIdOpt = Option.builder()
.longOpt("cve-id")
.desc("The CVE identifier of a vulnerability")
.hasArg()
.build();
opts.addOption(cveIdOpt);
Option fixCommitOpt = Option.builder()
.longOpt("fix-commit")
.desc("The commit that fixed a vulnerability")
.hasArg()
.build();
opts.addOption(fixCommitOpt);
Option trackerTypeOpt = Option.builder()
.longOpt("tracker-type")
.desc("The type of the vulnerability molerat.evidence tracker")
.hasArg()
.build();
opts.addOption(trackerTypeOpt);
Option inputFileOpt = Option.builder()
.longOpt("input-file")
.desc("Path to the input .csv file")
.hasArg()
.argName("input-file-path")
.build();
opts.addOption(inputFileOpt);
Option outputFileOpt = Option.builder()
.longOpt("output-file")
.desc("Path to the output .csv file")
.hasArg()
.argName("output-file-path")
.build();
opts.addOption(outputFileOpt);
CommandLineParser cmdParser = new DefaultParser();
HelpFormatter helpFormatter = new HelpFormatter();
CommandLine cmd;
try {
cmd = cmdParser.parse(opts, args);
if (cmd.getOptions().length == 0) {
throw new ParseException("Arguments are not specified");
}
if (cmd.hasOption("list-trackers")) {
System.out.println(VulnerabilityEvidenceTrackerFactory.getTrackersList());
return;
}
if (cmd.hasOption("input-file")) {
String i = cmd.getOptionValue("input-file");
collectVulnEvidence(i);
}
if (cmd.hasOption("output-file")) {
String o = cmd.getOptionValue("output-file");
generateCsv(o);
}
if (!cmd.hasOption("input-file") && !cmd.hasOption("output-file")) {
String projectName = null;
String repoType = null;
String repoPath = null;
String cveId = null;
String fixCommit = null;
String trackerType = null;
if (cmd.hasOption("project-name")) {
projectName = cmd.getOptionValue("project-name");
}
if (cmd.hasOption("repo-type")) {
repoType = cmd.getOptionValue("repo-type");
}
if(cmd.hasOption("repo-path")) {
repoPath = cmd.getOptionValue("repo-path");
}
if(cmd.hasOption("cve-id")) {
cveId = cmd.getOptionValue("cve-id");
}
if(cmd.hasOption("fix-commit")) {
fixCommit = cmd.getOptionValue("fix-commit");
}
if(cmd.hasOption("tracker-type")) {
trackerType = cmd.getOptionValue("tracker-type");
}
if (projectName == null) {
throw new ParseException("The 'project-name' parameter is not specified");
}
if (repoType == null) {
throw new ParseException("The 'repo-type' parameter is not specified");
}
if (repoPath == null) {
throw new ParseException("The 'repo-path' parameter is not specified");
}
if (cveId == null) {
throw new ParseException("The 'cve-id' parameter is not specified");
}
if (fixCommit == null) {
throw new ParseException("The 'fix-commit' parameter is not specified");
}
if (trackerType == null) {
throw new ParseException("The 'tracker-type' parameter is not specified");
}
collect(projectName, repoType, repoPath, cveId, fixCommit, trackerType);
}
}
catch (ParseException e) {
System.out.println("ERROR: " + e.getMessage());
helpFormatter.printHelp("utility-name", opts);
}
}
private static void collectVulnEvidence(String inputCsvPath) {
try {
Set<InputDataPoint> inputs = IORoutines.readInputDataPoints(inputCsvPath);
for (InputDataPoint ip : inputs) {
collect(ip.PROJECTID,
ip.REPO_TYPE,
ip.REPO_ROOT,
ip.CVEID,
ip.FIX_REV,
ip.TRACKER_TYPE);
}
} catch (Exception e) {
System.out.println("ERROR: " + e.getMessage());
}
}
private static void collect(String projectName, String repositoryType,
String repositoryPath, String cveName, String fixCommit, String trackerType) {
try {
boolean entryExists = db.analysisEntryExists(projectName, cveName, repositoryType, repositoryPath);
if (entryExists) {
System.out.format("WARNING: The analysis entry for '%s' from '%s' already exists, skipping\n", cveName, projectName);
return;
}
System.out.format("INFO: Collecting the molerat.evidence for '%s' from '%s'\n", cveName, projectName);
VulnerabilityEvidenceTracker vulnTracker = VulnerabilityEvidenceTrackerFactory.getTracker(
repositoryPath,
fixCommit,
repositoryType,
trackerType
);
vulnTracker.trackEvidence();
AnalysisEntry entry = new AnalysisEntry(
cveName,
projectName,
repositoryType,
repositoryPath,
fixCommit,
vulnTracker.getProcessedCommits(),
vulnTracker.getEvidences(),
null // we're not tracking any changes so far
);
boolean isSuccessful = db.insertAnalysisEntry(entry);
if (!isSuccessful) {
System.out.format("WARNING: Could not get any molerat.evidence for '%s' in '%s'\n", cveName, projectName);
}
System.out.println("INFO: Done!");
}
catch (Exception e) {
System.out.println("ERROR: " + e.getMessage());
}
}
private static void generateCsv(String outFilePath) {
System.out.format("INFO: generating the .csv file '%s'\n", outFilePath);
String csvHeader = new VulnEvidenceDataPoint().getHeader() + "\n";
try {
IORoutines.writeFile(outFilePath, csvHeader);
Set<ObjectId> cveIds = db.getAllCveIds();
for (ObjectId cveId : cveIds) {
AnalysisEntry analysis = db.getAnalysisEntry(cveId);
if (analysis == null) {
continue;
}
String projectName = analysis.getProjectName();
String cveName = analysis.getCveName();
System.out.format("INFO: writing the entries for '%s'\n", cveName);
Set<VulnerabilityEvidence> evidences = analysis.getVulnEvidencesSet();
if (evidences.size() == 0) {
throw new Exception(String.format("There is no molerat.evidence for '%s' in the database", cveName));
}
Map<String, Integer> locsCount = new LinkedHashMap<>();
for (VulnerabilityEvidence evd : evidences) {
if (locsCount.containsKey(evd.getCommit())) {
int count = locsCount.get(evd.getCommit());
locsCount.remove(evd.getCommit());
locsCount.put(evd.getCommit(), ++count);
} else {
locsCount.put(evd.getCommit(), 1);
}
}
int timestamp = 0;
for (Map.Entry<String, Integer> entry : locsCount.entrySet()) {
VulnEvidenceDataPoint dp = new VulnEvidenceDataPoint(new String[]{
projectName,
cveName,
entry.getKey(),
String.valueOf(timestamp--),
String.valueOf(entry.getValue())
});
IORoutines.writeFile(outFilePath, dp.toString() + "\n");
}
}
} catch (Exception e) {
System.out.println("ERROR: " + e.getMessage());
}
}
}

16
molerat/src/main/java/it/unitn/molerat/cmd/experiments/AbstractExperiment.java

@ -0,0 +1,16 @@
package it.unitn.molerat.cmd.experiments;
import it.unitn.molerat.data.csv.InputDataPoint;
import it.unitn.molerat.repos.utils.IORoutines;
import java.util.Set;
public class AbstractExperiment {
protected static Set<InputDataPoint> readInputDataPoints(String path) throws Exception {
return IORoutines.readInputDataPoints(path);
}
protected static void saveOutput(String path, String output) throws Exception {
IORoutines.writeFile(path, output + "\n");
}
}

61
molerat/src/main/java/it/unitn/molerat/cmd/experiments/CalculateAllChangesExperiment.java

@ -0,0 +1,61 @@
package it.unitn.molerat.cmd.experiments;
import it.unitn.molerat.data.csv.InputDataPoint;
import it.unitn.molerat.data.csv.OutputDataPoint;
import it.unitn.molerat.data.csv.PerfOutputDataPoint;
import it.unitn.molerat.repos.utils.CommitMetrics;
import it.unitn.molerat.repos.wrappers.GitRepoWrapper;
import it.unitn.molerat.repos.wrappers.RepoWrapper;
import java.util.Set;
public class CalculateAllChangesExperiment extends AbstractExperiment {
public static void main(String[] args) {
if (args.length < 2) {
System.out.println("USAGE: [input.csv] [output.csv]");
return;
}
String inputCsvPath = args[0];
String outputCsvPath = args[1];
try {
Set<InputDataPoint> inputs = readInputDataPoints(inputCsvPath);
saveOutput(outputCsvPath, new PerfOutputDataPoint().getHeader());
for (InputDataPoint idp : inputs) {
System.out.println("Processing the " + idp.CVEID + "...");
long timeNow = System.currentTimeMillis();
RepoWrapper wrapper = new GitRepoWrapper(idp.REPO_ROOT);
Set<String> processedCommits = wrapper.getRevisionNumbers(idp.FIX_REV);
int timestamp = 0;
for (String commit : processedCommits) {
int publicAPICount = CommitMetrics.getNumberOfPublicMethodsPerRevision(commit, wrapper);
int rem = CommitMetrics.getGlobalPublicMethodsRemoved(commit, idp.FIX_REV, wrapper);
double currentUntouched = (((double) publicAPICount - (double) rem) / (double) publicAPICount) * 100.0;
currentUntouched = Math.round(currentUntouched * 100.0) / 100.0;
String[] output = new String[]{
idp.CVEID,
idp.TRACKER_TYPE,
"empty",
commit,
String.valueOf(timestamp),
"empty",
String.valueOf(publicAPICount),
String.valueOf(rem),
String.valueOf(currentUntouched)
};
OutputDataPoint odp = new OutputDataPoint(output);
saveOutput(outputCsvPath, odp.toString());
timestamp--;
}
long timeAfter = System.currentTimeMillis();
System.out.println("TIME (ms): " + (timeAfter - timeNow));
}
} catch (Exception e) {
e.printStackTrace();
}
}
}

53
molerat/src/main/java/it/unitn/molerat/cmd/experiments/ChangeEvdPerformanceExperiment.java

@ -0,0 +1,53 @@
package it.unitn.molerat.cmd.experiments;
import it.unitn.molerat.data.csv.InputDataPoint;
import it.unitn.molerat.data.csv.PerfOutputDataPoint;
import it.unitn.molerat.repos.trackers.vuln.VulnerabilityEvidenceTracker;
import it.unitn.molerat.repos.trackers.vuln.VulnerabilityEvidenceTrackerFactory;
import it.unitn.molerat.repos.utils.CommitMetrics;
import java.util.Set;
public class ChangeEvdPerformanceExperiment extends AbstractExperiment {
public static void main(String[] args) {
if (args.length < 2) {
System.out.println("USAGE: [input.csv] [output.csv]");
return;
}
String inputCsvPath = args[0];
String outputCsvPath = args[1];
try {
Set<InputDataPoint> inputs = readInputDataPoints(inputCsvPath);
saveOutput(outputCsvPath, new PerfOutputDataPoint().getHeader());
for (InputDataPoint idp : inputs) {
System.out.println("Processing the " + idp.CVEID + "...");
VulnerabilityEvidenceTracker vulnTracker = VulnerabilityEvidenceTrackerFactory.getTracker(idp.REPO_ROOT, idp.FIX_REV, idp.REPO_TYPE, idp.TRACKER_TYPE);
vulnTracker.trackEvidence();
Set<String> processedCommits = vulnTracker.getProcessedCommits();
for (String commit : processedCommits) {
long timeNow = System.currentTimeMillis();
int publicAPICount = CommitMetrics.getNumberOfPublicMethodsPerRevision(commit, vulnTracker.getRepoWrapper());
int rem = CommitMetrics.getGlobalPublicMethodsRemoved(commit, vulnTracker.getFixedRevision(), vulnTracker.getRepoWrapper());
double currentUntouched = (((double) publicAPICount - (double) rem) / (double) publicAPICount) * 100.0;
currentUntouched = Math.round(currentUntouched * 100.0) / 100.0;
long timeAfter = System.currentTimeMillis();
String[] output = new String[]{
idp.CVEID,
idp.TRACKER_TYPE,
String.valueOf(timeAfter - timeNow)
};
PerfOutputDataPoint odp = new PerfOutputDataPoint(output);
saveOutput(outputCsvPath, odp.toString());
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
}

45
molerat/src/main/java/it/unitn/molerat/cmd/experiments/VulnEvdPerformanceExperiment.java

@ -0,0 +1,45 @@
package it.unitn.molerat.cmd.experiments;
import it.unitn.molerat.data.csv.InputDataPoint;
import it.unitn.molerat.data.csv.PerfOutputDataPoint;
import it.unitn.molerat.repos.trackers.vuln.VulnerabilityEvidenceTracker;
import it.unitn.molerat.repos.trackers.vuln.VulnerabilityEvidenceTrackerFactory;
import java.util.Set;
public class VulnEvdPerformanceExperiment extends AbstractExperiment {
public static void main(String[] args) {
if (args.length < 2) {
System.out.println("USAGE: [input.csv] [output.csv]");
return;
}
String inputCsvPath = args[0];
String outputCsvPath = args[1];
try {
Set<InputDataPoint> inputs = readInputDataPoints(inputCsvPath);
saveOutput(outputCsvPath, new PerfOutputDataPoint().getHeader());
for (InputDataPoint idp : inputs) {
System.out.println("Processing the " + idp.CVEID + "...");
long timeNow = System.currentTimeMillis();
VulnerabilityEvidenceTracker vulnTracker = VulnerabilityEvidenceTrackerFactory.getTracker(idp.REPO_ROOT, idp.FIX_REV, idp.REPO_TYPE, idp.TRACKER_TYPE);
vulnTracker.trackEvidence();
long timeAfter = System.currentTimeMillis();
String[] output = new String[]{
idp.CVEID,
idp.TRACKER_TYPE,
String.valueOf(timeAfter - timeNow)
};
PerfOutputDataPoint odp = new PerfOutputDataPoint(output);
saveOutput(outputCsvPath, odp.toString());
}
} catch (Exception e) {
e.printStackTrace();
}
}
}

68
molerat/src/main/java/it/unitn/molerat/data/csv/AbstractDataPoint.java

@ -0,0 +1,68 @@
package it.unitn.molerat.data.csv;
import java.lang.reflect.Field;
public abstract class AbstractDataPoint {
public AbstractDataPoint(String[] entries) throws Exception {
Field[] fields = this.getClass().getFields();
if (entries.length != fields.length) {
throw new Exception("Bad entries for initializing a molerat.data point!");
}
int i = 0;
for (Field field : fields) {
if (!field.getName().equals("this$0")) {
try {
field.set(this, entries[i++]);
} catch (IllegalArgumentException e) {
System.out.println("ERROR: " + e.getMessage());
} catch (IllegalAccessException e) {
System.out.println("ERROR: " + e.getMessage());
}
}
}
}
public AbstractDataPoint(String str) throws Exception {
this(str.replace(" ", "").split(","));
}
public AbstractDataPoint() {
}
@Override
public final String toString() {
StringBuilder builder = new StringBuilder();
Field[] fields = this.getClass().getFields();
for (Field field : fields) {
if (!field.getName().equals("this$0")) {
try {
builder.append(field.get(this) + ",");
} catch (IllegalArgumentException e) {
System.out.println("ERROR: " + e.getMessage());
} catch (IllegalAccessException e) {
System.out.println("ERROR: " + e.getMessage());
}
}
}
builder.deleteCharAt(builder.length()-1);
return builder.toString();
}
public final String getHeader() {
StringBuilder builder = new StringBuilder();
Field[] fields = this.getClass().getFields();
for (Field field : fields) {
if (!field.getName().equals("this$0")) {
try {
builder.append(field.getName() + ",");
} catch (IllegalArgumentException e) {
System.out.println("ERROR: " + e.getMessage());
}
}
}
builder.deleteCharAt(builder.length()-1);
return builder.toString();
}
}

25
molerat/src/main/java/it/unitn/molerat/data/csv/ChangeStatsDataPoint.java

@ -0,0 +1,25 @@
package it.unitn.molerat.data.csv;
/**
* Created by standash on 06/12/2016.
*/
public class ChangeStatsDataPoint extends AbstractDataPoint {
public String PROJECT;
public String CURRENT_COMMIT;
public String ADDED_LINES;
public String DELETED_LINES;
public String TOUCHED_METHODS;
public String TOUCHED_FILES;
public ChangeStatsDataPoint(String[] entries) throws Exception {
super(entries);
}
public ChangeStatsDataPoint(String str) throws Exception {
super(str);
}
public ChangeStatsDataPoint() {
}
}

21
molerat/src/main/java/it/unitn/molerat/data/csv/FixStatsDataPoint.java

@ -0,0 +1,21 @@
package it.unitn.molerat.data.csv;
public class FixStatsDataPoint extends AbstractDataPoint {
public String CVEID;
public String ADDED_LINES;
public String DELETED_LINES;
public String TOUCHED_METHODS;
public String TOUCHED_FILES;
public FixStatsDataPoint(String[] entries) throws Exception {
super(entries);
}
public FixStatsDataPoint(String str) throws Exception {
super(str);
}
public FixStatsDataPoint() {
}
}

23
molerat/src/main/java/it/unitn/molerat/data/csv/InputDataPoint.java

@ -0,0 +1,23 @@
package it.unitn.molerat.data.csv;
public final class InputDataPoint extends AbstractDataPoint {
public String PROJECTID;
public String CVEID;
public String REPO_TYPE;
public String REPO_ROOT;
public String FIX_REV;
public String TRACKER_TYPE;
public InputDataPoint(String[] entries) throws Exception {
super(entries);
}
public InputDataPoint(String str) throws Exception {
super(str);
}
public InputDataPoint() {
}
}

26
molerat/src/main/java/it/unitn/molerat/data/csv/OutputDataPoint.java

@ -0,0 +1,26 @@
package it.unitn.molerat.data.csv;
public class OutputDataPoint extends AbstractDataPoint {
public String CVEID;
public String TRACKING_METHOD;
public String CURRENT_FILE;
public String CURRENT_REV;
public String TIMESTAMP;
public String EVIDENCE_LOC;
//public String GLOBAL_PUBLIC_API_COUNT;
//public String GLOBAL_PUBLIC_API_REMOVED;
//public String GLOBAL_PUBLIC_API_UNTOUCHED;
public OutputDataPoint(String[] entries) throws Exception {
super(entries);
}
public OutputDataPoint(String str) throws Exception {
super(str);
}
public OutputDataPoint() {
}
}

21
molerat/src/main/java/it/unitn/molerat/data/csv/PerfOutputDataPoint.java

@ -0,0 +1,21 @@
package it.unitn.molerat.data.csv;
public class PerfOutputDataPoint extends AbstractDataPoint {
public String CVEID;
public String TRACKING_METHOD;
public String TIME_MS;
public PerfOutputDataPoint(String[] entries) throws Exception {
super(entries);
}
public PerfOutputDataPoint(String str) throws Exception {
super(str);
}
public PerfOutputDataPoint() {
}
}

22
molerat/src/main/java/it/unitn/molerat/data/csv/VulnEvidenceDataPoint.java

@ -0,0 +1,22 @@
package it.unitn.molerat.data.csv;
public class VulnEvidenceDataPoint extends AbstractDataPoint {
public String PROJECT_NAME;
public String CVEID;
public String CURRENT_REV;
public String TIMESTAMP;
public String EVIDENCE_LOC;
public VulnEvidenceDataPoint(String[] entries) throws Exception {
super(entries);
}
public VulnEvidenceDataPoint(String str) throws Exception {
super(str);
}
public VulnEvidenceDataPoint() {
}
}

245
molerat/src/main/java/it/unitn/molerat/data/db/MongoWrapper.java

@ -0,0 +1,245 @@
package it.unitn.molerat.data.db;
import com.mongodb.BasicDBObject;
import com.mongodb.MongoClient;
import com.mongodb.client.FindIterable;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;
import com.mongodb.client.model.Indexes;
import it.unitn.molerat.data.memory.AnalysisEntry;
import it.unitn.molerat.evidence.VulnerabilityEvidence;
import org.bson.Document;
import org.bson.types.ObjectId;
import java.util.*;
public class MongoWrapper {
private static final String projectCollection = "projects";
private static final String vulnsCollection = "vulns";
private static final String vulnEvdCollection = "vuln_evidences";
private MongoClient client = null;
private MongoDatabase db = null;
public MongoWrapper(String dbName) {
this.client = new MongoClient();
this.db = client.getDatabase(dbName);
}
private FindIterable<Document> getEntries(BasicDBObject query, String collection) {
MongoCollection<Document> docs = db.getCollection(collection);
return docs.find(query);
}
private Document getEntry(BasicDBObject query, String collection) {
return getEntries(query, collection).first();
}
private ObjectId getEntryId(Document doc) {
return (doc != null) ? doc.getObjectId("_id") : null;
}
private Document getProjectEntry(String name, String repoType, String repoPath) {
BasicDBObject query = new BasicDBObject();
query.put("name", name);
query.put("repo_type", repoType);
query.put("repo_path", repoPath);
return getEntry(query, projectCollection);
}
private Document getProjectEntry(ObjectId projectId) {
BasicDBObject query = new BasicDBObject();
query.put("_id", projectId);
return getEntry(query, projectCollection);
}
private ObjectId getProjectId(String name, String repoType, String repoPath) {
return getEntryId(getProjectEntry(name, repoType, repoPath));
}
private ObjectId getProjectId(ObjectId cveId) {
return getCveEntry(cveId).getObjectId("owner_id");
}
private Document getCveEntry(ObjectId projectId, String cve) {
BasicDBObject query = new BasicDBObject();
query.put("cve", cve);
query.put("owner_id", projectId);
return getEntry(query, vulnsCollection);
}
private Document getCveEntry(ObjectId cveId) {
BasicDBObject query = new BasicDBObject();
query.put("_id", cveId);
return getEntry(query, vulnsCollection);
}
private ObjectId getCveId(ObjectId projectId, String cve) {
return getEntryId(getCveEntry(projectId, cve));
}
public Set<ObjectId> getAllCveIds() {
Set<ObjectId> ids = new LinkedHashSet<>();
MongoCollection<Document> vulns = db.getCollection(vulnsCollection);
FindIterable<Document> docs = vulns.find();
for (Document doc : docs) {
ids.add(doc.getObjectId("_id"));
}
return ids;
}
private FindIterable<Document> getVulnEvidences(ObjectId cveId) {
BasicDBObject query = new BasicDBObject() ;
query.put("owner_id", cveId);
FindIterable<Document> result = getEntries(query, vulnEvdCollection);
result.sort(new BasicDBObject("order", -1));
return result;
}
private ObjectId insertProject(String name, String repoType, String repoPath) {
Document proj = getProjectEntry(name, repoType, repoPath);
if (proj == null) {
MongoCollection<Document> projects = db.getCollection(projectCollection);
proj = new Document("name", name)
.append("repo_type", repoType)
.append("repo_path", repoPath);
projects.insertOne(proj);
}
return getEntryId(proj);
}
private ObjectId insertCve(ObjectId projectId, String cve, String fixCommit) {
Document vuln = getCveEntry(projectId, cve);
if (vuln == null) {
MongoCollection<Document> vulns = db.getCollection(vulnsCollection);
vuln = new Document("cve", cve)
.append("fix_commit", fixCommit)
.append("owner_id", projectId)
.append("processed", false);
vulns.insertOne(vuln);
MongoCollection<Document> projects = db.getCollection(projectCollection);
projects.findOneAndUpdate(new BasicDBObject("_id", projectId), new BasicDBObject("$push",
new BasicDBObject("vulns", getEntryId(vuln))));
}
return getEntryId(vuln);
}
private boolean isVulnProcessed(ObjectId cveId) {
MongoCollection<Document> vulns = db.getCollection(vulnsCollection);
Document cve = vulns.find(new BasicDBObject("_id", cveId)).first();
return (Boolean)cve.get("processed");
}
public boolean insertAnalysisEntry(AnalysisEntry entry) {
// if there's no molerat.evidence for some reason, return false
if (entry.getVulnEvidences().isEmpty()) {
return false;
}
ObjectId projectId = insertProject(entry.getProjectName(), entry.getRepositoryType(), entry.getRepositoryPath());
ObjectId cveId = insertCve(projectId, entry.getCveName(), entry.getFixCommit());
MongoCollection<Document> vulns = db.getCollection(vulnsCollection);
MongoCollection<Document> evds = db.getCollection(vulnEvdCollection);
if (isVulnProcessed(cveId)) {
vulns.updateOne(
new BasicDBObject("_id", cveId),
new BasicDBObject("$set", new BasicDBObject("processed", false))
);
evds.deleteMany(new BasicDBObject("owner_id", cveId));
}
List<Document> records = new LinkedList<Document>();
int order= 0;
Set<String> commits = entry.getCommits();
for (String commit : commits) {
Set<VulnerabilityEvidence> evidences = entry.getVulnEvidences().get(commit);
if (evidences != null) {
for (VulnerabilityEvidence e : evidences) {
BasicDBObject record = new BasicDBObject();
record.put("owner_id", cveId);
record.put("revision", commit);
record.put("order", order);
record.put("file_path", e.getPath());
record.put("container", e.getContainer());
record.put("line_number", e.getLineNumber());
record.put("line_contents", e.getLineContents());
Document doc = new Document(record);
records.add(doc);
}
}
order--;
}
evds.insertMany(records);
vulns.updateOne(
new BasicDBObject("_id", cveId),
new BasicDBObject("$set", new BasicDBObject("processed", true))
);
//create index
evds.createIndex(Indexes.descending("order"));
return true;
}
public AnalysisEntry getAnalysisEntry(String projectName, String cveName, String repoType, String repoPath) {
ObjectId projectId = getProjectId(projectName, repoType, repoPath);
ObjectId cveId = getCveId(projectId, cveName);
return getAnalysisEntry(cveId);
}
public boolean analysisEntryExists(String projectName, String cveName, String repoType, String repoPath) {
ObjectId projectId = getProjectId(projectName, repoType, repoPath);
ObjectId cveId = getCveId(projectId, cveName);
return (projectId != null && cveId != null);
}
public Set<AnalysisEntry> getAnalysisEntries() {
Set<AnalysisEntry> entries = new LinkedHashSet<>();
FindIterable<Document> cves = db.getCollection(vulnsCollection).find();
for (Document cve : cves) {
ObjectId cveId = cve.getObjectId("_id");
AnalysisEntry entry = getAnalysisEntry(cveId);
entries.add(entry);
}
return entries;
}
public AnalysisEntry getAnalysisEntry(ObjectId cveId) {
if (!isVulnProcessed(cveId)) {
return null;
}
ObjectId projectId = getProjectId(cveId);
Document projectDoc = getProjectEntry(projectId);
String projectName = projectDoc.getString("name");
String repoType = projectDoc.getString("repo_type");
String repoPath = projectDoc.getString("repo_path");
Document cveDoc = getCveEntry(cveId);
String cveName = cveDoc.getString("cve");
String fixCommit = cveDoc.getString("fix_commit");
Map<String, Set<VulnerabilityEvidence>> vulnEvidences = new LinkedHashMap<>();
FindIterable<Document> docs = getVulnEvidences(cveId);
for (Document doc : docs) {
String filePath = doc.getString("file_path");
String container = doc.getString("container");
String revision = doc.getString("revision");
int lineNumber = doc.getInteger("line_number");
String lineContents = doc.getString("line_contents");
VulnerabilityEvidence evidence = new VulnerabilityEvidence(filePath, revision, container, lineNumber, lineContents);
if (vulnEvidences.containsKey(revision)) {
vulnEvidences.get(revision).add(evidence);
}
else {
Set<VulnerabilityEvidence> set = new LinkedHashSet<>();
set.add(evidence);
vulnEvidences.put(evidence.getCommit(), set);
}
}
/* TODO: add stuff for getting the changes molerat.evidence
Map<String, Set<ChangeEvidence>> chgEvidences = new TreeMap<>();
*/
return new AnalysisEntry(cveName, projectName, repoType, repoPath, fixCommit, vulnEvidences.keySet(), vulnEvidences, null);
}
}

74
molerat/src/main/java/it/unitn/molerat/data/memory/AnalysisEntry.java

@ -0,0 +1,74 @@
package it.unitn.molerat.data.memory;
import it.unitn.molerat.evidence.ChangeEvidence;
import it.unitn.molerat.evidence.VulnerabilityEvidence;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;
public class AnalysisEntry {
private final String cveId;
private final String projectId;
private final String fixCommit;
private final String repositoryType;
private final String repositoryPath;
private final Set<String> commits;
private final Map<String, Set<VulnerabilityEvidence>> vulnEvidences;
private final Map<String, Set<ChangeEvidence>> changEvidences;
public AnalysisEntry(String cveId, String projectId, String repositoryType, String repositoryPath,
String fixCommit, Set<String> commits,
Map<String, Set<VulnerabilityEvidence>> vulnEvidences,
Map<String, Set<ChangeEvidence>> changEvidences) {
this.cveId = cveId;
this.projectId = projectId;
this.repositoryType = repositoryType;
this.repositoryPath = repositoryPath;
this.commits = commits;
this.fixCommit = fixCommit;
this.vulnEvidences = vulnEvidences;
this.changEvidences = changEvidences;
}
public String getCveName() {
return cveId;
}
public Set<String> getCommits() {
return commits;
}
public String getFixCommit() {
return fixCommit;
}
public String getRepositoryType() {
return repositoryType;
}
public String getRepositoryPath() {
return repositoryPath;
}
public Map<String, Set<VulnerabilityEvidence>> getVulnEvidences() {
return vulnEvidences;
}
public Set<VulnerabilityEvidence> getVulnEvidencesSet() {
Set<VulnerabilityEvidence> evidences = new LinkedHashSet<>();
for (Set<VulnerabilityEvidence> eset : vulnEvidences.values()) {
evidences.addAll(eset);
}
return evidences;
}
public Map<String, Set<ChangeEvidence>> getChangEvidences() {
return changEvidences;
}
public String getProjectName() {
return projectId;
}
}

46
molerat/src/main/java/it/unitn/molerat/data/printers/VulnerabilityEvidencePrinter.java

@ -0,0 +1,46 @@
package it.unitn.molerat.data.printers;
import it.unitn.molerat.evidence.VulnerabilityEvidence;
import java.util.*;
public class VulnerabilityEvidencePrinter {
private final StringBuilder builder = new StringBuilder();
public VulnerabilityEvidencePrinter(Set<VulnerabilityEvidence> evidence) {
List<VulnerabilityEvidence> sortedEvidence = new LinkedList<>(evidence);
Collections.sort(sortedEvidence, new Comparator<VulnerabilityEvidence>() {
@Override
public int compare(VulnerabilityEvidence o1, VulnerabilityEvidence o2) {
if (o1.getLineNumber() < o2.getLineNumber()) {
return -1;
} else if (o1.getLineNumber() > o2.getLineNumber()) {
return 1;
} else {
return 0;
}
}
});
String path = "";
String container = "";
for (VulnerabilityEvidence e : sortedEvidence) {
if (!path.equals(e.getPath())) {
path = e.getPath();
builder.append("<" + path + ">\n");
}
if (!container.equals(e.getContainer())) {
container = e.getContainer();
builder.append("\t" + container + "\n");
}
builder.append("\t\t" + e.getLineNumber() + ": " + e.getLineContents() + "\n");
}
}
@Override
public String toString() {
return this.builder.toString();
}
}

45
molerat/src/main/java/it/unitn/molerat/evidence/ChangeEvidence.java

@ -0,0 +1,45 @@
package it.unitn.molerat.evidence;
public class ChangeEvidence extends GenericEvidence {
private final boolean removed;
public ChangeEvidence(String file, String commit, String container, boolean removed) {
super(file, commit, container);
this.removed = removed;
}
public boolean isMethodOrConstructor() {
return this.container.contains("(") && this.container.contains(")");
}
public boolean isAbsentInFix() {
return removed;
}
public String getAccessModifier() {
if (this.container.startsWith("public")) {
return "public";
}
else if (this.container.startsWith("protected")) {
return "protected";
}
else if (this.container.startsWith("private")) {
return "private";
}
return "";
}
public boolean isPublicMethodOrConstructor() {
return this.isMethodOrConstructor() && this.getAccessModifier().equals("public");
}
public boolean isProtectedMethodOrConstructor() {
return this.isMethodOrConstructor() && this.getAccessModifier().equals("protected");
}
public boolean isPrivateMethodOrConstructor() {
return this.isMethodOrConstructor() && this.getAccessModifier().equals("private");
}
}

93
molerat/src/main/java/it/unitn/molerat/evidence/Changes.java

@ -0,0 +1,93 @@
package it.unitn.molerat.evidence;
import java.util.Map;
import java.util.TreeMap;
public class Changes {
private final String path;
private final String leftRev;
private final String rightRev;
private final Map<Integer, String> deletions;
private final Map<Integer, String> additions;
private String renamedTo = "";
public Changes(String path, String leftRev, String rightRev) {
this.path = path;
this.leftRev = leftRev;
this.rightRev = rightRev;
this.deletions = new TreeMap<>();
this.additions = new TreeMap<>();
}
public Changes(String path, String renamedTo, String leftRev, String rightRev) {
this(path, leftRev, rightRev);
this.renamedTo = renamedTo;
}
public Changes(Changes changes, String leftRev, String rightRev) {
this.path = changes.path;
this.leftRev = leftRev;
this.rightRev = rightRev;
this.deletions = changes.deletions;
this.additions = changes.additions;
}
public String getRenamedTo() {
return renamedTo;
}
public boolean wasRenamed() {
return !renamedTo.equals("");
}
public String getPath() {
return this.path;
}
public String getLeftRevision() {
return this.leftRev;
}
public String getRightRevision() {
return this.rightRev;
}
public void putDeletedLine(int lineNumber, String line) {
this.deletions.put(lineNumber, line);
}
public void putAddedLine(int lineNumber, String line) {
this.additions.put(lineNumber, line);
}
public Map<Integer, String> getDeletions() {
return this.deletions;
}
public Map<Integer, String> getAdditions() {
return this.additions;
}
@Override
public String toString() {
StringBuilder builder = new StringBuilder();
builder.append("CHANGES:\n");
for (Map.Entry<Integer, String> entry : this.deletions.entrySet()) {
builder.append("-, " + this.formatAsCSV(entry) + "\n");
}
for (Map.Entry<Integer, String> entry : this.additions.entrySet()) {
builder.append("+, " + this.formatAsCSV(entry) + "\n");
}
return builder.toString();
}
private String formatAsCSV(Map.Entry<Integer, String> entry) {
return this.leftRev + ", " +
this.path + ", " +
entry.getKey() + ", " +
entry.getValue();
}
}

30
molerat/src/main/java/it/unitn/molerat/evidence/GenericEvidence.java

@ -0,0 +1,30 @@
package it.unitn.molerat.evidence;
public class GenericEvidence {
protected final String commit;
protected final String container;
protected String path;
protected GenericEvidence(String file, String commit, String container) {
this.path = file;
this.commit = commit;
this.container = container;
}
public String getContainer() {
return this.container;
}
public String getCommit() {
return this.commit;
}
public String getPath() {
return this.path;
}
public void setPath(String path) {
this.path = path;
}
}

32
molerat/src/main/java/it/unitn/molerat/evidence/VulnerabilityEvidence.java

@ -0,0 +1,32 @@
package it.unitn.molerat.evidence;
public class VulnerabilityEvidence extends GenericEvidence {
protected final int lineNumber;
protected final String lineContents;
public VulnerabilityEvidence(String file, String commit, String container, int lineNumber, String lineContents) {
super(file, commit, container);
this.lineNumber = lineNumber;
this.lineContents = lineContents
.trim()
.replace("\t","");
}
public int getLineNumber() {
return this.lineNumber;
}
public String getLineContents() {
return this.lineContents;
}
@Override
public String toString() {
StringBuilder builder = new StringBuilder();
builder.append("<" + commit + ": " + path + ">\n");
builder.append("\t" + container + "\n");
builder.append("\t\t" + lineNumber + ": " + lineContents + "\n\t\n");
return builder.toString();
}
}

56
molerat/src/main/java/it/unitn/molerat/repos/trackers/AbstractEvidenceTracker.java

@ -0,0 +1,56 @@
package it.unitn.molerat.repos.trackers;
import it.unitn.molerat.evidence.Changes;
import it.unitn.molerat.repos.wrappers.RepoWrapper;
import java.util.*;
public abstract class AbstractEvidenceTracker {
protected final String fixedRevision;
protected final String vulnRevision;
protected final String repoPath;
protected final RepoWrapper repoWrapper;
protected Set<Changes> changes;
protected Set<String> commits = new LinkedHashSet<>();
protected Set<String> restOfCommits = new LinkedHashSet<>();
public AbstractEvidenceTracker(RepoWrapper wrapper, String fixedRev) throws Exception {
this.fixedRevision = fixedRev;
this.repoPath = wrapper.getBasePath();
this.repoWrapper = wrapper;
Iterator<String> commitsIter = repoWrapper.getRevisionNumbers(fixedRev).iterator();
this.vulnRevision = commitsIter.next();
while (commitsIter.hasNext()) {
restOfCommits.add(commitsIter.next());
}
String initDiff = repoWrapper.doDiff(vulnRevision, fixedRevision);
this.changes = repoWrapper.inferChangesFromDiff(initDiff, vulnRevision, fixedRevision);
}
public RepoWrapper getRepoWrapper() {
return this.repoWrapper;
}
public String getFixedRevision() {
return this.fixedRevision;
}
public Set<String> getProcessedCommits() {
return this.commits;
}
// apart from non-java files, filter the files that contain "test" in ther name as well
protected Set<Changes> filterNonJavaChanges(Set<Changes> changes) {
Set<Changes> filteredChanges = new HashSet<>();
for (Changes change : changes) {
if (!change.getPath().endsWith(".java") || change.getPath().toLowerCase().contains("test")) {
filteredChanges.add(change);
}
this.repoWrapper.filterCommentsAndBlanks(change.getDeletions());
this.repoWrapper.filterCommentsAndBlanks(change.getAdditions());
}
changes.removeAll(filteredChanges);
return changes;
}
public abstract void trackEvidence() throws Exception;
}

108
molerat/src/main/java/it/unitn/molerat/repos/trackers/changes/ChangeEvidenceTracker.java

@ -0,0 +1,108 @@
package it.unitn.molerat.repos.trackers.changes;
import it.unitn.molerat.evidence.ChangeEvidence;
import it.unitn.molerat.repos.trackers.AbstractEvidenceTracker;
import it.unitn.molerat.repos.utils.CommitMetrics;
import it.unitn.molerat.repos.utils.SignatureExtractor;
import it.unitn.molerat.repos.wrappers.RepoWrapper;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
public class ChangeEvidenceTracker extends AbstractEvidenceTracker {
protected final Map<String, Set<ChangeEvidence>> evidences = new TreeMap<>();
protected Map<String, Set<String>> fixConstructs;
protected ChangeEvidenceTracker(RepoWrapper wrapper, String fixedRev) throws Exception {
super(wrapper, fixedRev);
}
public ChangeEvidenceTracker(RepoWrapper repoWrapper, String fixedCommit, Set<String> commits) throws Exception {
this(repoWrapper, fixedCommit);
this.commits = commits;
this.fixConstructs = extractSignatures(this.fixedRevision);
}
protected Map<String, Set<String>> extractSignatures(String commit) throws Exception {
Map<String, Set<String>> result = new TreeMap<>();
Map<String, String> files = CommitMetrics.getFileContentsPerRevision(commit, this.repoWrapper);
for (Map.Entry<String, String> file : files.entrySet()) {
SignatureExtractor se = new SignatureExtractor(file.getValue());
Set<String> signatures = new HashSet<>();
for (String signature : se.getSignaturesWithLines().keySet()) {
signatures.add(signature);
}
result.put(file.getKey(), signatures);
}
return result;
}
@Override
public void trackEvidence() throws Exception {
for (String commit : this.commits) {
// long timeNow = System.currentTimeMillis();
Map<String, Set<String>> currentSignatures = extractSignatures(commit);
for (Map.Entry<String, Set<String>> currentSignature : currentSignatures.entrySet()) {
Set<String> temp = this.fixConstructs.get(currentSignature.getKey());
if (temp == null) {
for (String sign : currentSignature.getValue()) {
ChangeEvidence evidence = new ChangeEvidence(
currentSignature.getKey(),
commit,
sign,
true
);
if (evidence