Browse Source

Added the modules and everything

tags/v1.0
standash 3 years ago
parent
commit
b4f1ccff9b
87 changed files with 28121 additions and 0 deletions
  1. +7
    -0
      .gitignore
  2. +136
    -0
      molerat/pom.xml
  3. +267
    -0
      molerat/src/main/java/it/unitn/molerat/cmd/Main.java
  4. +16
    -0
      molerat/src/main/java/it/unitn/molerat/cmd/experiments/AbstractExperiment.java
  5. +61
    -0
      molerat/src/main/java/it/unitn/molerat/cmd/experiments/CalculateAllChangesExperiment.java
  6. +53
    -0
      molerat/src/main/java/it/unitn/molerat/cmd/experiments/ChangeEvdPerformanceExperiment.java
  7. +45
    -0
      molerat/src/main/java/it/unitn/molerat/cmd/experiments/VulnEvdPerformanceExperiment.java
  8. +68
    -0
      molerat/src/main/java/it/unitn/molerat/data/csv/AbstractDataPoint.java
  9. +25
    -0
      molerat/src/main/java/it/unitn/molerat/data/csv/ChangeStatsDataPoint.java
  10. +21
    -0
      molerat/src/main/java/it/unitn/molerat/data/csv/FixStatsDataPoint.java
  11. +23
    -0
      molerat/src/main/java/it/unitn/molerat/data/csv/InputDataPoint.java
  12. +26
    -0
      molerat/src/main/java/it/unitn/molerat/data/csv/OutputDataPoint.java
  13. +21
    -0
      molerat/src/main/java/it/unitn/molerat/data/csv/PerfOutputDataPoint.java
  14. +22
    -0
      molerat/src/main/java/it/unitn/molerat/data/csv/VulnEvidenceDataPoint.java
  15. +245
    -0
      molerat/src/main/java/it/unitn/molerat/data/db/MongoWrapper.java
  16. +74
    -0
      molerat/src/main/java/it/unitn/molerat/data/memory/AnalysisEntry.java
  17. +46
    -0
      molerat/src/main/java/it/unitn/molerat/data/printers/VulnerabilityEvidencePrinter.java
  18. +45
    -0
      molerat/src/main/java/it/unitn/molerat/evidence/ChangeEvidence.java
  19. +93
    -0
      molerat/src/main/java/it/unitn/molerat/evidence/Changes.java
  20. +30
    -0
      molerat/src/main/java/it/unitn/molerat/evidence/GenericEvidence.java
  21. +32
    -0
      molerat/src/main/java/it/unitn/molerat/evidence/VulnerabilityEvidence.java
  22. +56
    -0
      molerat/src/main/java/it/unitn/molerat/repos/trackers/AbstractEvidenceTracker.java
  23. +108
    -0
      molerat/src/main/java/it/unitn/molerat/repos/trackers/changes/ChangeEvidenceTracker.java
  24. +13
    -0
      molerat/src/main/java/it/unitn/molerat/repos/trackers/changes/FullChangeEvidenceTracker.java
  25. +92
    -0
      molerat/src/main/java/it/unitn/molerat/repos/trackers/vuln/DeletionVulnerabilityEvidenceTracker.java
  26. +72
    -0
      molerat/src/main/java/it/unitn/molerat/repos/trackers/vuln/EnhancedDeletionVulnerabilityEvidenceTracker.java
  27. +71
    -0
      molerat/src/main/java/it/unitn/molerat/repos/trackers/vuln/FixStatisticsVulnerabilityEvidenceTracker.java
  28. +148
    -0
      molerat/src/main/java/it/unitn/molerat/repos/trackers/vuln/PatchedMethodBodyTracker.java
  29. +85
    -0
      molerat/src/main/java/it/unitn/molerat/repos/trackers/vuln/SliceDecayVulnerabilityEvidenceTracker.java
  30. +139
    -0
      molerat/src/main/java/it/unitn/molerat/repos/trackers/vuln/SliceVulnerabilityEvidenceTracker.java
  31. +234
    -0
      molerat/src/main/java/it/unitn/molerat/repos/trackers/vuln/VulnerabilityEvidenceTracker.java
  32. +73
    -0
      molerat/src/main/java/it/unitn/molerat/repos/trackers/vuln/VulnerabilityEvidenceTrackerFactory.java
  33. +98
    -0
      molerat/src/main/java/it/unitn/molerat/repos/utils/CommitMetrics.java
  34. +150
    -0
      molerat/src/main/java/it/unitn/molerat/repos/utils/IORoutines.java
  35. +233
    -0
      molerat/src/main/java/it/unitn/molerat/repos/utils/SignatureExtractor.java
  36. +204
    -0
      molerat/src/main/java/it/unitn/molerat/repos/wrappers/GitRepoWrapper.java
  37. +161
    -0
      molerat/src/main/java/it/unitn/molerat/repos/wrappers/RepoWrapper.java
  38. +185
    -0
      molerat/src/main/java/it/unitn/molerat/repos/wrappers/SvnRepoWrapper.java
  39. +20
    -0
      pom.xml
  40. +69
    -0
      repoman/pom.xml
  41. +87
    -0
      repoman/src/main/java/it/unitn/repoman/cmd/Main.java
  42. +9
    -0
      repoman/src/main/java/it/unitn/repoman/core/exceptions/InvalidLineException.java
  43. +59
    -0
      repoman/src/main/java/it/unitn/repoman/core/lang/LanguageFactory.java
  44. +221
    -0
      repoman/src/main/java/it/unitn/repoman/core/lang/parsers/c/C.tokens
  45. +1048
    -0
      repoman/src/main/java/it/unitn/repoman/core/lang/parsers/c/CBaseListener.java
  46. +618
    -0
      repoman/src/main/java/it/unitn/repoman/core/lang/parsers/c/CLexer.java
  47. +221
    -0
      repoman/src/main/java/it/unitn/repoman/core/lang/parsers/c/CLexer.tokens
  48. +852
    -0
      repoman/src/main/java/it/unitn/repoman/core/lang/parsers/c/CListener.java
  49. +7491
    -0
      repoman/src/main/java/it/unitn/repoman/core/lang/parsers/c/CParser.java
  50. +201
    -0
      repoman/src/main/java/it/unitn/repoman/core/lang/parsers/java/Java.tokens
  51. +1252
    -0
      repoman/src/main/java/it/unitn/repoman/core/lang/parsers/java/JavaBaseListener.java
  52. +545
    -0
      repoman/src/main/java/it/unitn/repoman/core/lang/parsers/java/JavaLexer.java
  53. +201
    -0
      repoman/src/main/java/it/unitn/repoman/core/lang/parsers/java/JavaLexer.tokens
  54. +1021
    -0
      repoman/src/main/java/it/unitn/repoman/core/lang/parsers/java/JavaListener.java
  55. +8421
    -0
      repoman/src/main/java/it/unitn/repoman/core/lang/parsers/java/JavaParser.java
  56. +110
    -0
      repoman/src/main/java/it/unitn/repoman/core/lang/traversals/c/CMethodCallTraversal.java
  57. +56
    -0
      repoman/src/main/java/it/unitn/repoman/core/lang/traversals/generic/AssignmentSymbolExtractTraversal.java
  58. +25
    -0
      repoman/src/main/java/it/unitn/repoman/core/lang/traversals/generic/ConditionTraversal.java
  59. +58
    -0
      repoman/src/main/java/it/unitn/repoman/core/lang/traversals/generic/DFSTraversal.java
  60. +51
    -0
      repoman/src/main/java/it/unitn/repoman/core/lang/traversals/generic/DotSymbolTraversal.java
  61. +52
    -0
      repoman/src/main/java/it/unitn/repoman/core/lang/traversals/generic/FormalParameterTraversal.java
  62. +64
    -0
      repoman/src/main/java/it/unitn/repoman/core/lang/traversals/generic/MethodCallTraversal.java
  63. +69
    -0
      repoman/src/main/java/it/unitn/repoman/core/lang/traversals/generic/MethodDeclarationTraversal.java
  64. +27
    -0
      repoman/src/main/java/it/unitn/repoman/core/lang/traversals/generic/NodeContainmentTraversal.java
  65. +56
    -0
      repoman/src/main/java/it/unitn/repoman/core/lang/traversals/generic/ScopesTraversal.java
  66. +39
    -0
      repoman/src/main/java/it/unitn/repoman/core/lang/traversals/generic/SeedTraversal.java
  67. +31
    -0
      repoman/src/main/java/it/unitn/repoman/core/lang/traversals/generic/StatementMappingTraversal.java
  68. +42
    -0
      repoman/src/main/java/it/unitn/repoman/core/lang/traversals/generic/SymbolExtractTraversal.java
  69. +9
    -0
      repoman/src/main/java/it/unitn/repoman/core/lang/traversals/generic/Traversal.java
  70. +125
    -0
      repoman/src/main/java/it/unitn/repoman/core/lang/traversals/java/JavaAPISignatureTraversal.java
  71. +114
    -0
      repoman/src/main/java/it/unitn/repoman/core/lang/traversals/java/JavaMethodCallTraversal.java
  72. +103
    -0
      repoman/src/main/java/it/unitn/repoman/core/lang/wrappers/c/CWrapper.java
  73. +137
    -0
      repoman/src/main/java/it/unitn/repoman/core/lang/wrappers/generic/Wrapper.java
  74. +114
    -0
      repoman/src/main/java/it/unitn/repoman/core/lang/wrappers/java/JavaWrapper.java
  75. +124
    -0
      repoman/src/main/java/it/unitn/repoman/core/slicers/LightweightSlice.java
  76. +35
    -0
      repoman/src/main/java/it/unitn/repoman/core/slicers/LightweightSlicePessimist.java
  77. +73
    -0
      repoman/src/main/java/it/unitn/repoman/core/slicers/tainters/AbstractTainter.java
  78. +75
    -0
      repoman/src/main/java/it/unitn/repoman/core/slicers/tainters/BackwardTainter.java
  79. +104
    -0
      repoman/src/main/java/it/unitn/repoman/core/slicers/tainters/BackwardTainterPessimist.java
  80. +140
    -0
      repoman/src/main/java/it/unitn/repoman/core/slicers/tainters/ForwardTainter.java
  81. +119
    -0
      repoman/src/main/java/it/unitn/repoman/core/slicers/tainters/ForwardTainterPessimist.java
  82. +38
    -0
      repoman/src/main/java/it/unitn/repoman/core/slicers/tainters/MethodCallTainter.java
  83. +29
    -0
      repoman/src/main/java/it/unitn/repoman/core/slicers/tainters/TaintedVariable.java
  84. +95
    -0
      repoman/src/main/java/it/unitn/repoman/core/slicers/tainters/TaintedVariableSet.java
  85. +102
    -0
      repoman/src/main/java/it/unitn/repoman/core/utils/printers/ConsolePrinterListener.java
  86. +18
    -0
      repoman/src/main/java/it/unitn/repoman/core/utils/printers/HTMLPrinter.java
  87. +73
    -0
      repoman/src/main/java/it/unitn/repoman/core/utils/printers/TextPrinterListener.java

+ 7
- 0
.gitignore View File

@@ -0,0 +1,7 @@
.classpath
.project
*.iml
*.ipr
*.iws
.idea/
target/

+ 136
- 0
molerat/pom.xml View File

@@ -0,0 +1,136 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<!-- parent project -->
<parent>
<groupId>it.unitn</groupId>
<artifactId>foss-vuln-tracker</artifactId>
<version>1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>it.unitn.molerat</groupId>
<artifactId>molerat</artifactId>
<version>1.0-SNAPSHOT</version>
<packaging>jar</packaging>

<!-- declare dependencies -->
<dependencies>
<dependency>
<groupId>org.antlr</groupId>
<artifactId>antlr4-runtime</artifactId>
<version>4.7</version>
</dependency>
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>bson</artifactId>
<version>3.4.2</version>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
<version>1.4</version>
</dependency>
<dependency>
<groupId>it.unitn.repoman</groupId>
<artifactId>repoman</artifactId>
<version>1.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.5</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.4</version>
</dependency>
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongodb-driver</artifactId>
<version>3.4.2</version>
</dependency>
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongodb-driver-core</artifactId>
<version>3.4.2</version>
</dependency>
<dependency>
<groupId>org.eclipse.jgit</groupId>
<artifactId>org.eclipse.jgit</artifactId>
<version>4.2.0.201601211800-r</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.25</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-nop</artifactId>
<version>1.7.25</version>
</dependency>
<dependency>
<groupId>org.tmatesoft.svnkit</groupId>
<artifactId>svnkit-cli</artifactId>
<version>1.8.13</version>
</dependency>
<dependency>
<groupId>org.tmatesoft.svnkit</groupId>
<artifactId>svnkit</artifactId>
<version>1.8.13</version>
</dependency>
<dependency>
<groupId>org.tmatesoft.svnkit</groupId>
<artifactId>svnkit-javahl16</artifactId>
<version>1.8.13</version>
</dependency>
<!--
<dependency>
<groupId>org.tmatesoft.svnkit</groupId>
<artifactId>svnkit-javahl</artifactId>
<version>1.3.5</version>
</dependency>
-->
</dependencies>

<!-- build plugins -->
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<version>3.0.1</version>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>prepare-package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>
${project.build.directory}/libs
</outputDirectory>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<version>3.0.2</version>
<configuration>
<archive>
<manifest>
<addClasspath>true</addClasspath>
<classpathPrefix>libs/</classpathPrefix>
<mainClass>it.unitn.molerat.cmd.Main</mainClass>
</manifest>
</archive>
</configuration>
</plugin>
</plugins>
</build>
</project>

+ 267
- 0
molerat/src/main/java/it/unitn/molerat/cmd/Main.java View File

@@ -0,0 +1,267 @@
package it.unitn.molerat.cmd;

import it.unitn.molerat.data.csv.InputDataPoint;
import it.unitn.molerat.data.csv.VulnEvidenceDataPoint;
import it.unitn.molerat.data.db.MongoWrapper;
import it.unitn.molerat.data.memory.AnalysisEntry;
import it.unitn.molerat.evidence.VulnerabilityEvidence;
import it.unitn.molerat.repos.trackers.vuln.VulnerabilityEvidenceTracker;
import it.unitn.molerat.repos.trackers.vuln.VulnerabilityEvidenceTrackerFactory;
import it.unitn.molerat.repos.utils.IORoutines;

import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
import org.apache.commons.cli.*;
import org.bson.types.ObjectId;

public class Main {

private static MongoWrapper db = new MongoWrapper("molerat");

public static void main(String[] args) {
Options opts = new Options();

Option listTrackersOpt = Option.builder()
.longOpt("list-trackers")
.desc("List the available trackers for vulnerability molerat.evidence")
.build();
opts.addOption(listTrackersOpt);

Option projectNameOpt = Option.builder()
.longOpt("project-name")
.desc("The name of the project")
.hasArg()
.build();
opts.addOption(projectNameOpt);

Option repoTypeOpt = Option.builder()
.longOpt("repo-type")
.desc("The type of the source code repository ('git' or 'svn')")
.hasArg()
.build();
opts.addOption(repoTypeOpt);

Option repoPathOpt = Option.builder()
.longOpt("repo-path")
.desc("The path of the working copy of the source code repository")
.hasArg()
.build();
opts.addOption(repoPathOpt);

Option cveIdOpt = Option.builder()
.longOpt("cve-id")
.desc("The CVE identifier of a vulnerability")
.hasArg()
.build();
opts.addOption(cveIdOpt);

Option fixCommitOpt = Option.builder()
.longOpt("fix-commit")
.desc("The commit that fixed a vulnerability")
.hasArg()
.build();
opts.addOption(fixCommitOpt);

Option trackerTypeOpt = Option.builder()
.longOpt("tracker-type")
.desc("The type of the vulnerability molerat.evidence tracker")
.hasArg()
.build();
opts.addOption(trackerTypeOpt);

Option inputFileOpt = Option.builder()
.longOpt("input-file")
.desc("Path to the input .csv file")
.hasArg()
.argName("input-file-path")
.build();
opts.addOption(inputFileOpt);

Option outputFileOpt = Option.builder()
.longOpt("output-file")
.desc("Path to the output .csv file")
.hasArg()
.argName("output-file-path")
.build();
opts.addOption(outputFileOpt);

CommandLineParser cmdParser = new DefaultParser();
HelpFormatter helpFormatter = new HelpFormatter();
CommandLine cmd;
try {
cmd = cmdParser.parse(opts, args);
if (cmd.getOptions().length == 0) {
throw new ParseException("Arguments are not specified");
}

if (cmd.hasOption("list-trackers")) {
System.out.println(VulnerabilityEvidenceTrackerFactory.getTrackersList());
return;
}

if (cmd.hasOption("input-file")) {
String i = cmd.getOptionValue("input-file");
collectVulnEvidence(i);
}
if (cmd.hasOption("output-file")) {
String o = cmd.getOptionValue("output-file");
generateCsv(o);
}

if (!cmd.hasOption("input-file") && !cmd.hasOption("output-file")) {
String projectName = null;
String repoType = null;
String repoPath = null;
String cveId = null;
String fixCommit = null;
String trackerType = null;

if (cmd.hasOption("project-name")) {
projectName = cmd.getOptionValue("project-name");
}
if (cmd.hasOption("repo-type")) {
repoType = cmd.getOptionValue("repo-type");
}
if(cmd.hasOption("repo-path")) {
repoPath = cmd.getOptionValue("repo-path");
}
if(cmd.hasOption("cve-id")) {
cveId = cmd.getOptionValue("cve-id");
}
if(cmd.hasOption("fix-commit")) {
fixCommit = cmd.getOptionValue("fix-commit");
}
if(cmd.hasOption("tracker-type")) {
trackerType = cmd.getOptionValue("tracker-type");
}
if (projectName == null) {
throw new ParseException("The 'project-name' parameter is not specified");
}
if (repoType == null) {
throw new ParseException("The 'repo-type' parameter is not specified");
}
if (repoPath == null) {
throw new ParseException("The 'repo-path' parameter is not specified");
}
if (cveId == null) {
throw new ParseException("The 'cve-id' parameter is not specified");
}
if (fixCommit == null) {
throw new ParseException("The 'fix-commit' parameter is not specified");
}
if (trackerType == null) {
throw new ParseException("The 'tracker-type' parameter is not specified");
}
collect(projectName, repoType, repoPath, cveId, fixCommit, trackerType);
}
}
catch (ParseException e) {
System.out.println("ERROR: " + e.getMessage());
helpFormatter.printHelp("utility-name", opts);
}
}

private static void collectVulnEvidence(String inputCsvPath) {
try {
Set<InputDataPoint> inputs = IORoutines.readInputDataPoints(inputCsvPath);
for (InputDataPoint ip : inputs) {
collect(ip.PROJECTID,
ip.REPO_TYPE,
ip.REPO_ROOT,
ip.CVEID,
ip.FIX_REV,
ip.TRACKER_TYPE);
}
} catch (Exception e) {
System.out.println("ERROR: " + e.getMessage());
}
}

private static void collect(String projectName, String repositoryType,
String repositoryPath, String cveName, String fixCommit, String trackerType) {
try {
boolean entryExists = db.analysisEntryExists(projectName, cveName, repositoryType, repositoryPath);
if (entryExists) {
System.out.format("WARNING: The analysis entry for '%s' from '%s' already exists, skipping\n", cveName, projectName);
return;
}

System.out.format("INFO: Collecting the molerat.evidence for '%s' from '%s'\n", cveName, projectName);
VulnerabilityEvidenceTracker vulnTracker = VulnerabilityEvidenceTrackerFactory.getTracker(
repositoryPath,
fixCommit,
repositoryType,
trackerType
);
vulnTracker.trackEvidence();

AnalysisEntry entry = new AnalysisEntry(
cveName,
projectName,
repositoryType,
repositoryPath,
fixCommit,
vulnTracker.getProcessedCommits(),
vulnTracker.getEvidences(),
null // we're not tracking any changes so far
);
boolean isSuccessful = db.insertAnalysisEntry(entry);
if (!isSuccessful) {
System.out.format("WARNING: Could not get any molerat.evidence for '%s' in '%s'\n", cveName, projectName);
}
System.out.println("INFO: Done!");
}
catch (Exception e) {
System.out.println("ERROR: " + e.getMessage());
}
}

private static void generateCsv(String outFilePath) {
System.out.format("INFO: generating the .csv file '%s'\n", outFilePath);
String csvHeader = new VulnEvidenceDataPoint().getHeader() + "\n";
try {
IORoutines.writeFile(outFilePath, csvHeader);
Set<ObjectId> cveIds = db.getAllCveIds();
for (ObjectId cveId : cveIds) {
AnalysisEntry analysis = db.getAnalysisEntry(cveId);
if (analysis == null) {
continue;
}
String projectName = analysis.getProjectName();
String cveName = analysis.getCveName();

System.out.format("INFO: writing the entries for '%s'\n", cveName);
Set<VulnerabilityEvidence> evidences = analysis.getVulnEvidencesSet();
if (evidences.size() == 0) {
throw new Exception(String.format("There is no molerat.evidence for '%s' in the database", cveName));
}
Map<String, Integer> locsCount = new LinkedHashMap<>();
for (VulnerabilityEvidence evd : evidences) {
if (locsCount.containsKey(evd.getCommit())) {
int count = locsCount.get(evd.getCommit());
locsCount.remove(evd.getCommit());
locsCount.put(evd.getCommit(), ++count);
} else {
locsCount.put(evd.getCommit(), 1);
}
}
int timestamp = 0;
for (Map.Entry<String, Integer> entry : locsCount.entrySet()) {
VulnEvidenceDataPoint dp = new VulnEvidenceDataPoint(new String[]{
projectName,
cveName,
entry.getKey(),
String.valueOf(timestamp--),
String.valueOf(entry.getValue())
});
IORoutines.writeFile(outFilePath, dp.toString() + "\n");
}

}
} catch (Exception e) {
System.out.println("ERROR: " + e.getMessage());
}
}
}


+ 16
- 0
molerat/src/main/java/it/unitn/molerat/cmd/experiments/AbstractExperiment.java View File

@@ -0,0 +1,16 @@
package it.unitn.molerat.cmd.experiments;

import it.unitn.molerat.data.csv.InputDataPoint;
import it.unitn.molerat.repos.utils.IORoutines;
import java.util.Set;

public class AbstractExperiment {

protected static Set<InputDataPoint> readInputDataPoints(String path) throws Exception {
return IORoutines.readInputDataPoints(path);
}

protected static void saveOutput(String path, String output) throws Exception {
IORoutines.writeFile(path, output + "\n");
}
}

+ 61
- 0
molerat/src/main/java/it/unitn/molerat/cmd/experiments/CalculateAllChangesExperiment.java View File

@@ -0,0 +1,61 @@
package it.unitn.molerat.cmd.experiments;

import it.unitn.molerat.data.csv.InputDataPoint;
import it.unitn.molerat.data.csv.OutputDataPoint;
import it.unitn.molerat.data.csv.PerfOutputDataPoint;
import it.unitn.molerat.repos.utils.CommitMetrics;
import it.unitn.molerat.repos.wrappers.GitRepoWrapper;
import it.unitn.molerat.repos.wrappers.RepoWrapper;

import java.util.Set;

public class CalculateAllChangesExperiment extends AbstractExperiment {
public static void main(String[] args) {
if (args.length < 2) {
System.out.println("USAGE: [input.csv] [output.csv]");
return;
}
String inputCsvPath = args[0];
String outputCsvPath = args[1];

try {
Set<InputDataPoint> inputs = readInputDataPoints(inputCsvPath);
saveOutput(outputCsvPath, new PerfOutputDataPoint().getHeader());
for (InputDataPoint idp : inputs) {

System.out.println("Processing the " + idp.CVEID + "...");
long timeNow = System.currentTimeMillis();

RepoWrapper wrapper = new GitRepoWrapper(idp.REPO_ROOT);
Set<String> processedCommits = wrapper.getRevisionNumbers(idp.FIX_REV);
int timestamp = 0;
for (String commit : processedCommits) {

int publicAPICount = CommitMetrics.getNumberOfPublicMethodsPerRevision(commit, wrapper);
int rem = CommitMetrics.getGlobalPublicMethodsRemoved(commit, idp.FIX_REV, wrapper);
double currentUntouched = (((double) publicAPICount - (double) rem) / (double) publicAPICount) * 100.0;
currentUntouched = Math.round(currentUntouched * 100.0) / 100.0;

String[] output = new String[]{
idp.CVEID,
idp.TRACKER_TYPE,
"empty",
commit,
String.valueOf(timestamp),
"empty",
String.valueOf(publicAPICount),
String.valueOf(rem),
String.valueOf(currentUntouched)
};
OutputDataPoint odp = new OutputDataPoint(output);
saveOutput(outputCsvPath, odp.toString());
timestamp--;
}
long timeAfter = System.currentTimeMillis();
System.out.println("TIME (ms): " + (timeAfter - timeNow));
}
} catch (Exception e) {
e.printStackTrace();
}
}
}

+ 53
- 0
molerat/src/main/java/it/unitn/molerat/cmd/experiments/ChangeEvdPerformanceExperiment.java View File

@@ -0,0 +1,53 @@
package it.unitn.molerat.cmd.experiments;

import it.unitn.molerat.data.csv.InputDataPoint;
import it.unitn.molerat.data.csv.PerfOutputDataPoint;
import it.unitn.molerat.repos.trackers.vuln.VulnerabilityEvidenceTracker;
import it.unitn.molerat.repos.trackers.vuln.VulnerabilityEvidenceTrackerFactory;
import it.unitn.molerat.repos.utils.CommitMetrics;
import java.util.Set;

public class ChangeEvdPerformanceExperiment extends AbstractExperiment {
public static void main(String[] args) {
if (args.length < 2) {
System.out.println("USAGE: [input.csv] [output.csv]");
return;
}
String inputCsvPath = args[0];
String outputCsvPath = args[1];

try {
Set<InputDataPoint> inputs = readInputDataPoints(inputCsvPath);
saveOutput(outputCsvPath, new PerfOutputDataPoint().getHeader());
for (InputDataPoint idp : inputs) {

System.out.println("Processing the " + idp.CVEID + "...");

VulnerabilityEvidenceTracker vulnTracker = VulnerabilityEvidenceTrackerFactory.getTracker(idp.REPO_ROOT, idp.FIX_REV, idp.REPO_TYPE, idp.TRACKER_TYPE);
vulnTracker.trackEvidence();

Set<String> processedCommits = vulnTracker.getProcessedCommits();
for (String commit : processedCommits) {
long timeNow = System.currentTimeMillis();

int publicAPICount = CommitMetrics.getNumberOfPublicMethodsPerRevision(commit, vulnTracker.getRepoWrapper());
int rem = CommitMetrics.getGlobalPublicMethodsRemoved(commit, vulnTracker.getFixedRevision(), vulnTracker.getRepoWrapper());
double currentUntouched = (((double) publicAPICount - (double) rem) / (double) publicAPICount) * 100.0;
currentUntouched = Math.round(currentUntouched * 100.0) / 100.0;

long timeAfter = System.currentTimeMillis();
String[] output = new String[]{
idp.CVEID,
idp.TRACKER_TYPE,
String.valueOf(timeAfter - timeNow)
};
PerfOutputDataPoint odp = new PerfOutputDataPoint(output);
saveOutput(outputCsvPath, odp.toString());
}
}
} catch (Exception e) {
e.printStackTrace();
}

}
}

+ 45
- 0
molerat/src/main/java/it/unitn/molerat/cmd/experiments/VulnEvdPerformanceExperiment.java View File

@@ -0,0 +1,45 @@
package it.unitn.molerat.cmd.experiments;

import it.unitn.molerat.data.csv.InputDataPoint;
import it.unitn.molerat.data.csv.PerfOutputDataPoint;
import it.unitn.molerat.repos.trackers.vuln.VulnerabilityEvidenceTracker;
import it.unitn.molerat.repos.trackers.vuln.VulnerabilityEvidenceTrackerFactory;
import java.util.Set;

public class VulnEvdPerformanceExperiment extends AbstractExperiment {

public static void main(String[] args) {
if (args.length < 2) {
System.out.println("USAGE: [input.csv] [output.csv]");
return;
}
String inputCsvPath = args[0];
String outputCsvPath = args[1];

try {
Set<InputDataPoint> inputs = readInputDataPoints(inputCsvPath);
saveOutput(outputCsvPath, new PerfOutputDataPoint().getHeader());
for (InputDataPoint idp : inputs) {

System.out.println("Processing the " + idp.CVEID + "...");
long timeNow = System.currentTimeMillis();

VulnerabilityEvidenceTracker vulnTracker = VulnerabilityEvidenceTrackerFactory.getTracker(idp.REPO_ROOT, idp.FIX_REV, idp.REPO_TYPE, idp.TRACKER_TYPE);
vulnTracker.trackEvidence();

long timeAfter = System.currentTimeMillis();

String[] output = new String[]{
idp.CVEID,
idp.TRACKER_TYPE,
String.valueOf(timeAfter - timeNow)
};
PerfOutputDataPoint odp = new PerfOutputDataPoint(output);
saveOutput(outputCsvPath, odp.toString());

}
} catch (Exception e) {
e.printStackTrace();
}
}
}

+ 68
- 0
molerat/src/main/java/it/unitn/molerat/data/csv/AbstractDataPoint.java View File

@@ -0,0 +1,68 @@
package it.unitn.molerat.data.csv;

import java.lang.reflect.Field;

public abstract class AbstractDataPoint {

public AbstractDataPoint(String[] entries) throws Exception {
Field[] fields = this.getClass().getFields();
if (entries.length != fields.length) {
throw new Exception("Bad entries for initializing a molerat.data point!");
}
int i = 0;
for (Field field : fields) {
if (!field.getName().equals("this$0")) {
try {
field.set(this, entries[i++]);
} catch (IllegalArgumentException e) {
System.out.println("ERROR: " + e.getMessage());
} catch (IllegalAccessException e) {
System.out.println("ERROR: " + e.getMessage());
}
}
}
}
public AbstractDataPoint(String str) throws Exception {
this(str.replace(" ", "").split(","));
}
public AbstractDataPoint() {

}
@Override
public final String toString() {
StringBuilder builder = new StringBuilder();
Field[] fields = this.getClass().getFields();
for (Field field : fields) {
if (!field.getName().equals("this$0")) {
try {
builder.append(field.get(this) + ",");
} catch (IllegalArgumentException e) {
System.out.println("ERROR: " + e.getMessage());
} catch (IllegalAccessException e) {
System.out.println("ERROR: " + e.getMessage());
}
}
}
builder.deleteCharAt(builder.length()-1);
return builder.toString();
}
public final String getHeader() {
StringBuilder builder = new StringBuilder();
Field[] fields = this.getClass().getFields();
for (Field field : fields) {
if (!field.getName().equals("this$0")) {
try {
builder.append(field.getName() + ",");
} catch (IllegalArgumentException e) {
System.out.println("ERROR: " + e.getMessage());
}
}
}
builder.deleteCharAt(builder.length()-1);
return builder.toString();
}
}

+ 25
- 0
molerat/src/main/java/it/unitn/molerat/data/csv/ChangeStatsDataPoint.java View File

@@ -0,0 +1,25 @@
package it.unitn.molerat.data.csv;

/**
* Created by standash on 06/12/2016.
*/
public class ChangeStatsDataPoint extends AbstractDataPoint {
public String PROJECT;
public String CURRENT_COMMIT;
public String ADDED_LINES;
public String DELETED_LINES;
public String TOUCHED_METHODS;
public String TOUCHED_FILES;

public ChangeStatsDataPoint(String[] entries) throws Exception {
super(entries);
}

public ChangeStatsDataPoint(String str) throws Exception {
super(str);
}

public ChangeStatsDataPoint() {

}
}

+ 21
- 0
molerat/src/main/java/it/unitn/molerat/data/csv/FixStatsDataPoint.java View File

@@ -0,0 +1,21 @@
package it.unitn.molerat.data.csv;

public class FixStatsDataPoint extends AbstractDataPoint {
public String CVEID;
public String ADDED_LINES;
public String DELETED_LINES;
public String TOUCHED_METHODS;
public String TOUCHED_FILES;

public FixStatsDataPoint(String[] entries) throws Exception {
super(entries);
}

public FixStatsDataPoint(String str) throws Exception {
super(str);
}

public FixStatsDataPoint() {

}
}

+ 23
- 0
molerat/src/main/java/it/unitn/molerat/data/csv/InputDataPoint.java View File

@@ -0,0 +1,23 @@
package it.unitn.molerat.data.csv;

public final class InputDataPoint extends AbstractDataPoint {

public String PROJECTID;
public String CVEID;
public String REPO_TYPE;
public String REPO_ROOT;
public String FIX_REV;
public String TRACKER_TYPE;
public InputDataPoint(String[] entries) throws Exception {
super(entries);
}

public InputDataPoint(String str) throws Exception {
super(str);
}

public InputDataPoint() {

}
}

+ 26
- 0
molerat/src/main/java/it/unitn/molerat/data/csv/OutputDataPoint.java View File

@@ -0,0 +1,26 @@
package it.unitn.molerat.data.csv;

public class OutputDataPoint extends AbstractDataPoint {
public String CVEID;
public String TRACKING_METHOD;
public String CURRENT_FILE;
public String CURRENT_REV;
public String TIMESTAMP;
public String EVIDENCE_LOC;
//public String GLOBAL_PUBLIC_API_COUNT;
//public String GLOBAL_PUBLIC_API_REMOVED;
//public String GLOBAL_PUBLIC_API_UNTOUCHED;

public OutputDataPoint(String[] entries) throws Exception {
super(entries);
}

public OutputDataPoint(String str) throws Exception {
super(str);
}

public OutputDataPoint() {

}
}

+ 21
- 0
molerat/src/main/java/it/unitn/molerat/data/csv/PerfOutputDataPoint.java View File

@@ -0,0 +1,21 @@
package it.unitn.molerat.data.csv;

public class PerfOutputDataPoint extends AbstractDataPoint {

public String CVEID;
public String TRACKING_METHOD;
public String TIME_MS;

public PerfOutputDataPoint(String[] entries) throws Exception {
super(entries);
}

public PerfOutputDataPoint(String str) throws Exception {
super(str);
}

public PerfOutputDataPoint() {

}

}

+ 22
- 0
molerat/src/main/java/it/unitn/molerat/data/csv/VulnEvidenceDataPoint.java View File

@@ -0,0 +1,22 @@
package it.unitn.molerat.data.csv;

public class VulnEvidenceDataPoint extends AbstractDataPoint {

public String PROJECT_NAME;
public String CVEID;
public String CURRENT_REV;
public String TIMESTAMP;
public String EVIDENCE_LOC;

public VulnEvidenceDataPoint(String[] entries) throws Exception {
super(entries);
}

public VulnEvidenceDataPoint(String str) throws Exception {
super(str);
}

public VulnEvidenceDataPoint() {

}
}

+ 245
- 0
molerat/src/main/java/it/unitn/molerat/data/db/MongoWrapper.java View File

@@ -0,0 +1,245 @@
package it.unitn.molerat.data.db;

import com.mongodb.BasicDBObject;
import com.mongodb.MongoClient;
import com.mongodb.client.FindIterable;
import com.mongodb.client.MongoCollection;
import com.mongodb.client.MongoDatabase;
import com.mongodb.client.model.Indexes;
import it.unitn.molerat.data.memory.AnalysisEntry;
import it.unitn.molerat.evidence.VulnerabilityEvidence;
import org.bson.Document;
import org.bson.types.ObjectId;
import java.util.*;

public class MongoWrapper {

private static final String projectCollection = "projects";
private static final String vulnsCollection = "vulns";
private static final String vulnEvdCollection = "vuln_evidences";

private MongoClient client = null;
private MongoDatabase db = null;

public MongoWrapper(String dbName) {
this.client = new MongoClient();
this.db = client.getDatabase(dbName);
}

private FindIterable<Document> getEntries(BasicDBObject query, String collection) {
MongoCollection<Document> docs = db.getCollection(collection);
return docs.find(query);
}

private Document getEntry(BasicDBObject query, String collection) {
return getEntries(query, collection).first();
}

private ObjectId getEntryId(Document doc) {
return (doc != null) ? doc.getObjectId("_id") : null;
}

private Document getProjectEntry(String name, String repoType, String repoPath) {
BasicDBObject query = new BasicDBObject();
query.put("name", name);
query.put("repo_type", repoType);
query.put("repo_path", repoPath);
return getEntry(query, projectCollection);
}

private Document getProjectEntry(ObjectId projectId) {
BasicDBObject query = new BasicDBObject();
query.put("_id", projectId);
return getEntry(query, projectCollection);
}

private ObjectId getProjectId(String name, String repoType, String repoPath) {
return getEntryId(getProjectEntry(name, repoType, repoPath));
}

private ObjectId getProjectId(ObjectId cveId) {
return getCveEntry(cveId).getObjectId("owner_id");
}

private Document getCveEntry(ObjectId projectId, String cve) {
BasicDBObject query = new BasicDBObject();
query.put("cve", cve);
query.put("owner_id", projectId);
return getEntry(query, vulnsCollection);
}

private Document getCveEntry(ObjectId cveId) {
BasicDBObject query = new BasicDBObject();
query.put("_id", cveId);
return getEntry(query, vulnsCollection);
}

private ObjectId getCveId(ObjectId projectId, String cve) {
return getEntryId(getCveEntry(projectId, cve));
}

public Set<ObjectId> getAllCveIds() {
Set<ObjectId> ids = new LinkedHashSet<>();
MongoCollection<Document> vulns = db.getCollection(vulnsCollection);
FindIterable<Document> docs = vulns.find();
for (Document doc : docs) {
ids.add(doc.getObjectId("_id"));
}
return ids;
}

private FindIterable<Document> getVulnEvidences(ObjectId cveId) {
BasicDBObject query = new BasicDBObject() ;
query.put("owner_id", cveId);
FindIterable<Document> result = getEntries(query, vulnEvdCollection);
result.sort(new BasicDBObject("order", -1));
return result;
}

private ObjectId insertProject(String name, String repoType, String repoPath) {
Document proj = getProjectEntry(name, repoType, repoPath);
if (proj == null) {
MongoCollection<Document> projects = db.getCollection(projectCollection);
proj = new Document("name", name)
.append("repo_type", repoType)
.append("repo_path", repoPath);
projects.insertOne(proj);

}
return getEntryId(proj);
}

private ObjectId insertCve(ObjectId projectId, String cve, String fixCommit) {
Document vuln = getCveEntry(projectId, cve);
if (vuln == null) {
MongoCollection<Document> vulns = db.getCollection(vulnsCollection);
vuln = new Document("cve", cve)
.append("fix_commit", fixCommit)
.append("owner_id", projectId)
.append("processed", false);
vulns.insertOne(vuln);
MongoCollection<Document> projects = db.getCollection(projectCollection);
projects.findOneAndUpdate(new BasicDBObject("_id", projectId), new BasicDBObject("$push",
new BasicDBObject("vulns", getEntryId(vuln))));
}
return getEntryId(vuln);
}

private boolean isVulnProcessed(ObjectId cveId) {
MongoCollection<Document> vulns = db.getCollection(vulnsCollection);
Document cve = vulns.find(new BasicDBObject("_id", cveId)).first();
return (Boolean)cve.get("processed");
}

public boolean insertAnalysisEntry(AnalysisEntry entry) {
// if there's no molerat.evidence for some reason, return false
if (entry.getVulnEvidences().isEmpty()) {
return false;
}
ObjectId projectId = insertProject(entry.getProjectName(), entry.getRepositoryType(), entry.getRepositoryPath());
ObjectId cveId = insertCve(projectId, entry.getCveName(), entry.getFixCommit());
MongoCollection<Document> vulns = db.getCollection(vulnsCollection);
MongoCollection<Document> evds = db.getCollection(vulnEvdCollection);

if (isVulnProcessed(cveId)) {
vulns.updateOne(
new BasicDBObject("_id", cveId),
new BasicDBObject("$set", new BasicDBObject("processed", false))
);
evds.deleteMany(new BasicDBObject("owner_id", cveId));
}

List<Document> records = new LinkedList<Document>();
int order= 0;
Set<String> commits = entry.getCommits();
for (String commit : commits) {
Set<VulnerabilityEvidence> evidences = entry.getVulnEvidences().get(commit);
if (evidences != null) {
for (VulnerabilityEvidence e : evidences) {
BasicDBObject record = new BasicDBObject();
record.put("owner_id", cveId);
record.put("revision", commit);
record.put("order", order);
record.put("file_path", e.getPath());
record.put("container", e.getContainer());
record.put("line_number", e.getLineNumber());
record.put("line_contents", e.getLineContents());
Document doc = new Document(record);
records.add(doc);
}
}
order--;
}
evds.insertMany(records);
vulns.updateOne(
new BasicDBObject("_id", cveId),
new BasicDBObject("$set", new BasicDBObject("processed", true))
);
//create index
evds.createIndex(Indexes.descending("order"));
return true;
}

public AnalysisEntry getAnalysisEntry(String projectName, String cveName, String repoType, String repoPath) {
ObjectId projectId = getProjectId(projectName, repoType, repoPath);
ObjectId cveId = getCveId(projectId, cveName);
return getAnalysisEntry(cveId);
}

public boolean analysisEntryExists(String projectName, String cveName, String repoType, String repoPath) {
ObjectId projectId = getProjectId(projectName, repoType, repoPath);
ObjectId cveId = getCveId(projectId, cveName);
return (projectId != null && cveId != null);
}

public Set<AnalysisEntry> getAnalysisEntries() {
Set<AnalysisEntry> entries = new LinkedHashSet<>();
FindIterable<Document> cves = db.getCollection(vulnsCollection).find();
for (Document cve : cves) {
ObjectId cveId = cve.getObjectId("_id");
AnalysisEntry entry = getAnalysisEntry(cveId);
entries.add(entry);
}
return entries;
}

public AnalysisEntry getAnalysisEntry(ObjectId cveId) {
if (!isVulnProcessed(cveId)) {
return null;
}
ObjectId projectId = getProjectId(cveId);
Document projectDoc = getProjectEntry(projectId);
String projectName = projectDoc.getString("name");
String repoType = projectDoc.getString("repo_type");
String repoPath = projectDoc.getString("repo_path");

Document cveDoc = getCveEntry(cveId);
String cveName = cveDoc.getString("cve");
String fixCommit = cveDoc.getString("fix_commit");

Map<String, Set<VulnerabilityEvidence>> vulnEvidences = new LinkedHashMap<>();
FindIterable<Document> docs = getVulnEvidences(cveId);
for (Document doc : docs) {
String filePath = doc.getString("file_path");
String container = doc.getString("container");
String revision = doc.getString("revision");
int lineNumber = doc.getInteger("line_number");
String lineContents = doc.getString("line_contents");
VulnerabilityEvidence evidence = new VulnerabilityEvidence(filePath, revision, container, lineNumber, lineContents);
if (vulnEvidences.containsKey(revision)) {
vulnEvidences.get(revision).add(evidence);
}
else {
Set<VulnerabilityEvidence> set = new LinkedHashSet<>();
set.add(evidence);
vulnEvidences.put(evidence.getCommit(), set);
}
}
/* TODO: add stuff for getting the changes molerat.evidence
Map<String, Set<ChangeEvidence>> chgEvidences = new TreeMap<>();
*/
return new AnalysisEntry(cveName, projectName, repoType, repoPath, fixCommit, vulnEvidences.keySet(), vulnEvidences, null);
}
}

+ 74
- 0
molerat/src/main/java/it/unitn/molerat/data/memory/AnalysisEntry.java View File

@@ -0,0 +1,74 @@
package it.unitn.molerat.data.memory;

import it.unitn.molerat.evidence.ChangeEvidence;
import it.unitn.molerat.evidence.VulnerabilityEvidence;

import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;

public class AnalysisEntry {
private final String cveId;
private final String projectId;
private final String fixCommit;
private final String repositoryType;
private final String repositoryPath;
private final Set<String> commits;
private final Map<String, Set<VulnerabilityEvidence>> vulnEvidences;
private final Map<String, Set<ChangeEvidence>> changEvidences;

public AnalysisEntry(String cveId, String projectId, String repositoryType, String repositoryPath,
String fixCommit, Set<String> commits,
Map<String, Set<VulnerabilityEvidence>> vulnEvidences,
Map<String, Set<ChangeEvidence>> changEvidences) {
this.cveId = cveId;
this.projectId = projectId;
this.repositoryType = repositoryType;
this.repositoryPath = repositoryPath;
this.commits = commits;
this.fixCommit = fixCommit;
this.vulnEvidences = vulnEvidences;
this.changEvidences = changEvidences;
}

public String getCveName() {
return cveId;
}

public Set<String> getCommits() {
return commits;
}

public String getFixCommit() {
return fixCommit;
}

public String getRepositoryType() {
return repositoryType;
}

public String getRepositoryPath() {
return repositoryPath;
}

public Map<String, Set<VulnerabilityEvidence>> getVulnEvidences() {
return vulnEvidences;
}

public Set<VulnerabilityEvidence> getVulnEvidencesSet() {
Set<VulnerabilityEvidence> evidences = new LinkedHashSet<>();
for (Set<VulnerabilityEvidence> eset : vulnEvidences.values()) {
evidences.addAll(eset);
}
return evidences;
}

public Map<String, Set<ChangeEvidence>> getChangEvidences() {
return changEvidences;
}

public String getProjectName() {
return projectId;
}

}

+ 46
- 0
molerat/src/main/java/it/unitn/molerat/data/printers/VulnerabilityEvidencePrinter.java View File

@@ -0,0 +1,46 @@
package it.unitn.molerat.data.printers;

import it.unitn.molerat.evidence.VulnerabilityEvidence;

import java.util.*;

public class VulnerabilityEvidencePrinter {

private final StringBuilder builder = new StringBuilder();

public VulnerabilityEvidencePrinter(Set<VulnerabilityEvidence> evidence) {
List<VulnerabilityEvidence> sortedEvidence = new LinkedList<>(evidence);
Collections.sort(sortedEvidence, new Comparator<VulnerabilityEvidence>() {
@Override
public int compare(VulnerabilityEvidence o1, VulnerabilityEvidence o2) {
if (o1.getLineNumber() < o2.getLineNumber()) {
return -1;
} else if (o1.getLineNumber() > o2.getLineNumber()) {
return 1;
} else {
return 0;
}
}
});

String path = "";
String container = "";
for (VulnerabilityEvidence e : sortedEvidence) {
if (!path.equals(e.getPath())) {
path = e.getPath();
builder.append("<" + path + ">\n");
}
if (!container.equals(e.getContainer())) {
container = e.getContainer();
builder.append("\t" + container + "\n");
}
builder.append("\t\t" + e.getLineNumber() + ": " + e.getLineContents() + "\n");
}
}

@Override
public String toString() {
return this.builder.toString();
}

}

+ 45
- 0
molerat/src/main/java/it/unitn/molerat/evidence/ChangeEvidence.java View File

@@ -0,0 +1,45 @@
package it.unitn.molerat.evidence;

public class ChangeEvidence extends GenericEvidence {

private final boolean removed;

public ChangeEvidence(String file, String commit, String container, boolean removed) {
super(file, commit, container);
this.removed = removed;
}

public boolean isMethodOrConstructor() {
return this.container.contains("(") && this.container.contains(")");
}

public boolean isAbsentInFix() {
return removed;
}

public String getAccessModifier() {
if (this.container.startsWith("public")) {
return "public";
}
else if (this.container.startsWith("protected")) {
return "protected";
}
else if (this.container.startsWith("private")) {
return "private";
}
return "";
}

public boolean isPublicMethodOrConstructor() {
return this.isMethodOrConstructor() && this.getAccessModifier().equals("public");
}

public boolean isProtectedMethodOrConstructor() {
return this.isMethodOrConstructor() && this.getAccessModifier().equals("protected");
}

public boolean isPrivateMethodOrConstructor() {
return this.isMethodOrConstructor() && this.getAccessModifier().equals("private");
}

}

+ 93
- 0
molerat/src/main/java/it/unitn/molerat/evidence/Changes.java View File

@@ -0,0 +1,93 @@
package it.unitn.molerat.evidence;

import java.util.Map;
import java.util.TreeMap;

public class Changes {

private final String path;
private final String leftRev;
private final String rightRev;
private final Map<Integer, String> deletions;
private final Map<Integer, String> additions;
private String renamedTo = "";


public Changes(String path, String leftRev, String rightRev) {
this.path = path;
this.leftRev = leftRev;
this.rightRev = rightRev;
this.deletions = new TreeMap<>();
this.additions = new TreeMap<>();
}

public Changes(String path, String renamedTo, String leftRev, String rightRev) {
this(path, leftRev, rightRev);
this.renamedTo = renamedTo;
}

public Changes(Changes changes, String leftRev, String rightRev) {
this.path = changes.path;
this.leftRev = leftRev;
this.rightRev = rightRev;
this.deletions = changes.deletions;
this.additions = changes.additions;
}

public String getRenamedTo() {
return renamedTo;
}

public boolean wasRenamed() {
return !renamedTo.equals("");
}

public String getPath() {
return this.path;
}

public String getLeftRevision() {
return this.leftRev;
}

public String getRightRevision() {
return this.rightRev;
}

public void putDeletedLine(int lineNumber, String line) {
this.deletions.put(lineNumber, line);
}

public void putAddedLine(int lineNumber, String line) {
this.additions.put(lineNumber, line);
}

public Map<Integer, String> getDeletions() {
return this.deletions;
}

public Map<Integer, String> getAdditions() {
return this.additions;
}

@Override
public String toString() {
StringBuilder builder = new StringBuilder();
builder.append("CHANGES:\n");
for (Map.Entry<Integer, String> entry : this.deletions.entrySet()) {
builder.append("-, " + this.formatAsCSV(entry) + "\n");
}
for (Map.Entry<Integer, String> entry : this.additions.entrySet()) {
builder.append("+, " + this.formatAsCSV(entry) + "\n");
}
return builder.toString();
}

private String formatAsCSV(Map.Entry<Integer, String> entry) {
return this.leftRev + ", " +
this.path + ", " +
entry.getKey() + ", " +
entry.getValue();
}

}

+ 30
- 0
molerat/src/main/java/it/unitn/molerat/evidence/GenericEvidence.java View File

@@ -0,0 +1,30 @@
package it.unitn.molerat.evidence;

public class GenericEvidence {

protected final String commit;
protected final String container;
protected String path;

protected GenericEvidence(String file, String commit, String container) {
this.path = file;
this.commit = commit;
this.container = container;
}

public String getContainer() {
return this.container;
}

public String getCommit() {
return this.commit;
}

public String getPath() {
return this.path;
}

public void setPath(String path) {
this.path = path;
}
}

+ 32
- 0
molerat/src/main/java/it/unitn/molerat/evidence/VulnerabilityEvidence.java View File

@@ -0,0 +1,32 @@
package it.unitn.molerat.evidence;

public class VulnerabilityEvidence extends GenericEvidence {

protected final int lineNumber;
protected final String lineContents;

public VulnerabilityEvidence(String file, String commit, String container, int lineNumber, String lineContents) {
super(file, commit, container);
this.lineNumber = lineNumber;
this.lineContents = lineContents
.trim()
.replace("\t","");
}

public int getLineNumber() {
return this.lineNumber;
}

public String getLineContents() {
return this.lineContents;
}

@Override
public String toString() {
StringBuilder builder = new StringBuilder();
builder.append("<" + commit + ": " + path + ">\n");
builder.append("\t" + container + "\n");
builder.append("\t\t" + lineNumber + ": " + lineContents + "\n\t\n");
return builder.toString();
}
}

+ 56
- 0
molerat/src/main/java/it/unitn/molerat/repos/trackers/AbstractEvidenceTracker.java View File

@@ -0,0 +1,56 @@
package it.unitn.molerat.repos.trackers;

import it.unitn.molerat.evidence.Changes;
import it.unitn.molerat.repos.wrappers.RepoWrapper;
import java.util.*;

public abstract class AbstractEvidenceTracker {
protected final String fixedRevision;
protected final String vulnRevision;
protected final String repoPath;
protected final RepoWrapper repoWrapper;
protected Set<Changes> changes;
protected Set<String> commits = new LinkedHashSet<>();
protected Set<String> restOfCommits = new LinkedHashSet<>();

public AbstractEvidenceTracker(RepoWrapper wrapper, String fixedRev) throws Exception {
this.fixedRevision = fixedRev;
this.repoPath = wrapper.getBasePath();
this.repoWrapper = wrapper;
Iterator<String> commitsIter = repoWrapper.getRevisionNumbers(fixedRev).iterator();
this.vulnRevision = commitsIter.next();
while (commitsIter.hasNext()) {
restOfCommits.add(commitsIter.next());
}
String initDiff = repoWrapper.doDiff(vulnRevision, fixedRevision);
this.changes = repoWrapper.inferChangesFromDiff(initDiff, vulnRevision, fixedRevision);
}

public RepoWrapper getRepoWrapper() {
return this.repoWrapper;
}

public String getFixedRevision() {
return this.fixedRevision;
}

public Set<String> getProcessedCommits() {
return this.commits;
}

// apart from non-java files, filter the files that contain "test" in ther name as well
protected Set<Changes> filterNonJavaChanges(Set<Changes> changes) {
Set<Changes> filteredChanges = new HashSet<>();
for (Changes change : changes) {
if (!change.getPath().endsWith(".java") || change.getPath().toLowerCase().contains("test")) {
filteredChanges.add(change);
}
this.repoWrapper.filterCommentsAndBlanks(change.getDeletions());
this.repoWrapper.filterCommentsAndBlanks(change.getAdditions());
}
changes.removeAll(filteredChanges);
return changes;
}

public abstract void trackEvidence() throws Exception;
}

+ 108
- 0
molerat/src/main/java/it/unitn/molerat/repos/trackers/changes/ChangeEvidenceTracker.java View File

@@ -0,0 +1,108 @@
package it.unitn.molerat.repos.trackers.changes;

import it.unitn.molerat.evidence.ChangeEvidence;
import it.unitn.molerat.repos.trackers.AbstractEvidenceTracker;
import it.unitn.molerat.repos.utils.CommitMetrics;
import it.unitn.molerat.repos.utils.SignatureExtractor;
import it.unitn.molerat.repos.wrappers.RepoWrapper;

import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;

public class ChangeEvidenceTracker extends AbstractEvidenceTracker {

protected final Map<String, Set<ChangeEvidence>> evidences = new TreeMap<>();
protected Map<String, Set<String>> fixConstructs;

protected ChangeEvidenceTracker(RepoWrapper wrapper, String fixedRev) throws Exception {
super(wrapper, fixedRev);
}

public ChangeEvidenceTracker(RepoWrapper repoWrapper, String fixedCommit, Set<String> commits) throws Exception {
this(repoWrapper, fixedCommit);
this.commits = commits;
this.fixConstructs = extractSignatures(this.fixedRevision);
}

protected Map<String, Set<String>> extractSignatures(String commit) throws Exception {
Map<String, Set<String>> result = new TreeMap<>();
Map<String, String> files = CommitMetrics.getFileContentsPerRevision(commit, this.repoWrapper);
for (Map.Entry<String, String> file : files.entrySet()) {
SignatureExtractor se = new SignatureExtractor(file.getValue());
Set<String> signatures = new HashSet<>();
for (String signature : se.getSignaturesWithLines().keySet()) {
signatures.add(signature);
}
result.put(file.getKey(), signatures);
}
return result;
}


@Override
public void trackEvidence() throws Exception {
for (String commit : this.commits) {
// long timeNow = System.currentTimeMillis();
Map<String, Set<String>> currentSignatures = extractSignatures(commit);
for (Map.Entry<String, Set<String>> currentSignature : currentSignatures.entrySet()) {
Set<String> temp = this.fixConstructs.get(currentSignature.getKey());
if (temp == null) {
for (String sign : currentSignature.getValue()) {
ChangeEvidence evidence = new ChangeEvidence(
currentSignature.getKey(),
commit,
sign,
true
);
if (evidence.isPublicMethodOrConstructor()) {
addEvidence(evidence);
}
}
}
else {
for (String sign : currentSignature.getValue()) {
boolean removed = true;
for (String fixSign : temp) {
if (sign.equals(fixSign)) {
removed = false;
break;
}
}
ChangeEvidence evidence = new ChangeEvidence(
currentSignature.getKey(),
commit,
sign,
removed
);
if (evidence.isPublicMethodOrConstructor()) {
addEvidence(evidence);
}
}
}

}
}
}

protected void addEvidence(ChangeEvidence evidence) {
if (this.evidences.containsKey(evidence.getCommit())) {
this.evidences.get(evidence.getCommit()).add(evidence);
}
else {
Set<ChangeEvidence> set = new HashSet<>();
set.add(evidence);
this.evidences.put(evidence.getCommit(), set);
}
}

public Set<ChangeEvidence> getEvidence(String commit) {
Set<ChangeEvidence> evd = this.evidences.get(commit);
return (evd != null) ? evd : new HashSet<>();
}

public Map<String, Set<ChangeEvidence>> getEvidences() {
return this.evidences;
}
}

+ 13
- 0
molerat/src/main/java/it/unitn/molerat/repos/trackers/changes/FullChangeEvidenceTracker.java View File

@@ -0,0 +1,13 @@
package it.unitn.molerat.repos.trackers.changes;


import it.unitn.molerat.repos.wrappers.RepoWrapper;

public class FullChangeEvidenceTracker extends ChangeEvidenceTracker {

public FullChangeEvidenceTracker(RepoWrapper wrapper, String fixedRev) throws Exception {
super(wrapper, fixedRev);
this.commits = wrapper.getRevisionNumbers(fixedRev);
this.fixConstructs = extractSignatures(this.fixedRevision);
}
}

+ 92
- 0
molerat/src/main/java/it/unitn/molerat/repos/trackers/vuln/DeletionVulnerabilityEvidenceTracker.java View File

@@ -0,0 +1,92 @@
package it.unitn.molerat.repos.trackers.vuln;

import it.unitn.molerat.evidence.Changes;
import it.unitn.molerat.evidence.VulnerabilityEvidence;
import it.unitn.molerat.repos.wrappers.RepoWrapper;

import java.util.HashSet;
import java.util.Map;
import java.util.Set;

public class DeletionVulnerabilityEvidenceTracker extends VulnerabilityEvidenceTracker {

public DeletionVulnerabilityEvidenceTracker(RepoWrapper wrapper, String fixedRev) throws Exception {
super(wrapper, fixedRev);
}

@Override
protected Set<VulnerabilityEvidence> getInitialVulnerabilityEvidence(Changes changes) throws Exception {
return this.recordVulnerabilityEvidence(changes.getDeletions(), changes);
}

@Override
protected Set<VulnerabilityEvidence> getVulnerabilityEvidence(String currentEvidenceCommit, String previousEvidenceCommit, Set<Changes> changes) throws Exception {
Set<VulnerabilityEvidence> newEvidences = new HashSet<>();
Set<VulnerabilityEvidence> previousEvidences = getEvidences(previousEvidenceCommit);

Set<VulnerabilityEvidence> changedEvidence = new HashSet<>();
Set<Changes> changesToProcess = new HashSet<>();
Set<VulnerabilityEvidence> stillEvidence = new HashSet<>();

// filter non-Java files
changes = filterNonJavaChanges(changes);

if (previousEvidences == null) {
return newEvidences;
}

for (VulnerabilityEvidence previousEvidence : previousEvidences) {
for (Changes change : changes ) {
// The file has been just renamed
if (change.wasRenamed()) {
if (change.getRenamedTo().equals(previousEvidence.getPath())) {
previousEvidence.setPath(change.getPath());
}
}
// The file has been changed
else if (previousEvidence.getPath().equals(change.getPath())) {
changedEvidence.add(previousEvidence);
changesToProcess.add(change);
}
}
}

stillEvidence.addAll(previousEvidences);
stillEvidence.removeAll(changedEvidence);

// "Refresh" the molerat.evidence when a file was not changed
for (VulnerabilityEvidence e : stillEvidence) {
VulnerabilityEvidence newEvidence = new VulnerabilityEvidence(
e.getPath(),
currentEvidenceCommit,
e.getContainer(),
e.getLineNumber(),
e.getLineContents()
);
newEvidences.add(newEvidence);
}

////////////////////////////////////////////////////////////////////////////////////////////////////////////////
// "Triage" the molerat.evidence when a file was changed
for (Changes change : changesToProcess) {
Set<VulnerabilityEvidence> retain = new HashSet<>();
for (VulnerabilityEvidence e : changedEvidence) {
if (e.getPath().equals(change.getPath())) {
retain.add(e);
}
}
Map<Integer, String> linesToKeep = this.updateChangedLines(
change.getPath(),
change.getLeftRevision(),
change.getRightRevision(),
retain,
change
);
if (linesToKeep != null) {
newEvidences.addAll(this.recordVulnerabilityEvidence(linesToKeep, change));
}
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////
return newEvidences;
}
}

+ 72
- 0
molerat/src/main/java/it/unitn/molerat/repos/trackers/vuln/EnhancedDeletionVulnerabilityEvidenceTracker.java View File

@@ -0,0 +1,72 @@
package it.unitn.molerat.repos.trackers.vuln;

import it.unitn.molerat.evidence.Changes;
import it.unitn.molerat.evidence.VulnerabilityEvidence;
import it.unitn.molerat.repos.utils.SignatureExtractor;
import it.unitn.molerat.repos.wrappers.RepoWrapper;

import java.util.HashSet;
import java.util.Map;
import java.util.Set;

public class EnhancedDeletionVulnerabilityEvidenceTracker extends DeletionVulnerabilityEvidenceTracker{

public EnhancedDeletionVulnerabilityEvidenceTracker(RepoWrapper wrapper, String fixedRev) throws Exception {
super(wrapper, fixedRev);
}


@Override
protected Set<VulnerabilityEvidence> getInitialVulnerabilityEvidence(Changes changes) throws Exception {
if (changes.getDeletions().size() != 0) {
return super.getInitialVulnerabilityEvidence(changes);
}
else {
SignatureExtractor se = new SignatureExtractor(repoWrapper.doCat(changes.getPath(), changes.getRightRevision()));
Map<String, Set<Integer>> rightSignatures = se.getSignaturesWithLines();

Set<String> relevantSignatures = new HashSet<>();

for (int line : changes.getAdditions().keySet()) {
for (Map.Entry<String, Set<Integer>> entry : rightSignatures.entrySet()) {
if (entry.getValue().contains(line)) {
relevantSignatures.add(entry.getKey());
}
}
}

String leftFile = repoWrapper.doCat(changes.getPath(), changes.getLeftRevision());
Map<Integer, String> lineMappings = repoWrapper.getLineMappings(leftFile);
se = new SignatureExtractor(leftFile);
Map<String, Set<Integer>> leftSignatures = se.getSignaturesWithLines();

for (int line : changes.getDeletions().keySet()) {
for (Map.Entry<String, Set<Integer>> entry : leftSignatures.entrySet()) {
if (entry.getValue().contains(line)) {
relevantSignatures.add(entry.getKey());
}
}
}

Set<VulnerabilityEvidence> initialEvidence = new HashSet<>();
for (String signature : relevantSignatures) {
for (Map.Entry<String, Set<Integer>> entry : leftSignatures.entrySet()) {
if (signature.equals(entry.getKey())) {
for (int line : entry.getValue()) {
VulnerabilityEvidence evd = new VulnerabilityEvidence(
changes.getPath(),
changes.getLeftRevision(),
signature,
line,
lineMappings.get(line)
);
initialEvidence.add(evd);
}
}
}
}

return initialEvidence;
}
}
}

+ 71
- 0
molerat/src/main/java/it/unitn/molerat/repos/trackers/vuln/FixStatisticsVulnerabilityEvidenceTracker.java View File

@@ -0,0 +1,71 @@
package it.unitn.molerat.repos.trackers.vuln;

import it.unitn.molerat.evidence.Changes;
import it.unitn.molerat.evidence.VulnerabilityEvidence;
import it.unitn.molerat.repos.wrappers.RepoWrapper;

import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;

public class FixStatisticsVulnerabilityEvidenceTracker extends VulnerabilityEvidenceTracker {
private int addedLines = 0;
private int deletedLines = 0;
private int touchedFiles = 0;
private int touchedMethods = 0;

public FixStatisticsVulnerabilityEvidenceTracker(RepoWrapper wrapper, String fixedRev) throws Exception {
super(wrapper, fixedRev);
}

@Override
protected Set<VulnerabilityEvidence> getInitialVulnerabilityEvidence(Changes changes) throws Exception {
return null;
}

@Override
protected Set<VulnerabilityEvidence> getVulnerabilityEvidence(String currentEvidenceCommit, String previousEvidenceCommit, Set<Changes> changes) throws Exception {
return null;
}

@Override
public void trackEvidence() throws Exception {
this.changes = filterNonJavaChanges(this.changes);
Iterator<Changes> it = changes.iterator();
while (it.hasNext()) {
Changes currentChanges = it.next();
touchedFiles++;
addedLines += currentChanges.getAdditions().size();
deletedLines += currentChanges.getDeletions().size();

Set<String> methods = new HashSet<>();
Set<VulnerabilityEvidence> evidences = recordVulnerabilityEvidenceForRightFile(currentChanges.getAdditions(), currentChanges);
evidences.addAll(recordVulnerabilityEvidence(currentChanges.getDeletions(), currentChanges));
for (VulnerabilityEvidence evd : evidences) {
if (!evd.getLineContents().contains("private") &&
!evd.getLineContents().contains("protected") &&
!evd.getLineContents().contains("public")) {
methods.add(evd.getContainer());
}
}
touchedMethods += methods.size();
commits.add(currentChanges.getLeftRevision());
}
}

public int getNumberOfAddedLines() {
return addedLines;
}

public int getNumberOfDeletedLines() {
return deletedLines;
}

public int getNumberOfTouchedMethods() {
return touchedMethods;
}

public int getNumberOfTouchedFiles() {
return touchedFiles;
}
}

+ 148
- 0
molerat/src/main/java/it/unitn/molerat/repos/trackers/vuln/PatchedMethodBodyTracker.java View File

@@ -0,0 +1,148 @@
package it.unitn.molerat.repos.trackers.vuln;

import it.unitn.molerat.evidence.Changes;
import it.unitn.molerat.evidence.VulnerabilityEvidence;
import it.unitn.molerat.repos.utils.SignatureExtractor;
import it.unitn.molerat.repos.wrappers.RepoWrapper;

import java.util.HashSet;
import java.util.Map;
import java.util.Set;

public class PatchedMethodBodyTracker extends VulnerabilityEvidenceTracker {

public PatchedMethodBodyTracker(RepoWrapper wrapper, String fixedRev) throws Exception {
super(wrapper, fixedRev);
}

@Override
protected Set<VulnerabilityEvidence> getInitialVulnerabilityEvidence(Changes changes) throws Exception {
SignatureExtractor se = new SignatureExtractor(repoWrapper.doCat(changes.getPath(), changes.getRightRevision()));
Map<String, Set<Integer>> rightSignatures = se.getSignaturesWithLines();

Set<String> relevantSignatures = new HashSet<>();

for (int line : changes.getAdditions().keySet()) {
for (Map.Entry<String, Set<Integer>> entry : rightSignatures.entrySet()) {
if (entry.getValue().contains(line)) {
relevantSignatures.add(entry.getKey());
}
}
}

String leftFile = repoWrapper.doCat(changes.getPath(), changes.getLeftRevision());
Map<Integer, String> lineMappings = repoWrapper.getLineMappings(leftFile);
se = new SignatureExtractor(leftFile);
Map<String, Set<Integer>> leftSignatures = se.getSignaturesWithLines();

for (int line : changes.getDeletions().keySet()) {
for (Map.Entry<String, Set<Integer>> entry : leftSignatures.entrySet()) {
if (entry.getValue().contains(line)) {
relevantSignatures.add(entry.getKey());
}
}
}

Set<VulnerabilityEvidence> initialEvidence = new HashSet<>();
for (String signature : relevantSignatures) {
for (Map.Entry<String, Set<Integer>> entry : leftSignatures.entrySet()) {
if (signature.equals(entry.getKey())) {
for (int line : entry.getValue()) {
VulnerabilityEvidence evd = new VulnerabilityEvidence(
changes.getPath(),
changes.getLeftRevision(),
signature,
line,
lineMappings.get(line)
);
initialEvidence.add(evd);
}
}
}
}

return initialEvidence;
}


@Override
protected Set<VulnerabilityEvidence> getVulnerabilityEvidence(String currentEvidenceCommit, String previousEvidenceCommit, Set<Changes> changes) throws Exception {
Set<VulnerabilityEvidence> newEvidences = new HashSet<>();
Set<VulnerabilityEvidence> previousEvidences = getEvidences(previousEvidenceCommit);

Set<VulnerabilityEvidence> changedEvidence = new HashSet<>();
Set<Changes> changesToProcess = new HashSet<>();
Set<VulnerabilityEvidence> stillEvidence = new HashSet<>();

// filter non-Java files
changes = filterNonJavaChanges(changes);

if (previousEvidences == null) {
return newEvidences;
}

for (VulnerabilityEvidence previousEvidence : previousEvidences) {
for (Changes change : changes ) {
// The file has been just renamed
if (change.wasRenamed()) {
if (change.getRenamedTo().equals(previousEvidence.getPath())) {
previousEvidence.setPath(change.getPath());
}
}
// The file has been changed
else if (previousEvidence.getPath().equals(change.getPath())) {
changedEvidence.add(previousEvidence);
changesToProcess.add(change);
}
}
}

stillEvidence.addAll(previousEvidences);
stillEvidence.removeAll(changedEvidence);

// "Refresh" the molerat.evidence when a file was not changed
for (VulnerabilityEvidence e : stillEvidence) {
VulnerabilityEvidence newEvidence = new VulnerabilityEvidence(
e.getPath(),
currentEvidenceCommit,
e.getContainer(),
e.getLineNumber(),
e.getLineContents()
);
newEvidences.add(newEvidence);
}

// "Triage" the molerat.evidence when a file was changed
for (Changes change : changesToProcess) {
Set<String> relevantSignatures = new HashSet<>();
for (VulnerabilityEvidence e : changedEvidence) {
relevantSignatures.add(e.getContainer());
}

String leftFile = repoWrapper.doCat(change.getPath(), change.getLeftRevision());
Map<Integer, String> lineMappings = repoWrapper.getLineMappings(leftFile);
SignatureExtractor se = new SignatureExtractor(leftFile);
Map<String, Set<Integer>> leftSignatures = se.getSignaturesWithLines();

for (String signature : relevantSignatures) {
for (Map.Entry<String, Set<Integer>> entry : leftSignatures.entrySet()) {
if (signature.equals(entry.getKey())) {
for (int line : entry.getValue()) {
VulnerabilityEvidence evd = new VulnerabilityEvidence(
change.getPath(),
change.getLeftRevision(),
signature,
line,
lineMappings.get(line)
);
newEvidences.add(evd);
}
}
}
}
}
return newEvidences;
}


}

+ 85
- 0
molerat/src/main/java/it/unitn/molerat/repos/trackers/vuln/SliceDecayVulnerabilityEvidenceTracker.java View File

@@ -0,0 +1,85 @@
package it.unitn.molerat.repos.trackers.vuln;

import it.unitn.molerat.evidence.Changes;
import it.unitn.molerat.evidence.VulnerabilityEvidence;
import it.unitn.molerat.repos.wrappers.RepoWrapper;

import java.util.HashSet;
import java.util.Map;
import java.util.Set;

public class SliceDecayVulnerabilityEvidenceTracker extends SliceVulnerabilityEvidenceTracker {

public SliceDecayVulnerabilityEvidenceTracker(RepoWrapper wrapper, String fixedRev) throws Exception {
super(wrapper, fixedRev);
}

@Override
protected Set<VulnerabilityEvidence> getVulnerabilityEvidence(String currentEvidenceCommit, String previousEvidenceCommit, Set<Changes> changes) throws Exception {
Set<VulnerabilityEvidence> newEvidences = new HashSet<>();
Set<VulnerabilityEvidence> previousEvidences = getEvidences(previousEvidenceCommit);
Set<VulnerabilityEvidence> changedEvidence = new HashSet<>();

Set<Changes> changesToProcess = new HashSet<>();
Set<VulnerabilityEvidence> stillEvidence = new HashSet<>();

// filter non-Java files
changes = filterNonJavaChanges(changes);

if (previousEvidences == null) {
return newEvidences;
}

for (VulnerabilityEvidence previousEvidence : previousEvidences) {
for (Changes change : changes ) {
// The file has been just renamed
if (change.wasRenamed()) {
if (change.getRenamedTo().equals(previousEvidence.getPath())) {
previousEvidence.setPath(change.getPath());
}
}
// The file has been changed
else if (previousEvidence.getPath().equals(change.getPath())) {
changedEvidence.add(previousEvidence);
changesToProcess.add(change);
}
}
}

stillEvidence.addAll(previousEvidences);
stillEvidence.removeAll(changedEvidence);

// "Refresh" the molerat.evidence when a file was not changed
for (VulnerabilityEvidence e : stillEvidence) {
VulnerabilityEvidence newEvidence = new VulnerabilityEvidence(
e.getPath(),
currentEvidenceCommit,
e.getContainer(),
e.getLineNumber(),
e.getLineContents()
);
newEvidences.add(newEvidence);
}

// "Triage" the molerat.evidence when a file was changed