diff --git a/.gitignore b/.gitignore index 8f4ed576c..9468b8edb 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ target /bin/ /.project /.classpath -/.metadata \ No newline at end of file +/.metadata +neo4j/ diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..1322566bd --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,16 @@ +version: '3' +services: + neo4j: + container_name: neo4j + image: neo4j:4.3.3 + restart: unless-stopped + ports: + - 7474:7474 + - 7687:7687 + volumes: + - ./neo4j/data:/neo4j-data + - ./neo4j/import:/import + - ./neo4j/plugins:/plugins + user: "${UID}:${GID}" +volumes: + neo4j: diff --git a/microbat/.classpath b/microbat/.classpath index 41b714149..4d3034a68 100644 --- a/microbat/.classpath +++ b/microbat/.classpath @@ -1,25 +1,31 @@ - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/microbat/META-INF/MANIFEST.MF b/microbat/META-INF/MANIFEST.MF index ddd0547fd..b4997a5f0 100644 --- a/microbat/META-INF/MANIFEST.MF +++ b/microbat/META-INF/MANIFEST.MF @@ -39,6 +39,8 @@ Bundle-ClassPath: ., lib/asm-util-6.0.jar, lib/mysql-connector-java-5.1.44-bin.jar, lib/sqlite-jdbc-3.32.3.2.jar, + lib/neo4j-java-driver-4.3.4.jar, + lib/neo4j-jdbc-driver-4.0.2.jar, lib/instrumentator.jar Export-Package: microbat;uses:="org.osgi.framework,org.eclipse.jface.resource,org.eclipse.ui.plugin", microbat.agent, diff --git a/microbat/build.properties b/microbat/build.properties index b25383e94..fab08f089 100644 --- a/microbat/build.properties +++ b/microbat/build.properties @@ -31,7 +31,9 @@ bin.includes = META-INF/,\ ddl/Step.sql,\ ddl/StepVariableRelation.sql,\ ddl/Trace.sql,\ - lib/instrumentator.jar + lib/instrumentator.jar,\ + lib/neo4j-java-driver-4.3.4.jar,\ + lib/neo4j-jdbc-driver-4.0.2.jar src.excludes = src/test/ src.includes = icons/,\ plugin.xml,\ diff --git a/microbat/lib/instrumentator.jar b/microbat/lib/instrumentator.jar index 3ac7f73e2..ff84a6e50 100644 Binary files a/microbat/lib/instrumentator.jar and b/microbat/lib/instrumentator.jar differ diff --git a/microbat/lib/neo4j-java-driver-4.3.4.jar b/microbat/lib/neo4j-java-driver-4.3.4.jar new file mode 100644 index 000000000..4a440172d Binary files /dev/null and b/microbat/lib/neo4j-java-driver-4.3.4.jar differ diff --git a/microbat/lib/neo4j-jdbc-driver-4.0.2.jar b/microbat/lib/neo4j-jdbc-driver-4.0.2.jar new file mode 100644 index 000000000..0a974b4bb Binary files /dev/null and b/microbat/lib/neo4j-jdbc-driver-4.0.2.jar differ diff --git a/microbat/src/main/microbat/agent/TraceAgentRunner.java b/microbat/src/main/microbat/agent/TraceAgentRunner.java index ffaec254d..736c0044a 100644 --- a/microbat/src/main/microbat/agent/TraceAgentRunner.java +++ b/microbat/src/main/microbat/agent/TraceAgentRunner.java @@ -16,6 +16,7 @@ import microbat.instrumentation.precheck.PrecheckInfo; import microbat.model.trace.Trace; import microbat.preference.DatabasePreference; +import microbat.sql.DBSettings; import microbat.trace.Reader; import sav.common.core.SavException; import sav.common.core.SavRtException; @@ -39,7 +40,7 @@ public class TraceAgentRunner extends AgentVmRunner { private String testFailureMessage; private VMConfiguration config; private boolean enableSettingHeapSize = true; - + private List traces; public TraceAgentRunner(String agentJar, VMConfiguration vmConfig) { @@ -109,9 +110,11 @@ public boolean run(Reader reader) throws SavException { dumpFile = DatabasePreference.getDBFile(); break; } - addAgentParam(AgentParams.OPT_TRACE_RECORDER, reader.name()); // why is reader name used for recorder option? + addAgentParam(AgentParams.OPT_TRACE_RECORDER, reader.name()); // why is reader name used for recorder + // option? addAgentParam(AgentParams.OPT_RUN_ID, runId); addAgentParam(AgentParams.OPT_DUMP_FILE, String.valueOf(dumpFile.getPath())); + addAgentParam(AgentParams.OPT_IS_INC_STORAGE, DBSettings.INC_STORAGE); super.startAndWaitUntilStop(getConfig()); // Trace recording System.out.println("|"); timer.newPoint("Read output result"); @@ -127,8 +130,7 @@ public boolean run(Reader reader) throws SavException { System.out.println(timer.getResultString()); return true; } - - + // public boolean run01(Reader reader) throws SavException { // isPrecheckMode = false; // StopTimer timer = new StopTimer("Building trace"); @@ -212,9 +214,7 @@ public boolean runWithSocket() throws SavException { String msg = reader.readString(); updateTestResult(msg); List traces = reader.readTrace(); - int collected = traces.stream() - .mapToInt(trace -> trace.size()) - .sum(); + int collected = traces.stream().mapToInt(trace -> trace.size()).sum(); runningInfo = new RunningInfo(msg, traces, precheckInfo.getStepTotal(), collected); } catch (Exception e) { e.printStackTrace(); @@ -241,11 +241,11 @@ protected void printOut(String line, boolean error) { }; private void printProgress(int size, int stepNum) { - - if(stepNum == 0) { + + if (stepNum == 0) { return; } - + double progress = ((double) size) / stepNum; double preProgr = 0; diff --git a/microbat/src/main/microbat/codeanalysis/runtime/InstrumentationExecutor.java b/microbat/src/main/microbat/codeanalysis/runtime/InstrumentationExecutor.java index a1e19e322..91cc75ef7 100644 --- a/microbat/src/main/microbat/codeanalysis/runtime/InstrumentationExecutor.java +++ b/microbat/src/main/microbat/codeanalysis/runtime/InstrumentationExecutor.java @@ -9,6 +9,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import org.apache.commons.io.FileUtils; import org.eclipse.jdt.core.dom.ASTNode; @@ -198,7 +199,7 @@ public RunningInfo execute(PreCheckInformation info) { RunningInfo result = agentRunner.getRunningInfo(); // System.out.println(result); System.out.println("isExpectedStepsMet? " + result.isExpectedStepsMet()); - System.out.println("trace length: " + result.getMainTrace() == null ? "0" : result.getMainTrace().size()); + System.out.println("trace length: " + Optional.ofNullable(result.getMainTrace()).map(t -> t.size()).orElse(0)); System.out.println("isTestSuccessful? " + agentRunner.isTestSuccessful()); System.out.println("testFailureMessage: " + agentRunner.getTestFailureMessage()); System.out.println("finish!"); diff --git a/microbat/src/main/microbat/codeanalysis/runtime/ProgramExecutor.java b/microbat/src/main/microbat/codeanalysis/runtime/ProgramExecutor.java index 64150d8c6..66d460ecc 100644 --- a/microbat/src/main/microbat/codeanalysis/runtime/ProgramExecutor.java +++ b/microbat/src/main/microbat/codeanalysis/runtime/ProgramExecutor.java @@ -173,7 +173,7 @@ public ProgramExecutor() { */ public void run(List runningStatements, List executionOrderList, IProgressMonitor monitor, int stepNum, boolean isTestcaseEvaluation) throws SavException, TimeoutException { - this.trace = new Trace(appPath); + this.trace = new Trace(appPath, ""); List classScope = parseScope(runningStatements); List lvsList = parseLocalVariables(classScope, this.appPath); diff --git a/microbat/src/main/microbat/model/trace/Trace.java b/microbat/src/main/microbat/model/trace/Trace.java index 81e699686..567406c2c 100644 --- a/microbat/src/main/microbat/model/trace/Trace.java +++ b/microbat/src/main/microbat/model/trace/Trace.java @@ -35,7 +35,7 @@ public class Trace { private List excludedLibraryClasses = new ArrayList<>(); private boolean isMain; private String threadName; - private String id; + private String traceId; /** * This variable is to trace whether the variables in different lines are the same @@ -43,12 +43,9 @@ public class Trace { */ private LocalVariableScopes localVariableScopes = new LocalVariableScopes(); - public Trace(AppJavaClassPath appJavaClassPath) { + public Trace(AppJavaClassPath appJavaClassPath, String traceId) { this.setAppJavaClassPath(appJavaClassPath); - } - - public Trace(String id) { - this.id = id; + this.traceId = traceId; } /** @@ -153,7 +150,7 @@ public void setObservingIndex(int observingIndex) { } public String getId() { - return this.id; + return traceId; } public int searchBackwardTraceNode(String expression){ diff --git a/microbat/src/main/microbat/preference/DatabasePreference.java b/microbat/src/main/microbat/preference/DatabasePreference.java index 58e9bf170..1e4c598df 100644 --- a/microbat/src/main/microbat/preference/DatabasePreference.java +++ b/microbat/src/main/microbat/preference/DatabasePreference.java @@ -30,12 +30,14 @@ public class DatabasePreference extends PreferencePage implements IWorkbenchPreferencePage { private static final String ID = "microbat.preference.database"; + private static final Reader[] READERS = new Reader[] { Reader.SQLITE3, Reader.MYSQL, Reader.NEO4J }; public static final String HOST = "dbHost"; public static final String PORT = "dbPort"; public static final String DATABASE = "dbName"; public static final String USER_NAME = "dbUserName"; public static final String PASSWORD = "dbPassword"; public static final String IS_STARTDB = "startdb"; + public static final String IS_INC_STORAGE= "incrementalStorage"; public static final String DBMS = "dbms"; public static final String DBPATH = "dbPath`"; @@ -46,6 +48,7 @@ public class DatabasePreference extends PreferencePage implements IWorkbenchPref private StringFieldEditor passwordField; private Combo dataBaseDropDown; private Button startWithSQL; + private Button incrementalStorage; private DirectoryFieldEditor sqliteDBPath; @Override @@ -61,9 +64,11 @@ protected Control createContents(Composite parent) { contents.setLayout(layout); contents.setLayoutData(new GridData(GridData.FILL_BOTH)); startWithSQL = SWTFactory.createCheckbox(contents, "Start with SQL", 1); + incrementalStorage = SWTFactory.createCheckbox(contents, "Turn on incremental storage", 1); dataBaseDropDown = SWTFactory.creatDropdown(contents); dataBaseDropDown.add(Reader.SQLITE3.toString()); dataBaseDropDown.add(Reader.MYSQL.toString()); + dataBaseDropDown.add(Reader.NEO4J.toString()); dataBaseDropDown.select(0); SWTFactory.createLabel(contents, "Database Configuration:", 2); @@ -85,7 +90,8 @@ protected Control createContents(Composite parent) { public static Reader getReader() { IPreferenceStore pref = Activator.getDefault().getPreferenceStore(); if (pref.getBoolean(IS_STARTDB)) { - return pref.getInt(DBMS) == 1 ? Reader.MYSQL : Reader.SQLITE3; +// return pref.getInt(DBMS) == 1 ? Reader.MYSQL : Reader.SQLITE3; + return READERS[pref.getInt(DBMS)]; } else { return Reader.FILE; } @@ -93,13 +99,13 @@ public static Reader getReader() { public static File getDBFile() { String filePath = Activator.getDefault().getPreferenceStore().getString(DBPATH); - + if (filePath.trim().equals("") || filePath == null) { System.err.println("need to specify the db file location"); } else { return new File(filePath); } - + return null; } @@ -111,6 +117,7 @@ private void setDefaultValue() { userNameField.setStringValue(pref.getString(USER_NAME)); passwordField.setStringValue(pref.getString(PASSWORD)); startWithSQL.setSelection(pref.getBoolean(IS_STARTDB)); + incrementalStorage.setSelection(pref.getBoolean(IS_INC_STORAGE)); dataBaseDropDown.select(pref.getInt(DBMS)); sqliteDBPath.setStringValue(pref.getString(DBPATH)); } @@ -131,6 +138,7 @@ public boolean performOk() { preferences.put(USER_NAME, userNameField.getStringValue()); preferences.put(PASSWORD, passwordField.getStringValue()); preferences.putBoolean(IS_STARTDB, startWithSQL.getSelection()); + preferences.putBoolean(IS_INC_STORAGE, incrementalStorage.getSelection()); preferences.putInt(DBMS, dataBaseDropDown.getSelectionIndex()); preferences.put(DBPATH, sqliteDBPath.getStringValue()); try { @@ -146,6 +154,7 @@ public boolean performOk() { pref.putValue(USER_NAME, userNameField.getStringValue()); pref.putValue(PASSWORD, passwordField.getStringValue()); pref.putValue(IS_STARTDB, String.valueOf(startWithSQL.getSelection())); + pref.putValue(IS_INC_STORAGE, String.valueOf(incrementalStorage.getSelection())); pref.putValue(DBMS, String.valueOf(dataBaseDropDown.getSelectionIndex())); pref.putValue(DBPATH, String.valueOf(sqliteDBPath.getStringValue())); diff --git a/microbat/src/main/microbat/sql/DBSettings.java b/microbat/src/main/microbat/sql/DBSettings.java index 8b6ccf2ed..2cdb1fd55 100644 --- a/microbat/src/main/microbat/sql/DBSettings.java +++ b/microbat/src/main/microbat/sql/DBSettings.java @@ -9,6 +9,7 @@ public class DBSettings { public static final int SQLITE3_DBMS = 0; public static final int MYSQL_DBMS = 1; + public static final int NEO4J_DBMS = 2; public static String dbAddress = "localhost"; public static int dbPort = 3306; @@ -18,6 +19,7 @@ public class DBSettings { public static String dbPath = "microbat.db"; public static int DMBS_TYPE = SQLITE3_DBMS; public static String USE_DB = "false"; + public static String INC_STORAGE = "false"; public static boolean enableAutoUpdateDb = true; private static int version = -1; //keep track for the update @@ -35,6 +37,7 @@ public static void updateFromPreference() { password = pref.getString(PASSWORD); DMBS_TYPE = pref.getInt(DBMS); USE_DB = pref.getString(IS_STARTDB); + INC_STORAGE = pref.getString(IS_INC_STORAGE); dbPath = pref.getString(DBPATH); version++; } diff --git a/microbat/src/main/microbat/sql/DbService.java b/microbat/src/main/microbat/sql/DbService.java index 50e457fc7..2997d69a6 100644 --- a/microbat/src/main/microbat/sql/DbService.java +++ b/microbat/src/main/microbat/sql/DbService.java @@ -68,6 +68,9 @@ public static Connection getConnection() throws SQLException { case DBSettings.SQLITE3_DBMS: factory = new SqliteConnectionFactory(); break; + case DBSettings.NEO4J_DBMS: +// factory = new SqliteConnectionFactory(); + return null; default: throw new SQLException("we yet support dbms type other than mysql and sqlite3"); } diff --git a/microbat/src/main/microbat/sql/GraphTraceRetriever.java b/microbat/src/main/microbat/sql/GraphTraceRetriever.java new file mode 100644 index 000000000..43d73ba84 --- /dev/null +++ b/microbat/src/main/microbat/sql/GraphTraceRetriever.java @@ -0,0 +1,137 @@ +package microbat.sql; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import org.neo4j.driver.Driver; +import org.neo4j.driver.AuthTokens; +import org.neo4j.driver.GraphDatabase; +import org.neo4j.driver.Record; +import org.neo4j.driver.Result; +import org.neo4j.driver.Session; +import org.neo4j.driver.Transaction; +import static org.neo4j.driver.Values.parameters; + +import microbat.model.BreakPoint; +import microbat.model.trace.Trace; +import microbat.model.trace.TraceNode; +import microbat.model.value.VarValue; +import sav.common.core.Pair; + +public class GraphTraceRetriever implements TraceRetriever { + private final String runId; + private Driver driver; + private static final String CONNECTION_URI = "bolt://localhost"; + private static final String GET_TRACES_QUERY = "MATCH (t:Trace) WHERE t.runId = $runId RETURN t"; + private static final String GET_STEPS_QUERY = "MATCH (s:Step)-[:AT]->(l:Location) WHERE s.traceId = $traceId RETURN s,l ORDER BY s.stepOrder"; + private static final String GET_LOCATIONS_QUERY = "MATCH (l:Location) WHERE l.traceId = $traceId RETURN l"; + private Map controlDominator = new HashMap<>(); + private Map stepIn = new HashMap<>(); + private Map stepOver = new HashMap<>(); + private Map invocationParent = new HashMap<>(); + private Map loopParent = new HashMap<>(); + private Map map = new HashMap<>(); + private Map locationsMap = new HashMap<>(); + + public GraphTraceRetriever(String runId) { + this.runId = runId; + this.driver = GraphDatabase.driver(CONNECTION_URI, AuthTokens.basic("neo4j", "microbat")); + } + + private TraceNode buildStep(final Record rec, final Trace trace) { + int currOrder = rec.values().get(0).get("stepOrder").asInt(); + TraceNode step = new TraceNode(null, null, currOrder, trace); + Optional.of(rec.values().get(0).get("controlDominator")).filter(x -> !x.isNull()).ifPresent(target -> controlDominator.put(currOrder, target.asInt()));; + Optional.of(rec.values().get(0).get("stepIn")).filter(x -> !x.isNull()).ifPresent(target -> stepIn.put(currOrder, target.asInt()));; + Optional.of(rec.values().get(0).get("stepOver")).filter(x -> !x.isNull()).ifPresent(target -> stepOver.put(currOrder, target.asInt()));; + Optional.of(rec.values().get(0).get("invocationParent")).filter(x -> !x.isNull()).ifPresent(target -> invocationParent.put(currOrder, target.asInt()));; + Optional.of(rec.values().get(0).get("loopParent")).filter(x -> !x.isNull()).ifPresent(target -> loopParent.put(currOrder, target.asInt()));; + step.setTimestamp(rec.values().get(0).get("time").asLocalDateTime().toEpochSecond(ZoneOffset.UTC)); + map.put(currOrder, step); + String locationId = rec.values().get(1).get("locationId").asString(); + if (!locationsMap.containsKey(locationId)) { + String className = rec.values().get(1).get("className").asString(); + int lineNumber = rec.values().get(1).get("lineNumber").asInt(); + BreakPoint bp = new BreakPoint(className, className, lineNumber); + bp.setConditional(rec.values().get(1).get("isConditional").asBoolean()); + bp.setReturnStatement(rec.values().get(1).get("isReturn").asBoolean()); + locationsMap.put(locationId, bp); + } + step.setBreakPoint(locationsMap.get(locationId)); + return step; + } + + private List getSteps(final Transaction tx, final String traceId, final Trace trace) { + Result res = tx.run(GET_STEPS_QUERY, parameters("traceId", traceId)); + List nodes = res.list(rec -> buildStep(rec, trace)); + buildNodeRelations(nodes); + return nodes; + } + + private void buildNodeRelations(List nodes) { + for (Map.Entry entry: controlDominator.entrySet()) { + TraceNode a = map.get(entry.getKey()); + TraceNode b = map.get(entry.getValue()); + a.setControlDominator(b); + b.addControlDominatee(a); +// map.get(entry.getKey()).setControlDominator(map.get(entry.getValue())); +// map.get(entry.getValue()).addControlDominatee(map.get(entry.getKey())); + } + for (Map.Entry entry: stepIn.entrySet()) { + map.get(entry.getKey()).setStepInNext(map.get(entry.getValue())); + map.get(entry.getValue()).setStepInPrevious(map.get(entry.getKey())); + } + for (Map.Entry entry: stepOver.entrySet()) { + map.get(entry.getKey()).setStepOverNext(map.get(entry.getValue())); + map.get(entry.getValue()).setStepOverPrevious(map.get(entry.getKey())); + } + for (Map.Entry entry: invocationParent.entrySet()) { + map.get(entry.getKey()).setInvocationParent(map.get(entry.getValue())); + map.get(entry.getValue()).addInvocationChild(map.get(entry.getKey())); + } + for (Map.Entry entry: loopParent.entrySet()) { + map.get(entry.getKey()).setLoopParent(map.get(entry.getValue())); + map.get(entry.getValue()).addLoopChild(map.get(entry.getKey())); + } + } + + @Override + public List getTraces(String runId) { + try (Session session = driver.session()) { + List traces = new ArrayList<>(); + Result res = session.run(GET_TRACES_QUERY, parameters("runId", runId)); + for (Record rec : res.list()) { + String traceId = rec.values().get(0).get("traceId").asString(); + Trace trace = new Trace(null, traceId); + trace.setThreadId(rec.values().get(0).get("threadId").asLong()); + trace.setThreadName(rec.values().get(0).get("threadName").asString()); + trace.setMain(rec.values().get(0).get("isMain").asBoolean()); + List nodes = session.readTransaction(tx -> getSteps(tx, trace.getId(), trace)); + trace.setExecutionList(nodes); + traces.add(trace); + } + return traces; + } + } + + @Override + public Pair, List> loadRWVars(TraceNode step, String traceId) { +// ListreadVars = toVarValue(rs.getString("read_vars")); +// // written_vars +// loadVarStep = "written_vars"; +// ListwriteVars = toVarValue(rs.getString("written_vars")); +// return Pair.of(readVars, writeVars); + return null; + } +} diff --git a/microbat/src/main/microbat/sql/MysqlTraceRetriever.java b/microbat/src/main/microbat/sql/MysqlTraceRetriever.java index 1067090a7..6e65733fa 100644 --- a/microbat/src/main/microbat/sql/MysqlTraceRetriever.java +++ b/microbat/src/main/microbat/sql/MysqlTraceRetriever.java @@ -72,7 +72,7 @@ public Trace retrieveTrace(int traceId) throws SQLException{ } protected Trace loadTrace(int traceId, Connection conn, List closables) throws SQLException { - Trace trace = new Trace(""); + Trace trace = new Trace(null, ""); // load step List steps = loadSteps(traceId, conn, closables, trace); trace.setExecutionList(steps); diff --git a/microbat/src/main/microbat/sql/TraceRetrieverImpl.java b/microbat/src/main/microbat/sql/TraceRetrieverImpl.java index 42ac4d2e9..62349bcd8 100644 --- a/microbat/src/main/microbat/sql/TraceRetrieverImpl.java +++ b/microbat/src/main/microbat/sql/TraceRetrieverImpl.java @@ -56,7 +56,7 @@ public List getTraces(String runId) { this.closables.add(rs); while (rs.next()) { - Trace trace = new Trace(rs.getString("trace_id")); + Trace trace = new Trace(null, rs.getString("trace_id")); String threadId = rs.getString("thread_id"); String threadName = rs.getString("thread_name"); boolean isMain = rs.getBoolean("isMain"); diff --git a/microbat/src/main/microbat/trace/GraphTraceReader.java b/microbat/src/main/microbat/trace/GraphTraceReader.java new file mode 100644 index 000000000..3a6a2f941 --- /dev/null +++ b/microbat/src/main/microbat/trace/GraphTraceReader.java @@ -0,0 +1,42 @@ +/** + * + */ +package microbat.trace; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.List; +import java.util.Optional; + +import microbat.instrumentation.output.RunningInfo; +import microbat.instrumentation.precheck.PrecheckInfo; +import microbat.model.trace.Trace; +import microbat.sql.DbService; +import microbat.sql.GraphTraceRetriever; +import microbat.sql.TraceRetrieverImpl; +import microbat.sql.TraceRetriever; +import sav.commons.testdata.calculator.CalculatorTest1; + +public class GraphTraceReader implements TraceReader { + private String runId; + + public GraphTraceReader(String runId) { + this.runId = runId; + } + + /* + * @see microbat.trace.TraceReader#Read() + */ + @Override + public RunningInfo read(PrecheckInfo precheckInfo, String drumpFile) { + List traces = new GraphTraceRetriever(this.runId).getTraces(runId); + + int collectedSteps = traces.isEmpty() ? 0 : + traces.stream().mapToInt(trace -> trace.size()).sum(); + int expectedSteps = precheckInfo.getStepTotal(); + + return new RunningInfo(precheckInfo.getProgramMsg(), traces, expectedSteps, collectedSteps); + } + +} + diff --git a/microbat/src/main/microbat/trace/Reader.java b/microbat/src/main/microbat/trace/Reader.java index d92f55ec6..d0250493e 100644 --- a/microbat/src/main/microbat/trace/Reader.java +++ b/microbat/src/main/microbat/trace/Reader.java @@ -25,6 +25,12 @@ public TraceReader create(String runId) { public TraceReader create(String runId) { return new MysqlTraceReader(); } + }, + NEO4J { + @Override + public TraceReader create(String runId) { + return new GraphTraceReader(runId); + } }; @Override diff --git a/microbat_instrumentator/.classpath b/microbat_instrumentator/.classpath index 75ac85ffd..24f14ec59 100644 --- a/microbat_instrumentator/.classpath +++ b/microbat_instrumentator/.classpath @@ -1,18 +1,24 @@ - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/microbat_instrumentator/.settings/org.eclipse.jdt.core.prefs b/microbat_instrumentator/.settings/org.eclipse.jdt.core.prefs index 980b98c1d..ace45ceef 100644 --- a/microbat_instrumentator/.settings/org.eclipse.jdt.core.prefs +++ b/microbat_instrumentator/.settings/org.eclipse.jdt.core.prefs @@ -1,12 +1,12 @@ eclipse.preferences.version=1 org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate -org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve -org.eclipse.jdt.core.compiler.compliance=1.7 +org.eclipse.jdt.core.compiler.compliance=1.8 org.eclipse.jdt.core.compiler.debug.lineNumber=generate org.eclipse.jdt.core.compiler.debug.localVariable=generate org.eclipse.jdt.core.compiler.debug.sourceFile=generate org.eclipse.jdt.core.compiler.problem.assertIdentifier=error org.eclipse.jdt.core.compiler.problem.enumIdentifier=error -org.eclipse.jdt.core.compiler.source=1.7 +org.eclipse.jdt.core.compiler.source=1.8 diff --git a/microbat_instrumentator/lib/instrumentator_agent_v02.jar b/microbat_instrumentator/lib/instrumentator_agent_v02.jar index b56267093..4a4d07344 100644 Binary files a/microbat_instrumentator/lib/instrumentator_agent_v02.jar and b/microbat_instrumentator/lib/instrumentator_agent_v02.jar differ diff --git a/microbat_instrumentator/lib/neo4j-java-driver-4.3.4.jar b/microbat_instrumentator/lib/neo4j-java-driver-4.3.4.jar new file mode 100644 index 000000000..4a440172d Binary files /dev/null and b/microbat_instrumentator/lib/neo4j-java-driver-4.3.4.jar differ diff --git a/microbat_instrumentator/lib/neo4j-jdbc-driver-4.0.2.jar b/microbat_instrumentator/lib/neo4j-jdbc-driver-4.0.2.jar new file mode 100644 index 000000000..0a974b4bb Binary files /dev/null and b/microbat_instrumentator/lib/neo4j-jdbc-driver-4.0.2.jar differ diff --git a/microbat_instrumentator/src/core/microbat/model/trace/HookedTrace.java b/microbat_instrumentator/src/core/microbat/model/trace/HookedTrace.java new file mode 100644 index 000000000..730ae0579 --- /dev/null +++ b/microbat_instrumentator/src/core/microbat/model/trace/HookedTrace.java @@ -0,0 +1,40 @@ +package microbat.model.trace; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import microbat.sql.IntervalRecorder; +import sav.strategies.dto.AppJavaClassPath; + +/** + * This class manages the tracenodes and queues the node for storage if the threshold is reached + * + */ +public class HookedTrace extends Trace{ + private final IntervalRecorder recorder; + private List dependencies = new ArrayList<>(); + private List nodeCache = new ArrayList<>(); + + public HookedTrace(String traceId, AppJavaClassPath appJavaClassPath, IntervalRecorder recorder) { + super(appJavaClassPath, traceId); + this.recorder = recorder; + } + + @Override + public void addTraceNode(TraceNode node){ + super.addTraceNode(node); + nodeCache.add(node); + if (this.getExecutionList().size() > recorder.getThreshold()) { + this.recorder.partialStore(getTraceId(), nodeCache); // store what hasn't been stored + nodeCache = new ArrayList<>(); // remove from cache + List executionList = new ArrayList<>(dependencies); // set new nodes as those with invocation parent + latest node + executionList.add(node); + setExecutionList(executionList); + } else { + if (node.getInvocationParent() != null) { + dependencies.add(node); + } + } + } +} diff --git a/microbat_instrumentator/src/core/microbat/model/trace/Trace.java b/microbat_instrumentator/src/core/microbat/model/trace/Trace.java index 6447d70d5..7746720eb 100644 --- a/microbat_instrumentator/src/core/microbat/model/trace/Trace.java +++ b/microbat_instrumentator/src/core/microbat/model/trace/Trace.java @@ -37,6 +37,8 @@ public class Trace { private long threadId; private boolean isMain; private String threadName; + private final String traceId; + private int order = 1; /** * This variable is to trace whether the variables in different lines are the same @@ -44,7 +46,7 @@ public class Trace { */ // private LocalVariableScopes localVariableScopes = new LocalVariableScopes(); - public Trace(AppJavaClassPath appJavaClassPath) { + public Trace(AppJavaClassPath appJavaClassPath, String traceId) { if(threadName == null) { String threadName = Thread.currentThread().getName(); this.threadName = threadName; @@ -52,11 +54,7 @@ public Trace(AppJavaClassPath appJavaClassPath) { this.setAppJavaClassPath(appJavaClassPath); executionList = new ArrayList<>(); - } - - public Trace(AppJavaClassPath appJavaClassPath, int stepNums) { - this.setAppJavaClassPath(appJavaClassPath); - this.executionList = new ArrayList<>(stepNums); + this.traceId = traceId; } private List executionList; @@ -85,6 +83,7 @@ public void setExecutionList(List exectionList) { public void addTraceNode(TraceNode node){ this.executionList.add(node); + this.order++; } public int size(){ @@ -437,4 +436,12 @@ public boolean isMain() { public void setMain(boolean isMain) { this.isMain = isMain; } + + public String getTraceId() { + return traceId; + } + + public int getOrder() { + return order; + } } diff --git a/microbat_instrumentator/src/core/microbat/sql/FileRecorder.java b/microbat_instrumentator/src/core/microbat/sql/FileRecorder.java index 6403ea2cb..0e3ba9d59 100644 --- a/microbat_instrumentator/src/core/microbat/sql/FileRecorder.java +++ b/microbat_instrumentator/src/core/microbat/sql/FileRecorder.java @@ -11,6 +11,7 @@ import microbat.instrumentation.AgentParams; import microbat.instrumentation.output.RunningInfo; import microbat.model.trace.Trace; +import microbat.model.trace.TraceNode; /** * @author knightsong @@ -43,4 +44,8 @@ public void store(List traceList) { } + @Override + public void insertSteps(String traceId, List traces) { + // no op + } } diff --git a/microbat_instrumentator/src/core/microbat/sql/GraphRecorder.java b/microbat_instrumentator/src/core/microbat/sql/GraphRecorder.java new file mode 100644 index 000000000..486523bb3 --- /dev/null +++ b/microbat_instrumentator/src/core/microbat/sql/GraphRecorder.java @@ -0,0 +1,123 @@ +package microbat.sql; + +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; + +import org.neo4j.driver.AuthTokens; +import org.neo4j.driver.GraphDatabase; +import org.neo4j.driver.Result; +import org.neo4j.driver.Session; +import org.neo4j.driver.Transaction; + +import org.neo4j.driver.Driver; +import static org.neo4j.driver.Values.parameters; + +import microbat.handler.xml.VarValueXmlWriter; +import sav.common.core.utils.CollectionUtils; +import microbat.model.BreakPoint; +import microbat.model.trace.Trace; +import microbat.model.trace.TraceNode; +import microbat.model.value.VarValue; + +public class GraphRecorder implements TraceRecorder { + private final String runId; + private final Driver driver; + private static final String CONNECTION_URI = "bolt://localhost"; + private static final String CREATE_TRACE_QUERY = "CREATE (t: Trace) SET t = $props"; + private static final String CREATE_TRACE_STEP_RELATION = "MATCH (t: Trace), (s:Step) where t.traceId = s.traceId CREATE (t)-[:COMPRISES]->(s)"; + private static final String CREATE_STEP_IN_RELATIONS = "MATCH (a: Step), (b: Step) WHERE a.traceId = b.traceId AND a.stepIn= b.stepOrder CREATE (a)-[:STEP_IN]->(b)"; + private static final String CREATE_CONTROL_RELATIONS = "MATCH (a: Step), (b: Step) WHERE a.traceId = b.traceId AND a.stepOrder = b.controlDominator CREATE (a)-[:CONTROL_DOMINATES]->(b)"; + private static final String CREATE_INVOCATION_RELATIONS = "MATCH (a: Step), (b: Step) WHERE a.traceId = b.traceId AND a.stepOrder = b.invocationParent CREATE (a)-[:INVOKES]->(b)"; + private static final String CREATE_LOOP_RELATIONS = "MATCH (a: Step), (b: Step) WHERE a.traceId = b.traceId AND a.stepOrder = b.loopParent CREATE (a)-[:LOOPS]->(b)"; + private static final String CREATE_LOCATION_RELATIONS = "MATCH (a: Step), (b: Location) WHERE a.traceId = b.traceId AND a.locationId = b.locationId CREATE (a)-[:AT]->(b)"; + private static final String INSERT_STEPS_QUERY = "CREATE (n:Step) SET n = $props"; + private static final String INSERT_LOCATION = "MERGE (a:Location {locationId: $locationId, traceId: $traceId, className: $className, lineNumber: $lineNumber, isConditional: $isConditional, isReturn: $isReturn})"; + + public GraphRecorder(String runId) { + this.runId = runId; + driver = GraphDatabase.driver(CONNECTION_URI, AuthTokens.basic("neo4j", "microbat")); + } + + private Result insertLocation(final Transaction tx, final BreakPoint bp, final String traceId) { + Map props = new HashMap<>(); + props.put("locationId", bp.getDeclaringCompilationUnitName() + "_" + bp.getLineNumber()); + props.put("traceId", traceId); + props.put("className", bp.getDeclaringCompilationUnitName()); + props.put("lineNumber", bp.getLineNumber()); + props.put("isConditional", bp.isConditional()); + props.put("isReturn", bp.isReturnStatement()); + // Location might be batch inserted since the MERGE check takes a longer time to + // complete than CREATE + return tx.run(INSERT_LOCATION, props); + } + + private void insertSteps(final Session session, final Trace trace, final String traceId) { + for (TraceNode node : trace.getExecutionList()) { + Map props = new HashMap<>(); + props.put("traceId", traceId); + props.put("stepOrder", node.getOrder()); + Optional.ofNullable(node.getControlDominator()) + .ifPresent(cd -> props.put("controlDominator", cd.getOrder())); + Optional.ofNullable(node.getStepInNext()).ifPresent(cd -> props.put("stepIn", cd.getOrder())); + Optional.ofNullable(node.getStepOverNext()).ifPresent(cd -> props.put("stepOver", cd.getOrder())); + Optional.ofNullable(node.getInvocationParent()) + .ifPresent(cd -> props.put("invocationParent", cd.getOrder())); + Optional.ofNullable(node.getLoopParent()).ifPresent(cd -> props.put("loopParent", cd.getOrder())); + props.put("locationId", node.getDeclaringCompilationUnitName() + "_" + node.getLineNumber()); + props.put("time", LocalDateTime.ofEpochSecond(node.getTimestamp(), 0, ZoneOffset.UTC)); + Optional.ofNullable(generateXmlContent(node.getReadVariables())) + .ifPresent(xmlContent -> props.put("readVariables", xmlContent)); + Optional.ofNullable(generateXmlContent(node.getWrittenVariables())) + .ifPresent(xmlContent -> props.put("writeVariables", xmlContent)); + session.writeTransaction(tx -> tx.run(INSERT_STEPS_QUERY, parameters("props", props))); + BreakPoint bp = node.getBreakPoint(); + session.writeTransaction(tx -> insertLocation(tx, bp, traceId)); + } + } + + @Override + public void store(List traces) { + try (Session session = driver.session()) { + for (Trace trace : traces) { + String traceId = UUID.randomUUID().toString(); + Map props = new HashMap<>(); + props.put("runId", runId); + props.put("traceId", traceId); + props.put("threadId", trace.getThreadId()); + props.put("threadName", trace.getThreadName()); + props.put("isMain", trace.isMain()); + session.writeTransaction(tx -> tx.run(CREATE_TRACE_QUERY, parameters("props", props))); + insertSteps(session, trace, traceId); + } + session.writeTransaction(tx -> tx.run(CREATE_STEP_IN_RELATIONS)); + session.writeTransaction(tx -> tx.run(CREATE_CONTROL_RELATIONS)); + session.writeTransaction(tx -> tx.run(CREATE_INVOCATION_RELATIONS)); + session.writeTransaction(tx -> tx.run(CREATE_LOOP_RELATIONS)); + session.writeTransaction(tx -> tx.run(CREATE_LOCATION_RELATIONS)); + session.writeTransaction(tx -> tx.run(CREATE_TRACE_STEP_RELATION)); + } + driver.close(); + } + + protected String generateXmlContent(Collection varValues) { + if (CollectionUtils.isEmpty(varValues)) { + return null; + } + return VarValueXmlWriter.generateXmlContent(varValues); + } + + @Override + public void insertSteps(String traceId, List traces) { + try (Session session = driver.session()) { + Map props = new HashMap<>(); + props.put("traceId", traceId); + session.writeTransaction(tx -> tx.run(CREATE_TRACE_QUERY, parameters("props", props))); + } + } +} diff --git a/microbat_instrumentator/src/core/microbat/sql/IntervalRecorder.java b/microbat_instrumentator/src/core/microbat/sql/IntervalRecorder.java new file mode 100644 index 000000000..2cd4bfe7d --- /dev/null +++ b/microbat_instrumentator/src/core/microbat/sql/IntervalRecorder.java @@ -0,0 +1,49 @@ +package microbat.sql; + +import java.sql.Connection; +import java.sql.Date; +import java.sql.PreparedStatement; +import java.sql.SQLException; + +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +import microbat.handler.xml.VarValueXmlWriter; +import microbat.instrumentation.AgentParams; +import microbat.model.BreakPoint; +import microbat.model.trace.StepVariableRelationEntry; +import microbat.model.trace.Trace; +import microbat.model.trace.TraceNode; +import microbat.model.value.VarValue; +import sav.common.core.utils.CollectionUtils; + +/** + * @author dingyuchen + * + */ +public class IntervalRecorder { + + private final TraceRecorder recorderInstance; + private final int threshold; + + /** + * @param dbPath + */ + public IntervalRecorder(int threshold, TraceRecorder recorder) { + this.threshold = threshold; + this.recorderInstance = recorder; + } + + public synchronized void partialStore(String traceId, List traces) { + recorderInstance.insertSteps(traceId, traces); + } + + public int getThreshold() { + return threshold; + } +} diff --git a/microbat_instrumentator/src/core/microbat/sql/Recorder.java b/microbat_instrumentator/src/core/microbat/sql/Recorder.java index e3bea37c1..499e2bae3 100644 --- a/microbat_instrumentator/src/core/microbat/sql/Recorder.java +++ b/microbat_instrumentator/src/core/microbat/sql/Recorder.java @@ -3,25 +3,26 @@ */ package microbat.sql; - import microbat.instrumentation.AgentParams; + /** * @author knightsong * */ public enum Recorder { - FILE,SQLITE3,MYSQL; - + FILE, SQLITE3, MYSQL, GRAPH; + public static TraceRecorder create(AgentParams params) { switch (params.getTraceRecorderName()) { case "FILE": return new FileRecorder(params); case "SQLITE3": + case "MYSQL": return new SqliteRecorder(params.getDumpFile(), params.getRunId()); -// case "MYSQL": -// return new MysqlRecorder(params.getRunId()); + case "NEO4J": + return new GraphRecorder(params.getRunId()); default: - return new SqliteRecorder(params.getDumpFile(), params.getRunId()); + return new FileRecorder(params); } } diff --git a/microbat_instrumentator/src/core/microbat/sql/SocketRecorder.java b/microbat_instrumentator/src/core/microbat/sql/SocketRecorder.java index 448e7d58d..be9b51890 100644 --- a/microbat_instrumentator/src/core/microbat/sql/SocketRecorder.java +++ b/microbat_instrumentator/src/core/microbat/sql/SocketRecorder.java @@ -10,6 +10,7 @@ import microbat.instrumentation.output.TraceOutputWriter; import microbat.instrumentation.output.tcp.TcpConnector; import microbat.model.trace.Trace; +import microbat.model.trace.TraceNode; /** * @@ -38,5 +39,9 @@ public void store(List traceList) { } tcpConnector.close(); } + @Override + public void insertSteps(String traceId, List traces) { + // no op + } } diff --git a/microbat_instrumentator/src/core/microbat/sql/SqliteRecorder.java b/microbat_instrumentator/src/core/microbat/sql/SqliteRecorder.java index 3370aa3dc..12a2b990a 100644 --- a/microbat_instrumentator/src/core/microbat/sql/SqliteRecorder.java +++ b/microbat_instrumentator/src/core/microbat/sql/SqliteRecorder.java @@ -40,6 +40,7 @@ public SqliteRecorder(String dbPath, String runId) { this.runId = runId; } + @Override public void store(List traces) { Connection conn = null; List closables = new ArrayList(); @@ -59,7 +60,7 @@ public void store(List traces) { } } - public void insertRun(Connection conn, List closables) throws SQLException { + private void insertRun(Connection conn, List closables) throws SQLException { PreparedStatement ps; String sql = "INSERT INTO run (run_id,project_name,project_version,launch_method,thread_status,launch_class) " + "VALUES (?, ?, ?,?,?,?)"; @@ -80,7 +81,7 @@ public void insertRun(Connection conn, List closables) throws SQL * * */ - public String insertTrace(Trace trace, String runId, Connection conn, List closables) + private String insertTrace(Trace trace, String runId, Connection conn, List closables) throws SQLException { PreparedStatement ps; String sql = "INSERT INTO Trace (trace_id, run_id,thread_id,thread_name,isMain,generated_time) " @@ -88,7 +89,7 @@ public String insertTrace(Trace trace, String runId, Connection conn, List exectionList, Connection conn, List closables) - throws SQLException { + @Override + public void insertSteps(String traceId, List traces) { + Connection conn = null; + List closables = new ArrayList<>(); + try { + conn = getConnection(); + insertSteps(traceId, traces, conn, closables); + } catch (SQLException e) { + e.printStackTrace(); + } finally { + closeDb(conn, closables); + } + } + + private void insertSteps(String traceId, List exectionList, Connection conn, + List closables) throws SQLException { String sql = "INSERT INTO Step (trace_id, step_order, control_dominator, step_in, step_over, invocation_parent, loop_parent," + "location_id, read_vars, written_vars, time) VALUES (?,?,?,?,?,?,?,?,?,?,?)"; PreparedStatement ps = conn.prepareStatement(sql); @@ -155,14 +170,8 @@ private void insertStepVariableRelation(Trace trace, String traceId, Connection // ps.setString(idx++, traceId); // ps.addBatch(); // } - // if (++count >= BATCH_SIZE) { - // ps.executeBatch(); - // count = 0; // } - // } - // if (count > 0) { // ps.executeBatch(); - // } } private void insertLocation(String traceId, List nodes, Connection conn, List closables) @@ -171,31 +180,26 @@ private void insertLocation(String traceId, List nodes, Connection co + "VALUES (?,?, ?, ?, ?, ?)"; PreparedStatement ps = conn.prepareStatement(sql); closables.add(ps); - HashSet set = getLoactionSet(nodes); + HashSet set = getLocationSet(nodes); List ids = new ArrayList<>(); for (BreakPoint location : set) { int idx = 1; - String locationId = getUUID(); ps.setString(idx++, location.getDeclaringCompilationUnitName() + "_" + location.getLineNumber()); ps.setString(idx++, traceId); ps.setString(idx++, location.getDeclaringCompilationUnitName()); ps.setInt(idx++, location.getLineNumber()); ps.setBoolean(idx++, location.isConditional()); ps.setBoolean(idx++, location.isReturnStatement()); - ids.add(locationId); ps.addBatch(); } ps.executeBatch(); - if (ids.size() != set.size()) { - throw new SQLException("Number of locations is incorrect!"); - } // insertControlScope(traceId, result, conn, closables); // insertLoopScope(traceId, result, conn, closables); } - private HashSet getLoactionSet(List list) { + private HashSet getLocationSet(List list) { HashSet set = new HashSet<>(); for (TraceNode node : list) { set.add(node.getBreakPoint()); diff --git a/microbat_instrumentator/src/core/microbat/sql/SqliteServer.java b/microbat_instrumentator/src/core/microbat/sql/SqliteServer.java index 697bd1389..90541c692 100644 --- a/microbat_instrumentator/src/core/microbat/sql/SqliteServer.java +++ b/microbat_instrumentator/src/core/microbat/sql/SqliteServer.java @@ -67,12 +67,6 @@ public void closeDb(Connection connection, List closableList) { } } - public String getUUID(){ - UUID uuid=UUID.randomUUID(); - String uuidStr=uuid.toString(); - return uuidStr; - } - // protected int getFirstGeneratedIntCol(PreparedStatement ps) throws SQLException { // int id = -1; // try (ResultSet generatedKeys = ps.getGeneratedKeys()) { diff --git a/microbat_instrumentator/src/core/microbat/sql/TraceRecorder.java b/microbat_instrumentator/src/core/microbat/sql/TraceRecorder.java index 400295b9e..c633300f3 100644 --- a/microbat_instrumentator/src/core/microbat/sql/TraceRecorder.java +++ b/microbat_instrumentator/src/core/microbat/sql/TraceRecorder.java @@ -6,6 +6,7 @@ import java.util.List; import microbat.model.trace.Trace; +import microbat.model.trace.TraceNode; /** * @author knightsong @@ -13,4 +14,6 @@ */ public interface TraceRecorder { void store(List trace); + + void insertSteps(String traceId, List traces); } diff --git a/microbat_instrumentator/src/main/microbat/instrumentation/AgentParams.java b/microbat_instrumentator/src/main/microbat/instrumentation/AgentParams.java index dd720793b..a11a5c042 100644 --- a/microbat_instrumentator/src/main/microbat/instrumentation/AgentParams.java +++ b/microbat_instrumentator/src/main/microbat/instrumentation/AgentParams.java @@ -43,6 +43,7 @@ public class AgentParams extends CommonParams { public static final String OPT_CODE_RANGE = "code_range"; public static final String OPT_TRACE_RECORDER = "trace_recorder"; public static final String OPT_RUN_ID = "run_id"; + public static final String OPT_IS_INC_STORAGE = "incremental_storage"; private boolean precheck; private EntryPoint entryPoint; @@ -57,6 +58,7 @@ public class AgentParams extends CommonParams { private String excludesExpression; private int stepLimit; private int expectedSteps; + private boolean incrementalStorage; private Set overlongMethods; private boolean requireMethodSplit; private boolean avoidProxyToString; @@ -96,6 +98,7 @@ public AgentParams(CommandLine cmd) { overlongMethods = cmd.getStringSet(OPT_OVER_LONG_METHODS); requireMethodSplit = cmd.getBoolean(OPT_REQUIRE_METHOD_SPLITTING, false); avoidProxyToString = cmd.getBoolean(OPT_AVOID_TO_STRING_OF_PROXY_OBJ, false); + incrementalStorage = cmd.getBoolean(OPT_IS_INC_STORAGE, false); codeRanges = CodeRangeEntry.parse(cmd.getStringList(OPT_CODE_RANGE)); recorderName = cmd.getString(OPT_TRACE_RECORDER); runId = cmd.getString(OPT_RUN_ID); @@ -183,6 +186,10 @@ public int getExpectedSteps() { return expectedSteps; } + public boolean isIncrementalStorage() { + return incrementalStorage; + } + public Set getOverlongMethods() { return overlongMethods; } diff --git a/microbat_instrumentator/src/main/microbat/instrumentation/Premain.java b/microbat_instrumentator/src/main/microbat/instrumentation/Premain.java index 1846b942f..28a4df243 100644 --- a/microbat_instrumentator/src/main/microbat/instrumentation/Premain.java +++ b/microbat_instrumentator/src/main/microbat/instrumentation/Premain.java @@ -43,7 +43,7 @@ public static void premain(String agentArgs, Instrumentation inst) throws Except inst.addTransformer(new TestRunnerTranformer()); agent.retransformClasses(retransformableClasses); - debug("after retransform"); + debug("after retransform"); } /** @@ -53,6 +53,9 @@ public static void premain(String agentArgs, Instrumentation inst) throws Except */ private static void installBootstrap(Instrumentation inst) throws Exception { debug("install jar to boostrap..."); +// debug("install jar to boostrap..."); +// debug("install jar to boostrap..."); +// debug("install jar to boostrap..."); File tempFolder = FileUtils.createTempFolder("microbat"); debug("Temp folder to extract jars: " + tempFolder.getAbsolutePath()); List bootJarPaths = getJarFiles("instrumentator_all.jar"); @@ -68,14 +71,16 @@ private static void installBootstrap(Instrumentation inst) throws Exception { * being stored into sql database using TraceRecorder, but later, we don't use that approach to * record trace anymore, so these two lines are commented to avoid any unnecessary cause. * */ - "sqlite-jdbc-3.32.3.2.jar" + "sqlite-jdbc-3.32.3.2.jar", + "neo4j-java-driver-4.3.4.jar", + "neo4j-jdbc-driver-4.0.2.jar" // "slf4j-api-1.7.12.jar" ); } for (JarFile jarfile : bootJarPaths) { debug("append to boostrap classloader: " + jarfile.getName()); inst.appendToBootstrapClassLoaderSearch(jarfile); - if (jarfile.getName().contains("sqlite-jdbc")) { + if (jarfile.getName().contains("jdbc") || jarfile.getName().contains("neo4j")) { inst.appendToSystemClassLoaderSearch(jarfile); } } diff --git a/microbat_instrumentator/src/main/microbat/instrumentation/TraceAgent.java b/microbat_instrumentator/src/main/microbat/instrumentation/TraceAgent.java index d23bcfccc..2a20cd010 100644 --- a/microbat_instrumentator/src/main/microbat/instrumentation/TraceAgent.java +++ b/microbat_instrumentator/src/main/microbat/instrumentation/TraceAgent.java @@ -3,6 +3,7 @@ import java.lang.instrument.Instrumentation; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import microbat.instrumentation.filter.CodeRangeUserFilter; import microbat.instrumentation.filter.GlobalFilterChecker; @@ -13,7 +14,9 @@ import microbat.model.trace.Trace; import microbat.model.trace.TraceNode; import microbat.model.value.VarValue; +import microbat.sql.IntervalRecorder; import microbat.sql.Recorder; +import microbat.sql.TraceRecorder; import sav.common.core.utils.StopTimer; import sav.strategies.dto.AppJavaClassPath; @@ -34,6 +37,12 @@ public void startup(long vmStartupTime, long agentPreStartup) { ExecutionTracer.appJavaClassPath = appPath; ExecutionTracer.variableLayer = agentParams.getVariableLayer(); ExecutionTracer.setStepLimit(agentParams.getStepLimit()); + // TODO: set the proper condition + if (agentParams.isIncrementalStorage()) { // if agent sets recording threshold + TraceRecorder selectedRecorder = Recorder.create(agentParams); + final int threshold = 1000; // TODO: set threshold from input options + ExecutionTracer.setRecorderFactory(threshold, selectedRecorder); + } if (!agentParams.isRequireMethodSplit()) { agentParams.getUserFilters().register(new OverLongMethodFilter(agentParams.getOverlongMethods())); } @@ -57,6 +66,7 @@ public void shutdown() throws Exception { int size = tracers.size(); List traceList = new ArrayList<>(size); + TraceRecorder recorder = Recorder.create(agentParams); for (int i = 0; i < size; i++) { ExecutionTracer tracer = (ExecutionTracer) tracers.get(i); @@ -66,8 +76,11 @@ public void shutdown() throws Exception { trace.setThreadName(tracer.getThreadName()); trace.setMain(ExecutionTracer.getMainThreadStore().equals(tracer)); - constructTrace(trace); + // Construct Trace adds the relationship mappings for each trace node + // TODO: handle relationship construction somewhere + // constructTrace(trace); traceList.add(trace); + } timer.newPoint("Saving trace"); @@ -75,7 +88,7 @@ public void shutdown() throws Exception { AgentLogger.debug(timer.getResultString()); } - //FIXME this method can be handled in an asynchronized way + // FIXME this method can be handled in an asynchronized way public void constructTrace(Trace trace) { GlobalFilterChecker.addFilterInfo(trace); @@ -152,7 +165,7 @@ private void createVirtualDataRelation(Trace trace) { TraceNode firstChild = invocationParent.getInvocationChildren().get(0); if (firstChild.getOrder() == currentNode.getOrder()) { for (VarValue value : currentNode.getPassParameters()) { - invocationParent.addWrittenVariable(value); + invocationParent.addWrittenVariable(value); } } } diff --git a/microbat_instrumentator/src/main/microbat/instrumentation/output/TraceOutputReader.java b/microbat_instrumentator/src/main/microbat/instrumentation/output/TraceOutputReader.java index 1fbe28e49..a28e76e00 100644 --- a/microbat_instrumentator/src/main/microbat/instrumentation/output/TraceOutputReader.java +++ b/microbat_instrumentator/src/main/microbat/instrumentation/output/TraceOutputReader.java @@ -34,7 +34,7 @@ public List readTrace() throws IOException { List traceList = new ArrayList(); for(int i=0; i stepLimit) { shutdown(); Agent._exitProgram("fail;Trace is over long!"); diff --git a/microbat_instrumentator/src/main/microbat/instrumentation/runtime/ExecutionTracerStore.java b/microbat_instrumentator/src/main/microbat/instrumentation/runtime/ExecutionTracerStore.java index f6c240a26..0644620d0 100644 --- a/microbat_instrumentator/src/main/microbat/instrumentation/runtime/ExecutionTracerStore.java +++ b/microbat_instrumentator/src/main/microbat/instrumentation/runtime/ExecutionTracerStore.java @@ -1,6 +1,7 @@ package microbat.instrumentation.runtime; public class ExecutionTracerStore extends TracerStore { + private int threshold = 1000; @Override protected ExecutionTracer initTracer(long threadId) { diff --git a/microbat_instrumentator/src/test/java/microbat/tools/JarPackageTool.java b/microbat_instrumentator/src/test/java/microbat/tools/JarPackageTool.java index 627d6a18c..426407f41 100644 --- a/microbat_instrumentator/src/test/java/microbat/tools/JarPackageTool.java +++ b/microbat_instrumentator/src/test/java/microbat/tools/JarPackageTool.java @@ -89,7 +89,11 @@ public static void main(String[] args) throws Exception { .append("-C").append(BASE_DIR) .append("lib/sqlite-jdbc-3.32.3.2.jar") .append("-C").append(BASE_DIR) - .append("lib/slf4j-api-1.7.12.jar"); + .append("lib/slf4j-api-1.7.12.jar") + .append("-C").append(BASE_DIR) + .append("lib/neo4j-java-driver-4.3.4.jar") + .append("-C").append(BASE_DIR) + .append("lib/neo4j-jdbc-driver-4.0.2.jar"); vmRunner.startAndWaitUntilStop(cmd.toCollection()); System.out.println("Deploy instrumentator.jar to " + DEPLOY_JAR_PATH);