diff --git a/.classpath b/.classpath index 5a1fb79be..6e4d95d68 100644 --- a/.classpath +++ b/.classpath @@ -4,50 +4,23 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -57,51 +30,42 @@ - + - - + - + + - - - + + - + - - - - - - - + + - - - - - + - + - + + @@ -122,23 +86,13 @@ - - - - - + + - - - - - - - diff --git a/.project b/.project index fc24a0cbe..ccd625c78 100644 --- a/.project +++ b/.project @@ -3,6 +3,7 @@ NO_M2ECLIPSE_SUPPORT: Project files created with the maven-eclipse-plugin are not supported in M2Eclipse. hippo-commons + hippo-tomcat-profiler @@ -38,8 +39,12 @@ + + org.eclipse.m2e.core.maven2Builder + + org.eclipse.m2e.core.maven2Nature org.eclipse.jem.workbench.JavaEMFNature org.eclipse.wst.common.modulecore.ModuleCoreNature org.eclipse.wst.common.project.facet.core.nature diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs index 1d152d7ba..dc1b41496 100644 --- a/.settings/org.eclipse.core.resources.prefs +++ b/.settings/org.eclipse.core.resources.prefs @@ -1,3 +1,5 @@ -#Wed Sep 19 14:22:48 KST 2012 eclipse.preferences.version=1 -encoding/=UTF-8 +encoding//src/main/java=UTF-8 +encoding//src/main/resources=UTF-8 +encoding//src/test/java=UTF-8 +encoding//src/test/resources=UTF-8 diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs index 64eca929e..40b79d680 100644 --- a/.settings/org.eclipse.jdt.core.prefs +++ b/.settings/org.eclipse.jdt.core.prefs @@ -1,12 +1,12 @@ -#Wed Jul 04 18:38:57 KST 2012 -encoding//src/main/java=utf-8 -org.eclipse.jdt.core.compiler.compliance=1.6 -encoding//src/main/resources=utf-8 -encoding//src/test/resources=utf-8 -encoding//src/test/java=utf-8 -org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6 -org.eclipse.jdt.core.compiler.problem.enumIdentifier=error eclipse.preferences.version=1 +encoding//src/main/java=utf-8 +encoding//src/main/resources=utf-8 +encoding//src/test/java=utf-8 +encoding//src/test/resources=utf-8 org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled -org.eclipse.jdt.core.compiler.source=1.6 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6 +org.eclipse.jdt.core.compiler.compliance=1.6 org.eclipse.jdt.core.compiler.problem.assertIdentifier=error +org.eclipse.jdt.core.compiler.problem.enumIdentifier=error +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.compiler.source=1.6 diff --git a/.settings/org.eclipse.m2e.core.prefs b/.settings/org.eclipse.m2e.core.prefs new file mode 100644 index 000000000..f897a7f1c --- /dev/null +++ b/.settings/org.eclipse.m2e.core.prefs @@ -0,0 +1,4 @@ +activeProfiles= +eclipse.preferences.version=1 +resolveWorkspaceProjects=true +version=1 diff --git a/WebContent/WEB-INF/views/flow.jsp b/WebContent/WEB-INF/views/flow.jsp index 27b02b624..be9eb78e1 100644 --- a/WebContent/WEB-INF/views/flow.jsp +++ b/WebContent/WEB-INF/views/flow.jsp @@ -1,36 +1,17 @@ -<%@ page language="java" contentType="text/html; charset=EUC-KR" - pageEncoding="EUC-KR"%> +<%@ page language="java" contentType="text/html; charset=UTF-8" pageEncoding="UTF-8"%> <%@ taglib prefix="c" uri="http://java.sun.com/jsp/jstl/core"%> { "nodes": [ - {"name":"Apache"}, - {"name":"BLOC_1"}, - {"name":"BLOC_2"}, - {"name":"BLOC_3"}, - {"name":"Arcus11211"}, - {"name":"Arcus11212"}, - {"name":"Arcus11213"} + + {"name":"${node}"}, + ], - "links": [ - - {"source":0,"target":1,"value":11}, - {"source":0,"target":2,"value":31}, - {"source":0,"target":3,"value":11}, - - {"source":1,"target":4,"value":11}, - {"source":1,"target":5,"value":41}, - {"source":1,"target":6,"value":11}, - - {"source":2,"target":4,"value":111}, - {"source":2,"target":5,"value":114}, - {"source":2,"target":6,"value":11}, - - {"source":3,"target":4,"value":114}, - {"source":3,"target":5,"value":11}, - {"source":3,"target":6,"value":81} - + + {"source":${link.from.sequence},"target":${link.to.sequence},"value":${link.callCount}} + , + ] } \ No newline at end of file diff --git a/WebContent/index.html b/WebContent/index.html index 80116171d..b85195022 100644 --- a/WebContent/index.html +++ b/WebContent/index.html @@ -216,7 +216,7 @@ right : 1, bottom : 6, left : 1 - }, width = 800 - margin.left - margin.right, height = 300 - margin.top - margin.bottom; + }, width = 1300 - margin.left - margin.right, height = 500 - margin.top - margin.bottom; var formatNumber = d3.format(",.0f"), format = function(d) { return formatNumber(d) + " Requests"; @@ -231,7 +231,8 @@ var path = sankey.link(); - d3.json("data.json", function(energy) { +// d3.json("/flow.hippo?host=TEST_AGENT_ID&from=1348565386677&to=1348565386677", function(energy) { + d3.json("/flow.hippo?host=TEST_AGENT_ID&from=1348647506695&to=1348647536693", function(energy) { sankey.nodes(energy.nodes).links(energy.links).layout(32); /* diff --git a/src/main/java/com/nhn/hippo/web/calltree/RPC.java b/src/main/java/com/nhn/hippo/web/calltree/RPC.java new file mode 100644 index 000000000..b071aac6e --- /dev/null +++ b/src/main/java/com/nhn/hippo/web/calltree/RPC.java @@ -0,0 +1,55 @@ +package com.nhn.hippo.web.calltree; + +/** + * + * @author netspider + * + */ +public class RPC implements Comparable { + private int sequence; + private final String id; + private final String agentId; + private final String serviceName; + private final String rpc; + + public RPC(String agentId, String serviceName, String rpc) { + this.id = agentId + ":" + serviceName + ":" + rpc; + this.agentId = agentId; + this.serviceName = serviceName; + this.rpc = rpc; + } + + public String getId() { + return this.id; + } + + public void setSequence(int sequence) { + this.sequence = sequence; + } + + public int getSequence() { + return sequence; + } + + public String getAgentId() { + return agentId; + } + + public String getServiceName() { + return serviceName; + } + + public String getRpc() { + return rpc; + } + + @Override + public int compareTo(RPC rpc) { + return id.compareTo(rpc.id); + } + + @Override + public String toString() { + return id + ", seq=" + sequence; + } +} diff --git a/src/main/java/com/nhn/hippo/web/calltree/RPCCallTree.java b/src/main/java/com/nhn/hippo/web/calltree/RPCCallTree.java new file mode 100644 index 000000000..54739742d --- /dev/null +++ b/src/main/java/com/nhn/hippo/web/calltree/RPCCallTree.java @@ -0,0 +1,104 @@ +package com.nhn.hippo.web.calltree; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; + +import com.profiler.common.dto.thrift.Span; + +/** + * Call Tree + * + * @author netspider + * + */ +public class RPCCallTree { + + private final String PREFIX_CLIENT = "CLIENT:"; + + private final Map rpcs = new HashMap(); + private final Map spanIdToRPCId = new HashMap(); + private final Map requests = new HashMap(); + private final List spans = new ArrayList(); + + private boolean isBuilt = false; + + public void addSpan(Span span) { + System.out.println("Add span to calltree=" + span + "\n"); + + /** + * make RPCs + */ + // TODO: 여기에서 이러지말고 수집할 때 처음부터 table에 저장해둘 수 있나?? + RPC rpc = new RPC(span.getAgentID(), span.getServiceName(), span.getName()); + if (!rpcs.containsKey(rpc.getId())) { + rpcs.put(rpc.getId(), rpc); + } + spanIdToRPCId.put(String.valueOf(span.getSpanID()), rpc.getId()); + + if (span.getParentSpanId() == -1) { + RPC client = new RPC(PREFIX_CLIENT + span.getAgentID(), span.getServiceName(), span.getName()); + rpcs.put(client.getId(), client); + spanIdToRPCId.put(PREFIX_CLIENT + span.getSpanID(), client.getId()); + } + + /** + * Preparing makes link (requests) + */ + spans.add(span); + } + + public RPCCallTree build() { + if (isBuilt) + return this; + + int i = 0; + for (Entry entry : rpcs.entrySet()) { + entry.getValue().setSequence(i++); + } + + for (Span span : spans) { + String from = String.valueOf(span.getParentSpanId()); + String to = String.valueOf(span.getSpanID()); + + RPC fromRPC = rpcs.get(spanIdToRPCId.get(from)); + RPC toRPC = rpcs.get(spanIdToRPCId.get(to)); + + if (fromRPC == null) { + fromRPC = rpcs.get(spanIdToRPCId.get(PREFIX_CLIENT + to)); + } + + Request request = new Request(fromRPC, toRPC); + if (requests.containsKey(request.getId())) { + requests.get(request.getId()).increaseCallCount(); + } else { + requests.put(request.getId(), request); + } + } + + isBuilt = true; + return this; + } + + public Collection getNodes() { + return this.rpcs.values(); + } + + public Collection getLinks() { + return this.requests.values(); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + + sb.append("RPCS=").append(rpcs); + sb.append("\n"); + sb.append("REQUESTS=").append(requests.values()); + + return sb.toString(); + } +} diff --git a/src/main/java/com/nhn/hippo/web/calltree/Request.java b/src/main/java/com/nhn/hippo/web/calltree/Request.java new file mode 100644 index 000000000..c4c0486ac --- /dev/null +++ b/src/main/java/com/nhn/hippo/web/calltree/Request.java @@ -0,0 +1,46 @@ +package com.nhn.hippo.web.calltree; + +/** + * + * @author netspider + * + */ +public class Request { + private final String id; + private final RPC from; + private final RPC to; + private int callCount = 1; + + public Request(RPC from, RPC to) { + this.from = from; + this.to = to; + this.id = from.getId() + to.getId(); + } + + public void increaseCallCount() { + callCount++; + } + + public String getId() { + return id; + } + + public RPC getFrom() { + return from; + } + + public RPC getTo() { + return to; + } + + public int getCallCount() { + return callCount; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("{from=").append(from).append(", to=").append(to).append(", cc=").append(callCount).append("}"); + return sb.toString(); + } +} diff --git a/src/main/java/com/nhn/hippo/web/controller/FlowChartController.java b/src/main/java/com/nhn/hippo/web/controller/FlowChartController.java index f0dd840f8..03d7f71fb 100644 --- a/src/main/java/com/nhn/hippo/web/controller/FlowChartController.java +++ b/src/main/java/com/nhn/hippo/web/controller/FlowChartController.java @@ -10,8 +10,15 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RequestParam; +import com.nhn.hippo.web.calltree.RPCCallTree; import com.nhn.hippo.web.service.FlowChartService; +/** + * retrieve data for drawing call tree. + * + * @author netspider + * + */ @Controller public class FlowChartController { @@ -20,8 +27,7 @@ public class FlowChartController { /** *
-	 * testurl = http://localhost:7080/flow.hippo?host=TEST_AGENT_ID&from=1348453800000&to=1348453900000
-	 * testurl = http://localhost:7080/flow.hippo?host=TEST_AGENT_ID&from=0&to=1348493900000
+	 * testurl = netscurl "http://localhost:7080/flow.hippo?host=TEST_AGENT_ID&from=1348565386677&to=1348565386677"
 	 * 
* * @param model @@ -36,18 +42,22 @@ public class FlowChartController { * get agentId list from 'Servers' */ String[] selectAgentIds = flow.selectAgentIds(hosts); - System.out.println("selectedAgentIds=" + Arrays.toString(selectAgentIds)); /** * get traceId list from 'TraceIndex' */ - List iterator = flow.selectTraceIdsFromTraceIndex(hosts, from, to); + List traceIds = flow.selectTraceIdsFromTraceIndex(hosts, from, to); /** - * get all traces from 'Trace' + * get call tree */ - flow.selectTraces(iterator); + RPCCallTree callTree = flow.selectCallTree(traceIds); + model.addAttribute("nodes", callTree.getNodes()); + model.addAttribute("links", callTree.getLinks()); + model.addAttribute("value", "hello world"); + + System.out.println(callTree.toString()); return "flow"; } diff --git a/src/main/java/com/nhn/hippo/web/service/FlowChartService.java b/src/main/java/com/nhn/hippo/web/service/FlowChartService.java index 2b0d3f9e4..b094fb0bf 100755 --- a/src/main/java/com/nhn/hippo/web/service/FlowChartService.java +++ b/src/main/java/com/nhn/hippo/web/service/FlowChartService.java @@ -2,15 +2,48 @@ package com.nhn.hippo.web.service; import java.util.List; import java.util.Map; -import java.util.UUID; +import com.nhn.hippo.web.calltree.RPCCallTree; import com.profiler.common.dto.thrift.Span; +/** + * + * @author netspider + * + */ public interface FlowChartService { + /** + * select agentIds from Server table + * + * @param hosts + * @return + */ public String[] selectAgentIds(String[] hosts); + /** + * select traceIds from TraceIndex table + * + * @param agentIds + * @param from + * @param to + * @return + */ public List selectTraceIdsFromTraceIndex(String[] agentIds, long from, long to); + /** + * select Traces from Trace table + * + * @param traceIds + * @return + */ public Map> selectTraces(List traceIds); + + /** + * select call tree + * + * @param traceIds + * @return + */ + public RPCCallTree selectCallTree(List traceIds); } diff --git a/src/main/java/com/nhn/hippo/web/service/FlowChartServiceImpl.java b/src/main/java/com/nhn/hippo/web/service/FlowChartServiceImpl.java index 485d2d72f..11eb0dbe5 100755 --- a/src/main/java/com/nhn/hippo/web/service/FlowChartServiceImpl.java +++ b/src/main/java/com/nhn/hippo/web/service/FlowChartServiceImpl.java @@ -18,6 +18,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; +import com.nhn.hippo.web.calltree.RPCCallTree; import com.profiler.common.dto.thrift.Span; import com.profiler.common.hbase.HBaseClient; import com.profiler.common.hbase.HBaseQuery; @@ -99,20 +100,17 @@ public class FlowChartServiceImpl implements FlowChartService { TDeserializer deserializer = new TDeserializer(); - /** - * Map>> - */ NavigableMap>> map = res.getMap(); for (Entry>> entry : map.entrySet()) { - byte[] family = entry.getKey(); - System.out.println("family=" + Bytes.toString(family)); +// byte[] family = entry.getKey(); +// System.out.println("family=" + Bytes.toString(family)); NavigableMap> values = entry.getValue(); for (Entry> value : values.entrySet()) { - byte[] colname = value.getKey(); - System.out.println("colname=" + Bytes.toString(colname)); +// byte[] colname = value.getKey(); +// System.out.println("colname=" + Bytes.toString(colname)); NavigableMap valueSeries = value.getValue(); @@ -131,4 +129,15 @@ public class FlowChartServiceImpl implements FlowChartService { return list; } + @Override + public RPCCallTree selectCallTree(List traceIds) { + List gets = new ArrayList(traceIds.size()); + for (byte[] traceId : traceIds) { + gets.add(new Get(traceId)); + } + + Result[] results = client.get(HBaseTables.TRACES, gets); + + return TracesProcessor.process(results); + } } diff --git a/src/main/java/com/nhn/hippo/web/service/TracesProcessor.java b/src/main/java/com/nhn/hippo/web/service/TracesProcessor.java new file mode 100644 index 000000000..996999673 --- /dev/null +++ b/src/main/java/com/nhn/hippo/web/service/TracesProcessor.java @@ -0,0 +1,54 @@ +package com.nhn.hippo.web.service; + +import java.util.Map.Entry; +import java.util.NavigableMap; + +import org.apache.hadoop.hbase.client.Result; +import org.apache.thrift.TDeserializer; +import org.apache.thrift.TException; + +import com.nhn.hippo.web.calltree.RPCCallTree; +import com.profiler.common.dto.thrift.Span; + +public class TracesProcessor { + + public static RPCCallTree process(Result[] results) { + RPCCallTree callTree = new RPCCallTree(); + + TDeserializer deserializer = new TDeserializer(); + + for (Result res : results) { + /** + * res.getMap() represent + * "Map>>" + */ + NavigableMap>> map = res.getMap(); + + for (Entry>> entry : map.entrySet()) { + NavigableMap> values = entry.getValue(); + + /** + * For each column (SpanID) + */ + for (Entry> value : values.entrySet()) { + NavigableMap valueSeries = value.getValue(); + + /** + * Decode span object + */ + for (Entry v : valueSeries.entrySet()) { + Span span = new Span(); + try { + deserializer.deserialize(span, v.getValue()); + callTree.addSpan(span); + } catch (TException e) { + e.printStackTrace(); + } + } + } + } + } + + return callTree.build(); + } +} diff --git a/src/main/java/com/nhn/hippo/web/timeline/Timeline.java b/src/main/java/com/nhn/hippo/web/timeline/Timeline.java new file mode 100644 index 000000000..611f3b691 --- /dev/null +++ b/src/main/java/com/nhn/hippo/web/timeline/Timeline.java @@ -0,0 +1,11 @@ +package com.nhn.hippo.web.timeline; + +/** + * Trace Timeline + * + * @author netspider + * + */ +public class Timeline { + +} diff --git a/src/main/resources/hbase.properties b/src/main/resources/hbase.properties index 1e914f623..f9e28d69c 100644 --- a/src/main/resources/hbase.properties +++ b/src/main/resources/hbase.properties @@ -1,4 +1,2 @@ -#hbase.client.host=localhost -#hbase.client.port=2181 -hbase.client.host=10.25.131.38 +hbase.client.host=localhost hbase.client.port=2181 \ No newline at end of file