From a97d3b40bee658d5d70a62bf8ab4a0aa1869b9c1 Mon Sep 17 00:00:00 2001 From: Woonduk Kang Date: Thu, 7 May 2015 19:22:14 +0900 Subject: [PATCH] #377 support depth-limited search - add bfs search & process Emulation node --- .../web/controller/MapController.java | 24 +++- .../pinpoint/web/service/BFSLinkSelector.java | 119 +++++++++++++++--- .../web/service/LinkVisitChecker.java | 16 ++- .../pinpoint/web/service/MapServiceImpl.java | 8 +- .../pinpoint/web/service/SearchDepth.java | 5 +- .../map/AcceptApplicationLocalCacheV1.java | 65 ---------- .../pinpoint/web/vo/SearchOption.java | 9 ++ .../pinpoint/web/service/SearchDepthTest.java | 12 +- 8 files changed, 155 insertions(+), 103 deletions(-) delete mode 100644 web/src/main/java/com/navercorp/pinpoint/web/service/map/AcceptApplicationLocalCacheV1.java diff --git a/web/src/main/java/com/navercorp/pinpoint/web/controller/MapController.java b/web/src/main/java/com/navercorp/pinpoint/web/controller/MapController.java index 60df392df..4276afa92 100644 --- a/web/src/main/java/com/navercorp/pinpoint/web/controller/MapController.java +++ b/web/src/main/java/com/navercorp/pinpoint/web/controller/MapController.java @@ -64,6 +64,8 @@ public class MapController { @Autowired private ServiceTypeRegistryService registry; + private static final int DEAULT_MAX_SEARCH_DEPTH = 8; + /** * Server map data query within from ~ to timeframe * @@ -106,9 +108,12 @@ public class MapController { @RequestParam(value = "calleeRange", defaultValue = "64") int calleeRange) { final Range range = new Range(from, to); this.dateLimit.limit(from, to); - logger.debug("range:{}", TimeUnit.MILLISECONDS.toMinutes(range.getRange())); SearchOption searchOption = new SearchOption(callerRange, calleeRange); + assertSearchOption(searchOption); + + logger.info("getSearverMap() applicationName:{} range:{} searchOption:{}", applicationName, TimeUnit.MILLISECONDS.toMinutes(range.getRange()), searchOption); + ServiceType serviceType = registry.findServiceTypeByName(serviceTypeName); Application application = new Application(applicationName, serviceType); @@ -118,6 +123,23 @@ public class MapController { return new MapWrap(map); } + private void assertSearchOption(SearchOption searchOption) { + int callerSearchDepth = searchOption.getCalleeSearchDepth(); + assertSearchDepth(callerSearchDepth, "invalid caller depth:" + callerSearchDepth); + + int calleeSearchDepth = searchOption.getCalleeSearchDepth(); + assertSearchDepth(searchOption.getCallerSearchDepth(), "invalid callee depth:" + calleeSearchDepth); + } + + private void assertSearchDepth(int depth, String message) { + if (depth < 0) { + throw new IllegalArgumentException(message); + } + if (depth > DEAULT_MAX_SEARCH_DEPTH) { + throw new IllegalArgumentException(message); + } + } + /** * Server map data query for the last "Period" timeframe * diff --git a/web/src/main/java/com/navercorp/pinpoint/web/service/BFSLinkSelector.java b/web/src/main/java/com/navercorp/pinpoint/web/service/BFSLinkSelector.java index 6b5da17ef..19ff7d32c 100644 --- a/web/src/main/java/com/navercorp/pinpoint/web/service/BFSLinkSelector.java +++ b/web/src/main/java/com/navercorp/pinpoint/web/service/BFSLinkSelector.java @@ -37,6 +37,7 @@ import org.slf4j.LoggerFactory; import java.util.*; /** + * not thread safe * @author emeroad */ public class BFSLinkSelector implements LinkSelector { @@ -55,7 +56,7 @@ public class BFSLinkSelector implements LinkSelector { private final Set emulationLinkMarker = new HashSet(); - private Set nextNode = new HashSet(); + private final Queue nextQueue = new Queue(); public BFSLinkSelector(MapStatisticsCalleeDao mapStatisticsCalleeDao, MapStatisticsCallerDao mapStatisticsCallerDao, HostApplicationMapDao hostApplicationMapDao) { if (mapStatisticsCalleeDao == null) { @@ -75,15 +76,15 @@ public class BFSLinkSelector implements LinkSelector { /** * Queries for all applications(callee) called by the targetApplication * - * @param targetApplicationSet + * @param targetApplicationList * @param range * @return */ - private LinkDataDuplexMap selectLink(Set targetApplicationSet, Range range, SearchDepth callerDepth, SearchDepth calleeDepth) { + private LinkDataDuplexMap selectLink(List targetApplicationList, Range range, SearchDepth callerDepth, SearchDepth calleeDepth) { final LinkDataDuplexMap searchResult = new LinkDataDuplexMap(); - for (Application targetApplication : targetApplicationSet) { + for (Application targetApplication : targetApplicationList) { final boolean searchCallerNode = checkNextCaller(targetApplication, callerDepth); if (searchCallerNode) { final LinkDataMap caller = mapStatisticsCallerDao.selectCaller(targetApplication, range); @@ -124,10 +125,10 @@ public class BFSLinkSelector implements LinkSelector { return searchResult; } - private void addNextNode(Application nextApplication) { - final boolean add = this.nextNode.add(nextApplication); + private void addNextNode(Application sourceApplication) { + final boolean add = this.nextQueue.addNextNode(sourceApplication); if (!add) { - logger.debug("already visited nextApplication:{}", nextApplication); + logger.debug("already visited. nextNode:{}", sourceApplication); } } @@ -207,7 +208,7 @@ public class BFSLinkSelector implements LinkSelector { private void traceEmulationLink(LinkData acceptApplication) { final boolean add = emulationLinkMarker.add(acceptApplication); if (!add) { - logger.warn("emulationLink add error. {}", acceptApplication ); + logger.warn("emulationLink add error. {}", acceptApplication); } } @@ -305,15 +306,16 @@ public class BFSLinkSelector implements LinkSelector { SearchDepth callerDepth = new SearchDepth(searchOption.getCallerSearchDepth()); SearchDepth calleeDepth = new SearchDepth(searchOption.getCalleeSearchDepth()); + logger.debug("ApplicationMap select {}", sourceApplication); addNextNode(sourceApplication); LinkDataDuplexMap linkDataDuplexMap = new LinkDataDuplexMap(); - while (true) { - final Set currentNode = copyNextNode(); - if (currentNode.isEmpty()) { - break; - } + while (!this.nextQueue.isEmpty()) { + + final List currentNode = this.nextQueue.copyAndClear(); + + logger.debug("size:{} depth caller:{} callee:{} node:{}", currentNode.size(), callerDepth.getDepth(), calleeDepth.getDepth(), currentNode); LinkDataDuplexMap levelData = selectLink(currentNode, range, callerDepth, calleeDepth); linkDataDuplexMap.addLinkDataDuplexMap(levelData); @@ -322,15 +324,74 @@ public class BFSLinkSelector implements LinkSelector { calleeDepth = calleeDepth.nextDepth(); } - fillEmulationLink(linkDataDuplexMap); + if (!emulationLinkMarker.isEmpty()) { + logger.debug("Link emulation size:{}", emulationLinkMarker.size()); + // special case + checkUnsearchEmulationCalleeNode(linkDataDuplexMap, range); + fillEmulationLink(linkDataDuplexMap); + } return linkDataDuplexMap; } - private Set copyNextNode() { - Set currentQueue = this.nextNode; - this.nextNode = new HashSet(); - return currentQueue; + + private void checkUnsearchEmulationCalleeNode(LinkDataDuplexMap searchResult, Range range) { + + List unvisitedList = getUnvisitedEmulationNode(); + if (unvisitedList.isEmpty()) { + logger.debug("unvisited callee node not found"); + return; + } + + logger.info("unvisited callee node {}", unvisitedList); + + final LinkDataMap calleeLinkData = new LinkDataMap(); + for (Application application : unvisitedList) { + LinkDataMap callee = mapStatisticsCalleeDao.selectCallee(application, range); + logger.debug("calleeNode:{}", callee); + calleeLinkData.addLinkDataMap(callee); + } + + LinkDataMap unvisitedNodeFilter = new LinkDataMap(); + for (LinkData linkData : calleeLinkData.getLinkDataList()) { + Application fromApplication = linkData.getFromApplication(); + if (!fromApplication.getServiceType().isWas()) { + continue; + } + Application emulatedApplication = linkData.getToApplication(); + boolean unvisitedNode = isUnVisitedNode(unvisitedList, emulatedApplication, fromApplication); + if (unvisitedNode) { + logger.debug("EmulationCalleeNode:{}", linkData); + unvisitedNodeFilter.addLinkData(linkData); + } + } + logger.debug("UnVisitedNode:{}", unvisitedNodeFilter); + + for (LinkData linkData : unvisitedNodeFilter.getLinkDataList()) { + searchResult.addTargetLinkData(linkData); + } + + } + + private boolean isUnVisitedNode(List unvisitedList, Application toApplication, Application fromApplication) { + for (Application unvisitedApplication : unvisitedList) { + if (toApplication.equals(unvisitedApplication) && linkVisitChecker.isVisitedCaller(fromApplication)) { + return true; + } + } + return false; + } + + private List getUnvisitedEmulationNode() { + Set unvisitedList = new HashSet(); + for (LinkData linkData : this.emulationLinkMarker) { + Application toApplication = linkData.getToApplication(); + boolean isVisited = this.linkVisitChecker.isVisitedCaller(toApplication); + if (!isVisited) { + unvisitedList.add(toApplication); + } + } + return new ArrayList(unvisitedList); } @@ -346,4 +407,26 @@ public class BFSLinkSelector implements LinkSelector { } + static class Queue { + + private final Set nextNode = new HashSet(); + + public boolean addNextNode(Application application) { + return this.nextNode.add(application); + } + + public List copyAndClear() { + List copyList = new ArrayList(this.nextNode); + + this.nextNode.clear(); + + return copyList; + } + + public boolean isEmpty() { + return this.nextNode.isEmpty(); + } + + } + } diff --git a/web/src/main/java/com/navercorp/pinpoint/web/service/LinkVisitChecker.java b/web/src/main/java/com/navercorp/pinpoint/web/service/LinkVisitChecker.java index 10550c593..2bfc1b692 100644 --- a/web/src/main/java/com/navercorp/pinpoint/web/service/LinkVisitChecker.java +++ b/web/src/main/java/com/navercorp/pinpoint/web/service/LinkVisitChecker.java @@ -37,20 +37,24 @@ public class LinkVisitChecker { return visit(callerFound, caller, "Caller"); } + public boolean isVisitedCaller(Application caller) { + return callerFound.contains(caller); + } + public boolean visitCallee(Application callee) { return visit(calleeFound, callee, "Callee"); } - private boolean visit(Set visitedSet, Application caller, String type) { - if (caller == null) { - throw new NullPointerException("caller must not be null"); + private boolean visit(Set visitedSet, Application application, String type) { + if (application == null) { + throw new NullPointerException("application must not be null"); } - final boolean alreadyVisited = !visitedSet.add(caller); + final boolean alreadyVisited = !visitedSet.add(application); if (logger.isDebugEnabled()) { if (alreadyVisited) { - logger.debug("Finding {}. {}={}", type, type, caller); + logger.debug("Finding {}. {}={}", type, type, application); } else { - logger.debug("LinkData exists. Skip finding {}. {} ", type, caller); + logger.debug("LinkData exists. Skip finding {}. {} ", type, application); } } return alreadyVisited; diff --git a/web/src/main/java/com/navercorp/pinpoint/web/service/MapServiceImpl.java b/web/src/main/java/com/navercorp/pinpoint/web/service/MapServiceImpl.java index 47d972d22..fb58f49d6 100644 --- a/web/src/main/java/com/navercorp/pinpoint/web/service/MapServiceImpl.java +++ b/web/src/main/java/com/navercorp/pinpoint/web/service/MapServiceImpl.java @@ -81,13 +81,13 @@ public class MapServiceImpl implements MapService { logger.debug("SelectApplicationMap"); StopWatch watch = new StopWatch("ApplicationMap"); - watch.start("ApplicationMap Hbase Io Fetch Time"); - LinkSelector linkSelector = new DFSLinkSelector(this.mapStatisticsCalleeDao, this.mapStatisticsCallerDao, hostApplicationMapDao); -// LinkSelector linkSelector = new BFSLinkSelector(this.mapStatisticsCalleeDao, this.mapStatisticsCallerDao, hostApplicationMapDao); + watch.start("ApplicationMap Hbase Io Fetch(Caller,Callee) Time"); +// LinkSelector linkSelector = new DFSLinkSelector(this.mapStatisticsCalleeDao, this.mapStatisticsCallerDao, hostApplicationMapDao); + LinkSelector linkSelector = new BFSLinkSelector(this.mapStatisticsCalleeDao, this.mapStatisticsCallerDao, hostApplicationMapDao); LinkDataDuplexMap linkDataDuplexMap = linkSelector.select(sourceApplication, range, searchOption); watch.stop(); - watch.start("ApplicationMap Memory calculation Time"); + watch.start("ApplicationMap MapBuilding(Response) Time"); ApplicationMapBuilder builder = new ApplicationMapBuilder(range, matcherGroup); ApplicationMap map = builder.build(linkDataDuplexMap, agentInfoService, this.mapResponseDao); watch.stop(); diff --git a/web/src/main/java/com/navercorp/pinpoint/web/service/SearchDepth.java b/web/src/main/java/com/navercorp/pinpoint/web/service/SearchDepth.java index 259289fca..c54accf07 100644 --- a/web/src/main/java/com/navercorp/pinpoint/web/service/SearchDepth.java +++ b/web/src/main/java/com/navercorp/pinpoint/web/service/SearchDepth.java @@ -54,12 +54,9 @@ public class SearchDepth { return limit; } - public int getRemainDepth() { - return depth - limit; - } public boolean isDepthOverflow() { - if (limit < depth) { + if (limit <= depth) { return true; } return false; diff --git a/web/src/main/java/com/navercorp/pinpoint/web/service/map/AcceptApplicationLocalCacheV1.java b/web/src/main/java/com/navercorp/pinpoint/web/service/map/AcceptApplicationLocalCacheV1.java deleted file mode 100644 index c7c1d5871..000000000 --- a/web/src/main/java/com/navercorp/pinpoint/web/service/map/AcceptApplicationLocalCacheV1.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright 2014 NAVER Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.navercorp.pinpoint.web.service.map; - -import org.apache.commons.collections.CollectionUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.*; - -/** - * remove later - * @author emeroad - */ -@Deprecated -public class AcceptApplicationLocalCacheV1 { - - private final Map> acceptApplicationLocalCacheV1 = new HashMap>(); - private final Logger logger = LoggerFactory.getLogger(this.getClass()); - - - public Set get(String host) { - final Set hit = acceptApplicationLocalCacheV1.get(host); - if (CollectionUtils.isNotEmpty(hit)) { - logger.debug("acceptApplicationLocalCacheV1 hit"); - return hit; - } - - return Collections.emptySet(); - } - - public void put(String host, Set acceptApplicationSet) { - - if (CollectionUtils.isEmpty(acceptApplicationSet)) { - // initialize for empty value - Set emptySet = Collections.emptySet(); - acceptApplicationLocalCacheV1.put(host, emptySet); - return ; - } - // build cache - for (AcceptApplication acceptApplication : acceptApplicationSet) { - Set findSet = acceptApplicationLocalCacheV1.get(acceptApplication.getHost()); - if (findSet == null) { - findSet = new HashSet(); - acceptApplicationLocalCacheV1.put(acceptApplication.getHost(), findSet); - } - findSet.add(acceptApplication); - } - } - -} diff --git a/web/src/main/java/com/navercorp/pinpoint/web/vo/SearchOption.java b/web/src/main/java/com/navercorp/pinpoint/web/vo/SearchOption.java index 7e2762302..745199382 100644 --- a/web/src/main/java/com/navercorp/pinpoint/web/vo/SearchOption.java +++ b/web/src/main/java/com/navercorp/pinpoint/web/vo/SearchOption.java @@ -39,4 +39,13 @@ public class SearchOption { public int getCalleeSearchDepth() { return calleeSearchDepth; } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("SearchOption{"); + sb.append("callerSearchDepth=").append(callerSearchDepth); + sb.append(", calleeSearchDepth=").append(calleeSearchDepth); + sb.append('}'); + return sb.toString(); + } } diff --git a/web/src/test/java/com/navercorp/pinpoint/web/service/SearchDepthTest.java b/web/src/test/java/com/navercorp/pinpoint/web/service/SearchDepthTest.java index 1404d8624..8fc0e7f2b 100644 --- a/web/src/test/java/com/navercorp/pinpoint/web/service/SearchDepthTest.java +++ b/web/src/test/java/com/navercorp/pinpoint/web/service/SearchDepthTest.java @@ -26,16 +26,18 @@ public class SearchDepthTest { @Test public void testSearchDepth() throws Exception { - SearchDepth test = new SearchDepth(1); + SearchDepth zero = new SearchDepth(2); - Assert.assertEquals(test.getDepth(), 0); + Assert.assertEquals(zero.getDepth(), 0); + Assert.assertFalse(zero.isDepthOverflow()); - SearchDepth oneDepth = test.nextDepth(); + SearchDepth oneDepth = zero.nextDepth(); Assert.assertEquals(oneDepth.getDepth(), 1); Assert.assertFalse(oneDepth.isDepthOverflow()); - Assert.assertEquals(oneDepth.nextDepth().getDepth(), 2); - Assert.assertTrue(oneDepth.nextDepth().isDepthOverflow()); + SearchDepth twoDepth = oneDepth.nextDepth(); + Assert.assertEquals(twoDepth.getDepth(), 2); + Assert.assertTrue(twoDepth.isDepthOverflow()); } } \ No newline at end of file