From 0632deecc2e2540e7bbb8e7e53ed336d462043df Mon Sep 17 00:00:00 2001 From: Woonduk Kang Date: Tue, 15 Jan 2013 02:18:07 +0000 Subject: [PATCH] =?UTF-8?q?[=EA=B0=95=EC=9A=B4=EB=8D=95]=20[LUCYSUS-1744]?= =?UTF-8?q?=20sql=20=EC=B9=98=ED=99=98=20=EB=A1=9C=EC=A7=81=20commit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit git-svn-id: http://svn.bds.nhncorp.com/pe/hippo-web/trunk@1102 84d0f5b1-2673-498c-a247-62c4ff18d310 --- .../com/nhn/hippo/web/dao/AgentInfoDao.java | 9 ++ .../web/dao/hbase/HbaseAgentInfoDao.java | 78 +++++++++++ .../hippo/web/service/SpanServiceImpl.java | 127 +++++++++++------- .../web/dao/hbase/HbaseAgentInfoDaoTest.java | 56 ++++++++ .../resources/test-web-applicationContext.xml | 7 + 5 files changed, 227 insertions(+), 50 deletions(-) create mode 100644 src/main/java/com/nhn/hippo/web/dao/AgentInfoDao.java create mode 100644 src/main/java/com/nhn/hippo/web/dao/hbase/HbaseAgentInfoDao.java create mode 100644 src/test/java/com/nhn/hippo/web/dao/hbase/HbaseAgentInfoDaoTest.java diff --git a/src/main/java/com/nhn/hippo/web/dao/AgentInfoDao.java b/src/main/java/com/nhn/hippo/web/dao/AgentInfoDao.java new file mode 100644 index 000000000..7e1be2003 --- /dev/null +++ b/src/main/java/com/nhn/hippo/web/dao/AgentInfoDao.java @@ -0,0 +1,9 @@ +package com.nhn.hippo.web.dao; + +/** + * + */ +public interface AgentInfoDao { + + long selectAgentInfoBeforeStartTime(String agentInfo, long currentTime); +} diff --git a/src/main/java/com/nhn/hippo/web/dao/hbase/HbaseAgentInfoDao.java b/src/main/java/com/nhn/hippo/web/dao/hbase/HbaseAgentInfoDao.java new file mode 100644 index 000000000..26263fa90 --- /dev/null +++ b/src/main/java/com/nhn/hippo/web/dao/hbase/HbaseAgentInfoDao.java @@ -0,0 +1,78 @@ +package com.nhn.hippo.web.dao.hbase; + +import com.nhn.hippo.web.dao.AgentInfoDao; +import com.profiler.common.hbase.HBaseTables; +import com.profiler.common.hbase.HbaseOperations2; +import com.profiler.common.util.BytesUtils; +import com.profiler.common.util.RowKeyUtils; +import com.profiler.common.util.TimeUtils; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.ResultScanner; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.util.Bytes; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.hadoop.hbase.ResultsExtractor; +import org.springframework.stereotype.Repository; + +/** + * + */ +@Repository +public class HbaseAgentInfoDao implements AgentInfoDao { + + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + + @Autowired + private HbaseOperations2 hbaseOperations2; + + /** + * currentTime에서 가장 근접한 시간의 agent startTime을 find한다. + * + * @param agentInfo + * @param currentTime + * @return + */ + @Override + public long selectAgentInfoBeforeStartTime(final String agentInfo, final long currentTime) { + Scan scan = createScan(agentInfo, currentTime); + Long startTime = hbaseOperations2.find(HBaseTables.AGENTINFO, scan, new ResultsExtractor() { + @Override + public Long extractData(ResultScanner results) throws Exception { + for (Result next; (next = results.next()) != null; ) { + byte[] row = next.getRow(); + long reverseStartTime = BytesUtils.bytesToLong(row, RowKeyUtils.AGENT_NAME_LIMIT); + long startTime = TimeUtils.recoveryCurrentTimeMillis(reverseStartTime); + logger.debug("agent:{} startTime value {}", agentInfo, startTime); + // 바로 전 시작 시간을 찾아야 한다. + if (startTime < currentTime) { + logger.info("agent:{} startTime find {}", agentInfo, startTime); + return startTime; + } + } + return 0L; + } + }); + +// if (startTime == null) { +// return -1; +// } + return startTime; + } + + private Scan createScan(String agentInfo, long currentTime) { + Scan scan = new Scan(); + scan.setCaching(20); + + byte[] agentIdBytes = Bytes.toBytes(agentInfo); + long startTime = TimeUtils.reverseCurrentTimeMillis(currentTime); + byte[] startKeyBytes = RowKeyUtils.concatFixedByteAndLong(agentIdBytes, RowKeyUtils.AGENT_NAME_LIMIT, startTime); + scan.setStartRow(startKeyBytes); + + byte[] endKeyBytes = RowKeyUtils.concatFixedByteAndLong(agentIdBytes, RowKeyUtils.AGENT_NAME_LIMIT, Long.MAX_VALUE); + scan.setStopRow(endKeyBytes); + + return scan; + } +} diff --git a/src/main/java/com/nhn/hippo/web/service/SpanServiceImpl.java b/src/main/java/com/nhn/hippo/web/service/SpanServiceImpl.java index 5c6601e5c..623ea32ec 100644 --- a/src/main/java/com/nhn/hippo/web/service/SpanServiceImpl.java +++ b/src/main/java/com/nhn/hippo/web/service/SpanServiceImpl.java @@ -3,6 +3,7 @@ package com.nhn.hippo.web.service; import com.nhn.hippo.web.calltree.span.SpanAlign; import com.nhn.hippo.web.calltree.span.SpanAligner; import com.nhn.hippo.web.calltree.span.SpanPopulator; +import com.nhn.hippo.web.dao.AgentInfoDao; import com.nhn.hippo.web.dao.SqlMetaDataDao; import com.nhn.hippo.web.dao.TraceDao; import com.profiler.common.AnnotationNames; @@ -35,6 +36,9 @@ public class SpanServiceImpl implements SpanService { @Autowired private SqlMetaDataDao sqlMetaDataDao; + @Autowired + private AgentInfoDao agentInfoDao; + @Override public List selectSpan(String uuid) { UUID id = UUID.fromString(uuid); @@ -47,10 +51,7 @@ public class SpanServiceImpl implements SpanService { transitionApiId(order); transitionSqlId(order); // TODO root span not found시 row data라도 보여줘야 됨. - if (order.size() != spans.size()) { - // TODO 중간 노드 데이터 분실 ? 혹은 잘못된 데이터 생성? - logger.info("span node not complete! "); - } + return order; } @@ -73,39 +74,60 @@ public class SpanServiceImpl implements SpanService { this.transitionAnnotation(spans, new AnnotationReplacementCallback() { @Override public void replacement(SpanAlign spanAlign, List annotationBoList) { - for (AnnotationBo annotationBo : annotationBoList) { - // TODO SQL-ID 일단 날코딩 나중에 뭔가 key를 따자 - if ("SQL-ID".equals(annotationBo.getKey())) { - - String agentId = getAgentId(spanAlign); - // TODO 일단 시간까지 조회는 하지 말고 하자. - int hashCode = (Integer) annotationBo.getValue(); - List sqlMetaDataList = sqlMetaDataDao.getSqlMetaData(agentId, hashCode, 0); - int size = sqlMetaDataList.size(); - if (size == 0) { - AnnotationBo api = new AnnotationBo(); - api.setKey(AnnotationNames.SQL_METADATA); - api.setValue("SQL-ID not found hashCode:" + hashCode); - annotationBoList.add(api); - } else if (size == 1) { - AnnotationBo api = new AnnotationBo(); - api.setKey(AnnotationNames.SQL_METADATA); - api.setValue(sqlMetaDataList.get(0).getSql()); - annotationBoList.add(api); - } else { - AnnotationBo api = new AnnotationBo(); - api.setKey(AnnotationNames.SQL_METADATA); - api.setValue(collisionSqlHashCodeMessage(hashCode, sqlMetaDataList)); - annotationBoList.add(api); - } - - break; - } + AnnotationBo sqlIdAnnotation = findAnnotation(annotationBoList, AnnotationNames.SQL_ID); + if (sqlIdAnnotation == null) { + return; } + + String agentId = getAgentId(spanAlign); + long startTime = spanAlign.getSpan().getStartTime(); + long agentStartTime = agentInfoDao.selectAgentInfoBeforeStartTime(agentId, startTime); + logger.info("{} Agent StartTime fonud:{}", agentId, agentStartTime); + + // TODO 일단 시간까지 조회는 하지 말고 하자. + int hashCode = (Integer) sqlIdAnnotation.getValue(); + List sqlMetaDataList = sqlMetaDataDao.getSqlMetaData(agentId, hashCode, agentStartTime); + int size = sqlMetaDataList.size(); + if (size == 0) { + AnnotationBo api = new AnnotationBo(); + api.setKey(AnnotationNames.SQL_METADATA); + api.setValue("SQL-ID not found hashCode:" + hashCode); + annotationBoList.add(api); + } else if (size == 1) { + AnnotationBo sqlParamAnnotationBo = findAnnotation(annotationBoList, AnnotationNames.SQL_PARAM); + if (sqlParamAnnotationBo == null) { + AnnotationBo sqlMeta = new AnnotationBo(); + sqlMeta.setKey(AnnotationNames.SQL_METADATA); + sqlMeta.setValue(sqlMetaDataList.get(0).getSql()); + annotationBoList.add(sqlMeta); + + AnnotationBo sql = new AnnotationBo(); + sql.setKey(AnnotationNames.SQL); + sql.setValue(sqlMetaDataList.get(0).getSql()); + annotationBoList.add(sql); + } else { + // merge 해야 된다. + } + } else { + AnnotationBo api = new AnnotationBo(); + api.setKey(AnnotationNames.SQL_METADATA); + api.setValue(collisionSqlHashCodeMessage(hashCode, sqlMetaDataList)); + annotationBoList.add(api); + } + } }); } + private AnnotationBo findAnnotation(List annotationBoList, String key) { + for (AnnotationBo annotationBo : annotationBoList) { + if (key.equals(annotationBo.getKey())) { + return annotationBo; + } + } + return null; + } + private String collisionSqlHashCodeMessage(int hashCode, List sqlMetaDataList) { // TODO 이거 체크하는 테스트를 따로 만들어야 될듯 하다. 왠간하면 확율상 hashCode 충돌 케이스를 쉽게 만들수 없음. StringBuilder sb = new StringBuilder(64); @@ -134,25 +156,26 @@ public class SpanServiceImpl implements SpanService { this.transitionAnnotation(spans, new AnnotationReplacementCallback() { @Override public void replacement(SpanAlign spanAlign, List annotationBoList) { - for (AnnotationBo annotationBo : annotationBoList) { - // TODO API-ID 일단 날코딩 나중에 뭔가 key를 따자 - if ("API-ID".equals(annotationBo.getKey())) { - MethodMapping methodMapping = ApiMappingTable.findMethodMapping((Integer) annotationBo.getValue()); - if (methodMapping == null) { - continue; - } - String className = methodMapping.getClassMapping().getClassName(); - String methodName = methodMapping.getMethodName(); - String[] parameterType = methodMapping.getParameterType(); - String[] parameterName = methodMapping.getParameterName(); - String args = ApiUtils.mergeParameterVariableNameDescription(parameterType, parameterName); - AnnotationBo api = new AnnotationBo(); - api.setKey("API"); - api.setValue(className + "." + methodName + args); - annotationBoList.add(api); - break; - } + + AnnotationBo apiIdAnnotation = findAnnotation(annotationBoList, AnnotationNames.API_ID); + if (apiIdAnnotation == null) { + return; } + + MethodMapping methodMapping = ApiMappingTable.findMethodMapping((Integer) apiIdAnnotation.getValue()); + if (methodMapping == null) { + return; + } + String className = methodMapping.getClassMapping().getClassName(); + String methodName = methodMapping.getMethodName(); + String[] parameterType = methodMapping.getParameterType(); + String[] parameterName = methodMapping.getParameterName(); + String args = ApiUtils.mergeParameterVariableNameDescription(parameterType, parameterName); + AnnotationBo api = new AnnotationBo(); + api.setKey("API"); + api.setValue(className + "." + methodName + args); + annotationBoList.add(api); + } }); } @@ -164,6 +187,10 @@ public class SpanServiceImpl implements SpanService { private List order(List spans) { SpanAligner spanAligner = new SpanAligner(spans); List sort = spanAligner.sort(); + if (sort.size() != spans.size()) { + // TODO 중간 노드 데이터 분실 ? 혹은 잘못된 데이터 생성? + logger.warn("span node not complete! spans:{}, sort{}", spans, sort); + } SpanPopulator spanPopulator = new SpanPopulator(sort); List populatedList = spanPopulator.populateSubSpan(); return populatedList; diff --git a/src/test/java/com/nhn/hippo/web/dao/hbase/HbaseAgentInfoDaoTest.java b/src/test/java/com/nhn/hippo/web/dao/hbase/HbaseAgentInfoDaoTest.java new file mode 100644 index 000000000..290f1524c --- /dev/null +++ b/src/test/java/com/nhn/hippo/web/dao/hbase/HbaseAgentInfoDaoTest.java @@ -0,0 +1,56 @@ +package com.nhn.hippo.web.dao.hbase; + +import com.profiler.common.dto.thrift.AgentInfo; +import junit.framework.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; + +/** + * + */ +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration("classpath:test-web-applicationContext.xml") +public class HbaseAgentInfoDaoTest { + + @Autowired + private HbaseAgentInfoDao selectDao; + @Autowired + private com.profiler.server.dao.hbase.HbaseAgentInfoDao insertDao; + + @Test + public void testSelectAgentInfoStartTime() throws Exception { + AgentInfo agentInfo1 = createAgentInfo(10000); + insertDao.insert(agentInfo1); + + AgentInfo agentInfo2 = createAgentInfo(20000); + insertDao.insert(agentInfo2); + + AgentInfo agentInfo3 = createAgentInfo(30000); + insertDao.insert(agentInfo3); + + long testcaseAgent1 = selectDao.selectAgentInfoBeforeStartTime("testcaseAgent", 20005); + Assert.assertEquals(testcaseAgent1, 20000); + + long testcaseAgent2 = selectDao.selectAgentInfoBeforeStartTime("testcaseAgent", 10004); + Assert.assertEquals(testcaseAgent2, 10000); + + long testcaseAgent3 = selectDao.selectAgentInfoBeforeStartTime("testcaseAgent", 50000); + Assert.assertEquals(testcaseAgent3, 30000); + + + } + + private AgentInfo createAgentInfo(long startTime) { + AgentInfo agentInfo = new AgentInfo(); + agentInfo.setAgentId("testcaseAgent"); + agentInfo.setApplicationName("testcaseApplication"); + agentInfo.setHostname("testcaseHostName"); + agentInfo.setPorts("9995"); + agentInfo.setIsAlive(true); + agentInfo.setTimestamp(startTime); + return agentInfo; + } +} diff --git a/src/test/resources/test-web-applicationContext.xml b/src/test/resources/test-web-applicationContext.xml index 2d5a0dffe..faf4f1422 100644 --- a/src/test/resources/test-web-applicationContext.xml +++ b/src/test/resources/test-web-applicationContext.xml @@ -30,9 +30,16 @@ + + + + + + +