From 69fbc4853cf2516cd9403b396cc84af95cf50eb4 Mon Sep 17 00:00:00 2001 From: Woonduk Kang Date: Tue, 19 Jul 2016 14:03:47 +0900 Subject: [PATCH] #1819 Trace format v2 (#1913) reviewed by Xylus --- .../pinpoint/collector/dao/TraceDao.java | 13 + .../pinpoint/collector/dao/TracesDao.java | 5 +- .../dao/hbase/DualWriteHbaseTraceDao.java | 73 +++ .../collector/dao/hbase/HbaseTraceDao.java | 6 +- .../dao/hbase/HbaseTraceDaoFactory.java | 58 ++ .../collector/dao/hbase/HbaseTraceDaoV2.java | 179 +++++++ .../collector/handler/SpanChunkHandler.java | 10 +- .../collector/handler/SpanHandler.java | 6 +- .../resources/applicationContext-hbase.xml | 12 + .../resources/pinpoint-collector.properties | 4 + .../pinpoint/common/hbase/HBaseTables.java | 9 +- commons-server/pom.xml | 1 + .../common/server/bo/AnnotationBo.java | 17 +- .../pinpoint/common/server/bo/SpanBo.java | 21 +- .../common/server/bo/SpanChunkBo.java | 130 +++++ .../common/server/bo/SpanEventBo.java | 27 +- .../{ => trace/v1}/AnnotationSerializer.java | 4 +- .../{ => trace/v1}/SpanEventSerializer.java | 5 +- .../{ => trace/v1}/SpanSerializer.java | 4 +- .../trace/v2/SpanChunkSerializerV2.java | 34 ++ .../bo/serializer/trace/v2/SpanDecoder.java | 505 ++++++++++++++++++ .../trace/v2/SpanDecodingContext.java | 48 ++ .../bo/serializer/trace/v2/SpanEncoder.java | 428 +++++++++++++++ .../trace/v2/SpanEncodingContext.java | 30 ++ .../serializer/trace/v2/SpanSerializerV2.java | 41 ++ .../v2/bitfield/DepthEncodingStrategy.java | 16 + .../v2/bitfield/SequenceEncodingStrategy.java | 18 + .../bitfield/ServiceTypeEncodingStrategy.java | 18 + .../SimpleServiceTypeEncodingStrategy.java | 21 + .../trace/v2/bitfield/SpanBitFiled.java | 178 ++++++ .../trace/v2/bitfield/SpanEventBitField.java | 323 +++++++++++ .../bitfield/SpanEventQualifierBitField.java | 45 ++ .../StartElapsedTimeEncodingStrategy.java | 16 + .../common/server/bo/AnnotationBoTest.java | 2 +- .../common/server/bo/SpanEventBoTest.java | 4 +- .../serializer/{ => trace/v1}/SpanBoTest.java | 3 +- .../serializer/trace/v2/SpanEncoderTest.java | 20 + .../trace/v2/SpanSerializerV2Test.java | 26 + .../trace/v2/bitfield/SpanBitFiledTest.java | 151 ++++++ .../v2/bitfield/SpanEventBitFieldTest.java | 29 + hbase/scripts/hbase-create-snappy.hbase | 2 + hbase/scripts/hbase-create.hbase | 2 + hbase/scripts/hbase-drop.hbase | 1 + .../navercorp/pinpoint/web/dao/TraceDao.java | 2 + .../web/dao/hbase/HbaseDualReadDao.java | 148 +++++ .../dao/hbase/HbaseTraceCompatibilityDao.java | 80 +++ .../pinpoint/web/dao/hbase/HbaseTraceDao.java | 18 +- .../web/dao/hbase/HbaseTraceDaoFactory.java | 62 +++ .../web/dao/hbase/HbaseTraceDaoV2.java | 170 ++++++ .../pinpoint/web/mapper/CellTraceMapper.java | 46 ++ .../pinpoint/web/mapper/SpanMapper.java | 10 +- .../pinpoint/web/mapper/SpanMapperV2.java | 194 +++++++ .../web/service/FilteredMapServiceImpl.java | 2 + .../web/service/ScatterChartServiceImpl.java | 2 + .../pinpoint/web/service/SpanServiceImpl.java | 5 +- .../service/TransactionInfoServiceImpl.java | 2 + .../pinpoint/web/util/CellTracker.java | 14 + .../pinpoint/web/util/DefaultCellTracker.java | 38 ++ .../pinpoint/web/util/EmptyCellTracker.java | 24 + .../resources/applicationContext-hbase.xml | 9 + .../main/resources/pinpoint-web.properties | 6 +- .../pinpoint/web/filter/LinkFilterTest.java | 4 +- .../pinpoint/web/mapper/SpanMapperV2Test.java | 106 ++++ 63 files changed, 3428 insertions(+), 59 deletions(-) create mode 100644 collector/src/main/java/com/navercorp/pinpoint/collector/dao/TraceDao.java create mode 100644 collector/src/main/java/com/navercorp/pinpoint/collector/dao/hbase/DualWriteHbaseTraceDao.java create mode 100644 collector/src/main/java/com/navercorp/pinpoint/collector/dao/hbase/HbaseTraceDaoFactory.java create mode 100644 collector/src/main/java/com/navercorp/pinpoint/collector/dao/hbase/HbaseTraceDaoV2.java create mode 100644 commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/SpanChunkBo.java rename commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/{ => trace/v1}/AnnotationSerializer.java (91%) rename commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/{ => trace/v1}/SpanEventSerializer.java (93%) rename commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/{ => trace/v1}/SpanSerializer.java (93%) create mode 100644 commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/SpanChunkSerializerV2.java create mode 100644 commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/SpanDecoder.java create mode 100644 commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/SpanDecodingContext.java create mode 100644 commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/SpanEncoder.java create mode 100644 commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/SpanEncodingContext.java create mode 100644 commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/SpanSerializerV2.java create mode 100644 commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/DepthEncodingStrategy.java create mode 100644 commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/SequenceEncodingStrategy.java create mode 100644 commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/ServiceTypeEncodingStrategy.java create mode 100644 commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/SimpleServiceTypeEncodingStrategy.java create mode 100644 commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/SpanBitFiled.java create mode 100644 commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/SpanEventBitField.java create mode 100644 commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/SpanEventQualifierBitField.java create mode 100644 commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/StartElapsedTimeEncodingStrategy.java rename commons-server/src/test/java/com/navercorp/pinpoint/common/server/bo/serializer/{ => trace/v1}/SpanBoTest.java (97%) create mode 100644 commons-server/src/test/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/SpanEncoderTest.java create mode 100644 commons-server/src/test/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/SpanSerializerV2Test.java create mode 100644 commons-server/src/test/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/SpanBitFiledTest.java create mode 100644 commons-server/src/test/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/SpanEventBitFieldTest.java create mode 100644 web/src/main/java/com/navercorp/pinpoint/web/dao/hbase/HbaseDualReadDao.java create mode 100644 web/src/main/java/com/navercorp/pinpoint/web/dao/hbase/HbaseTraceCompatibilityDao.java create mode 100644 web/src/main/java/com/navercorp/pinpoint/web/dao/hbase/HbaseTraceDaoFactory.java create mode 100644 web/src/main/java/com/navercorp/pinpoint/web/dao/hbase/HbaseTraceDaoV2.java create mode 100644 web/src/main/java/com/navercorp/pinpoint/web/mapper/CellTraceMapper.java create mode 100644 web/src/main/java/com/navercorp/pinpoint/web/mapper/SpanMapperV2.java create mode 100644 web/src/main/java/com/navercorp/pinpoint/web/util/CellTracker.java create mode 100644 web/src/main/java/com/navercorp/pinpoint/web/util/DefaultCellTracker.java create mode 100644 web/src/main/java/com/navercorp/pinpoint/web/util/EmptyCellTracker.java create mode 100644 web/src/test/java/com/navercorp/pinpoint/web/mapper/SpanMapperV2Test.java diff --git a/collector/src/main/java/com/navercorp/pinpoint/collector/dao/TraceDao.java b/collector/src/main/java/com/navercorp/pinpoint/collector/dao/TraceDao.java new file mode 100644 index 000000000..04324d1df --- /dev/null +++ b/collector/src/main/java/com/navercorp/pinpoint/collector/dao/TraceDao.java @@ -0,0 +1,13 @@ +package com.navercorp.pinpoint.collector.dao; + +import com.navercorp.pinpoint.thrift.dto.TSpan; +import com.navercorp.pinpoint.thrift.dto.TSpanChunk; + +/** + * @author Woonduk Kang(emeroad) + */ +public interface TraceDao { + void insert(TSpan span); + + void insertSpanChunk(TSpanChunk spanChunk); +} diff --git a/collector/src/main/java/com/navercorp/pinpoint/collector/dao/TracesDao.java b/collector/src/main/java/com/navercorp/pinpoint/collector/dao/TracesDao.java index e6052edbf..79611a7fe 100644 --- a/collector/src/main/java/com/navercorp/pinpoint/collector/dao/TracesDao.java +++ b/collector/src/main/java/com/navercorp/pinpoint/collector/dao/TracesDao.java @@ -22,8 +22,7 @@ import com.navercorp.pinpoint.thrift.dto.TSpanChunk; /** * @author emeroad */ -public interface TracesDao { - void insert(TSpan span); +@Deprecated +public interface TracesDao extends TraceDao { - void insertSpanChunk(TSpanChunk spanChunk); } diff --git a/collector/src/main/java/com/navercorp/pinpoint/collector/dao/hbase/DualWriteHbaseTraceDao.java b/collector/src/main/java/com/navercorp/pinpoint/collector/dao/hbase/DualWriteHbaseTraceDao.java new file mode 100644 index 000000000..4992a6e44 --- /dev/null +++ b/collector/src/main/java/com/navercorp/pinpoint/collector/dao/hbase/DualWriteHbaseTraceDao.java @@ -0,0 +1,73 @@ +package com.navercorp.pinpoint.collector.dao.hbase; + +import com.navercorp.pinpoint.collector.dao.TraceDao; +import com.navercorp.pinpoint.thrift.dto.TSpan; +import com.navercorp.pinpoint.thrift.dto.TSpanChunk; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * usage for development env + * @author Woonduk Kang(emeroad) + */ +public class DualWriteHbaseTraceDao implements TraceDao { + + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + + private final TraceDao master; + private final TraceDao slave; + + public DualWriteHbaseTraceDao(TraceDao master, TraceDao slave) { + if (master == null) { + throw new NullPointerException("master must not be null"); + } + if (slave == null) { + throw new NullPointerException("slave must not be null"); + } + this.master = master; + this.slave = slave; + } + + @Override + public void insert(TSpan span) { + Throwable masterException = null; + try { + master.insert(span); + } catch (Throwable e) { + masterException = e; + } + try { + slave.insert(span); + } catch (Throwable e) { + logger.warn("slave insert(TSpan) Error:{}", e.getMessage(), e); + } + rethrowRuntimeException(masterException); + } + + @Override + public void insertSpanChunk(TSpanChunk spanChunk) { + Throwable masterException = null; + try { + master.insertSpanChunk(spanChunk); + } catch (Throwable e) { + masterException = e; + } + try { + slave.insertSpanChunk(spanChunk); + } catch (Throwable e) { + logger.warn("slave insertSpanChunk(TSpanChunk) Error:{}", e.getMessage(), e); + } + rethrowRuntimeException(masterException); + } + + private void rethrowRuntimeException(Throwable exception) { + if (exception != null) { + this.rethrowException(exception); + } + } + + @SuppressWarnings("unchecked") + private void rethrowException(final Throwable exception) throws T { + throw (T) exception; + } +} diff --git a/collector/src/main/java/com/navercorp/pinpoint/collector/dao/hbase/HbaseTraceDao.java b/collector/src/main/java/com/navercorp/pinpoint/collector/dao/hbase/HbaseTraceDao.java index ae78db22d..400ebdd48 100644 --- a/collector/src/main/java/com/navercorp/pinpoint/collector/dao/hbase/HbaseTraceDao.java +++ b/collector/src/main/java/com/navercorp/pinpoint/collector/dao/hbase/HbaseTraceDao.java @@ -18,9 +18,9 @@ package com.navercorp.pinpoint.collector.dao.hbase; import com.navercorp.pinpoint.collector.dao.TracesDao; import com.navercorp.pinpoint.collector.dao.hbase.filter.SpanEventFilter; -import com.navercorp.pinpoint.common.server.bo.serializer.AnnotationSerializer; -import com.navercorp.pinpoint.common.server.bo.serializer.SpanEventSerializer; -import com.navercorp.pinpoint.common.server.bo.serializer.SpanSerializer; +import com.navercorp.pinpoint.common.server.bo.serializer.trace.v1.AnnotationSerializer; +import com.navercorp.pinpoint.common.server.bo.serializer.trace.v1.SpanEventSerializer; +import com.navercorp.pinpoint.common.server.bo.serializer.trace.v1.SpanSerializer; import com.navercorp.pinpoint.common.server.util.AcceptedTimeService; import com.navercorp.pinpoint.common.server.bo.SpanBo; import com.navercorp.pinpoint.common.server.bo.SpanEventBo; diff --git a/collector/src/main/java/com/navercorp/pinpoint/collector/dao/hbase/HbaseTraceDaoFactory.java b/collector/src/main/java/com/navercorp/pinpoint/collector/dao/hbase/HbaseTraceDaoFactory.java new file mode 100644 index 000000000..20189d724 --- /dev/null +++ b/collector/src/main/java/com/navercorp/pinpoint/collector/dao/hbase/HbaseTraceDaoFactory.java @@ -0,0 +1,58 @@ +package com.navercorp.pinpoint.collector.dao.hbase; + +import com.navercorp.pinpoint.collector.dao.TraceDao; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.FactoryBean; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Repository; + +/** + * @author Woonduk Kang(emeroad) + */ +@Repository +public class HbaseTraceDaoFactory implements FactoryBean { + + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + + @Autowired + @Qualifier("hbaseTraceDao") + private TraceDao v1; + + @Autowired + @Qualifier("hbaseTraceDaoV2") + private TraceDao v2; + + @Value("#{pinpoint_collector_properties['collector.experimental.span.format.compatibility.version'] ?: 'v1'}") + private String mode = "v1"; + + @Override + public TraceDao getObject() throws Exception { + + logger.info("TraceDao Compatibility {}", mode); + + if (mode.equalsIgnoreCase("v1")) { + return v1; + } + else if (mode.equalsIgnoreCase("v2")) { + return v2; + } + else if(mode.equalsIgnoreCase("dualWrite")) { + return new DualWriteHbaseTraceDao(v1, v2); + } + + return v1; + } + + @Override + public Class getObjectType() { + return TraceDao.class; + } + + @Override + public boolean isSingleton() { + return true; + } +} diff --git a/collector/src/main/java/com/navercorp/pinpoint/collector/dao/hbase/HbaseTraceDaoV2.java b/collector/src/main/java/com/navercorp/pinpoint/collector/dao/hbase/HbaseTraceDaoV2.java new file mode 100644 index 000000000..55f8f2994 --- /dev/null +++ b/collector/src/main/java/com/navercorp/pinpoint/collector/dao/hbase/HbaseTraceDaoV2.java @@ -0,0 +1,179 @@ +package com.navercorp.pinpoint.collector.dao.hbase; + +import com.navercorp.pinpoint.collector.dao.TraceDao; +import com.navercorp.pinpoint.collector.dao.hbase.filter.SpanEventFilter; +import com.navercorp.pinpoint.common.hbase.HbaseOperations2; +import com.navercorp.pinpoint.common.server.bo.SpanBo; +import com.navercorp.pinpoint.common.server.bo.SpanChunkBo; +import com.navercorp.pinpoint.common.server.bo.SpanEventBo; +import com.navercorp.pinpoint.common.server.bo.serializer.trace.v2.SpanChunkSerializerV2; +import com.navercorp.pinpoint.common.server.bo.serializer.trace.v2.SpanSerializerV2; +import com.navercorp.pinpoint.common.server.util.AcceptedTimeService; +import com.navercorp.pinpoint.common.util.SpanUtils; +import com.navercorp.pinpoint.common.util.TransactionId; +import com.navercorp.pinpoint.common.util.TransactionIdUtils; +import com.navercorp.pinpoint.thrift.dto.TSpan; +import com.navercorp.pinpoint.thrift.dto.TSpanChunk; +import com.navercorp.pinpoint.thrift.dto.TSpanEvent; +import com.sematext.hbase.wd.AbstractRowKeyDistributor; +import org.apache.commons.collections.CollectionUtils; +import org.apache.hadoop.hbase.client.Put; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Repository; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static com.navercorp.pinpoint.common.hbase.HBaseTables.TRACE_V2; + +/** + * @author Woonduk Kang(emeroad) + */ +@Repository +public class HbaseTraceDaoV2 implements TraceDao { + + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + + @Autowired + private HbaseOperations2 hbaseTemplate; + + @Autowired + private AcceptedTimeService acceptedTimeService; + + @Autowired + private SpanEventFilter spanEventFilter; + + @Autowired + private SpanSerializerV2 spanSerializer; + + @Autowired + private SpanChunkSerializerV2 spanChunkSerializer; + + @Autowired + @Qualifier("traceV2Distributor") + private AbstractRowKeyDistributor rowKeyDistributor; + + + @Override + public void insert(final TSpan span) { + if (span == null) { + throw new NullPointerException("span must not be null"); + } + + final SpanBo spanBo = new SpanBo(span); + List spanEventBoList = buildSpanEventList(span); + spanBo.addSpanEventBoList(spanEventBoList); + + long acceptedTime = acceptedTimeService.getAcceptedTime(); + spanBo.setCollectorAcceptTime(acceptedTime); + + final byte[] rowKey = getDistributeRowKey(SpanUtils.getTransactionId(span)); + final Put put = new Put(rowKey, acceptedTime); + + this.spanSerializer.serialize(spanBo, put, null); + + + boolean success = hbaseTemplate.asyncPut(TRACE_V2, put); + if (!success) { + hbaseTemplate.put(TRACE_V2, put); + } + + } + + private List buildSpanEventList(TSpan span) { + final List spanEventList = span.getSpanEventList(); + if (CollectionUtils.isEmpty(spanEventList)) { + return Collections.emptyList(); + } + + List spanEventBoList = new ArrayList<>(spanEventList.size()); + for (TSpanEvent spanEvent : spanEventList) { + final SpanEventBo spanEventBo = new SpanEventBo(span, spanEvent); + if (!spanEventFilter.filter(spanEventBo)) { + continue; + } + spanEventBoList.add(spanEventBo); + } + + + return spanEventBoList; + } + + private List buildSpanEventBoList(TSpanChunk tSpanChunk) { + List spanEventList = tSpanChunk.getSpanEventList(); + if (CollectionUtils.isEmpty(spanEventList)) { + return new ArrayList<>(); + } + List spanEventBoList = new ArrayList<>(spanEventList.size()); + for (TSpanEvent tSpanEvent : spanEventList) { + SpanEventBo spanEventBo = new SpanEventBo(tSpanChunk, tSpanEvent); + if (!spanEventFilter.filter(spanEventBo)) { + continue; + } + spanEventBoList.add(spanEventBo); + } + + return spanEventBoList; + } + + private byte[] getDistributeRowKey(byte[] transactionId) { + byte[] distributedKey = rowKeyDistributor.getDistributedKey(transactionId); + return distributedKey; + } + + + + @Override + public void insertSpanChunk(TSpanChunk spanChunk) { + SpanChunkBo spanChunkBo = buildSpanChunkBo(spanChunk); + + final byte[] rowKey = getDistributeRowKey(SpanUtils.getTransactionId(spanChunk)); + final long acceptedTime = acceptedTimeService.getAcceptedTime(); + final Put put = new Put(rowKey, acceptedTime); + + final List spanEventBoList = spanChunk.getSpanEventList(); + if (CollectionUtils.isEmpty(spanEventBoList)) { + return; + } + + this.spanChunkSerializer.serialize(spanChunkBo, put, null); + + if (!put.isEmpty()) { + boolean success = hbaseTemplate.asyncPut(TRACE_V2, put); + if (!success) { + hbaseTemplate.put(TRACE_V2, put); + } + } + } + + public SpanChunkBo buildSpanChunkBo(TSpanChunk tSpanChunk) { + SpanChunkBo spanChunkBo = new SpanChunkBo(); + spanChunkBo.setAgentId(tSpanChunk.getAgentId()); + spanChunkBo.setApplicationId(tSpanChunk.getApplicationName()); + spanChunkBo.setAgentStartTime(tSpanChunk.getAgentStartTime()); + + final TransactionId transactionId = TransactionIdUtils.parseTransactionId(tSpanChunk.getTransactionId()); + final String traceAgentId = transactionId.getAgentId(); + if (traceAgentId == null) { + spanChunkBo.setTraceAgentId(spanChunkBo.getAgentId()); + } else { + spanChunkBo.setTraceAgentId(traceAgentId); + } + spanChunkBo.setTraceAgentStartTime(transactionId.getAgentStartTime()); + spanChunkBo.setTraceTransactionSequence(transactionId.getTransactionSequence()); + + spanChunkBo.setSpanId(tSpanChunk.getSpanId()); + + List spanEventBoList = buildSpanEventBoList(tSpanChunk); + spanChunkBo.addSpanEventBoList(spanEventBoList); + return spanChunkBo; + } + + + + +} diff --git a/collector/src/main/java/com/navercorp/pinpoint/collector/handler/SpanChunkHandler.java b/collector/src/main/java/com/navercorp/pinpoint/collector/handler/SpanChunkHandler.java index b85762ba3..a87a1c8a0 100644 --- a/collector/src/main/java/com/navercorp/pinpoint/collector/handler/SpanChunkHandler.java +++ b/collector/src/main/java/com/navercorp/pinpoint/collector/handler/SpanChunkHandler.java @@ -18,6 +18,7 @@ package com.navercorp.pinpoint.collector.handler; import java.util.List; +import com.navercorp.pinpoint.collector.dao.TraceDao; import com.navercorp.pinpoint.common.service.ServiceTypeRegistryService; import com.navercorp.pinpoint.common.trace.ServiceType; @@ -26,11 +27,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import com.navercorp.pinpoint.collector.dao.TracesDao; import com.navercorp.pinpoint.common.util.SpanEventUtils; import com.navercorp.pinpoint.thrift.dto.TSpanChunk; import com.navercorp.pinpoint.thrift.dto.TSpanEvent; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; /** @@ -42,7 +43,8 @@ public class SpanChunkHandler implements SimpleHandler { private final Logger logger = LoggerFactory.getLogger(getClass()); @Autowired - private TracesDao traceDao; + @Qualifier("hbaseTraceDaoFactory") + private TraceDao traceDao; @Autowired private StatisticsHandler statisticsHandler; @@ -69,7 +71,9 @@ public class SpanChunkHandler implements SimpleHandler { final ServiceType applicationServiceType = getApplicationServiceType(spanChunk); List spanEventList = spanChunk.getSpanEventList(); if (spanEventList != null) { - logger.debug("SpanChunk Size:{}", spanEventList.size()); + if (logger.isDebugEnabled()) { + logger.debug("SpanChunk Size:{}", spanEventList.size()); + } // TODO need to batch update later. for (TSpanEvent spanEvent : spanEventList) { final ServiceType spanEventType = registry.findServiceType(spanEvent.getServiceType()); diff --git a/collector/src/main/java/com/navercorp/pinpoint/collector/handler/SpanHandler.java b/collector/src/main/java/com/navercorp/pinpoint/collector/handler/SpanHandler.java index ba992d5aa..b7705c74d 100644 --- a/collector/src/main/java/com/navercorp/pinpoint/collector/handler/SpanHandler.java +++ b/collector/src/main/java/com/navercorp/pinpoint/collector/handler/SpanHandler.java @@ -18,6 +18,7 @@ package com.navercorp.pinpoint.collector.handler; import java.util.List; +import com.navercorp.pinpoint.collector.dao.TraceDao; import com.navercorp.pinpoint.common.service.ServiceTypeRegistryService; import com.navercorp.pinpoint.common.trace.ServiceType; @@ -29,11 +30,11 @@ import org.springframework.beans.factory.annotation.Autowired; import com.navercorp.pinpoint.collector.dao.ApplicationTraceIndexDao; import com.navercorp.pinpoint.collector.dao.HostApplicationMapDao; -import com.navercorp.pinpoint.collector.dao.TracesDao; import com.navercorp.pinpoint.common.util.SpanEventUtils; import com.navercorp.pinpoint.thrift.dto.TSpan; import com.navercorp.pinpoint.thrift.dto.TSpanEvent; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; /** @@ -46,7 +47,8 @@ public class SpanHandler implements SimpleHandler { private final Logger logger = LoggerFactory.getLogger(getClass()); @Autowired - private TracesDao traceDao; + @Qualifier("hbaseTraceDaoFactory") + private TraceDao traceDao; @Autowired private ApplicationTraceIndexDao applicationTraceIndexDao; diff --git a/collector/src/main/resources/applicationContext-hbase.xml b/collector/src/main/resources/applicationContext-hbase.xml index 1c8c0bb73..79b13e704 100644 --- a/collector/src/main/resources/applicationContext-hbase.xml +++ b/collector/src/main/resources/applicationContext-hbase.xml @@ -85,6 +85,18 @@ + + + + + + + + + + + + diff --git a/collector/src/main/resources/pinpoint-collector.properties b/collector/src/main/resources/pinpoint-collector.properties index a297836e3..15d87a683 100644 --- a/collector/src/main/resources/pinpoint-collector.properties +++ b/collector/src/main/resources/pinpoint-collector.properties @@ -73,3 +73,7 @@ cluster.listen.port= #collector.admin.api.jmx.active= collector.spanEvent.sequence.limit=10000 + +# span.binary format compatibility = v1 or v2 (WARNING : experimental feature) +# experimental feature span format v2 : https://github.com/naver/pinpoint/issues/1819 +collector.experimental.span.format.compatibility.version=v1 \ No newline at end of file diff --git a/commons-hbase/src/main/java/com/navercorp/pinpoint/common/hbase/HBaseTables.java b/commons-hbase/src/main/java/com/navercorp/pinpoint/common/hbase/HBaseTables.java index fb6840556..2a6005a6c 100644 --- a/commons-hbase/src/main/java/com/navercorp/pinpoint/common/hbase/HBaseTables.java +++ b/commons-hbase/src/main/java/com/navercorp/pinpoint/common/hbase/HBaseTables.java @@ -59,10 +59,17 @@ public final class HBaseTables { public static final byte[] AGENT_STAT_COL_ACTIVE_TRACE_HISTOGRAM = Bytes.toBytes("aH"); // qualifier for active trace histogram public static final int AGENT_STAT_ROW_DISTRIBUTE_SIZE = 1; // agent statistics hash size + @Deprecated public static final TableName TRACES = TableName.valueOf("Traces"); + @Deprecated public static final byte[] TRACES_CF_SPAN = Bytes.toBytes("S"); //Span + @Deprecated public static final byte[] TRACES_CF_ANNOTATION = Bytes.toBytes("A"); //Annotation - public static final byte[] TRACES_CF_TERMINALSPAN = Bytes.toBytes("T"); //TerminalSpan + @Deprecated + public static final byte[] TRACES_CF_TERMINALSPAN = Bytes.toBytes("T"); //SpanEvent + + public static final TableName TRACE_V2 = TableName.valueOf("TraceV2"); + public static final byte[] TRACE_V2_CF_SPAN = Bytes.toBytes("S"); //Span public static final TableName APPLICATION_INDEX = TableName.valueOf("ApplicationIndex"); public static final byte[] APPLICATION_INDEX_CF_AGENTS = Bytes.toBytes("Agents"); diff --git a/commons-server/pom.xml b/commons-server/pom.xml index e2a02c95e..858ee77f8 100644 --- a/commons-server/pom.xml +++ b/commons-server/pom.xml @@ -14,6 +14,7 @@ 1.7 ${env.JAVA_7_HOME} + java17 diff --git a/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/AnnotationBo.java b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/AnnotationBo.java index a72406d3e..2ee638b57 100644 --- a/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/AnnotationBo.java +++ b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/AnnotationBo.java @@ -52,10 +52,12 @@ public class AnnotationBo { this.byteValue = transcoder.encode(value, this.valueType); } + @Deprecated public long getSpanId() { return spanId; } + @Deprecated public void setSpanId(long spanId) { this.spanId = spanId; } @@ -136,20 +138,7 @@ public class AnnotationBo { buffer.putPrefixedBytes(this.byteValue); } -// public int getBufferSize() { -// // int key; // required 4+string.length -// // int valueTypeCode; // required 4 -// // ByteBuffer value; // optional 4 + buf.length -// int size = 0; -// size += 1 + 4 + 4 + 4; -// size += 4; -// if (this.getByteValue() != null) { -// size += this.getByteValue().length; -// } -// return size; -// } - - + @Deprecated public void readValue(Buffer buffer) { this.version = buffer.readByte(); this.key = buffer.readSVInt(); diff --git a/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/SpanBo.java b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/SpanBo.java index a1dd3ecdd..733f8e1ca 100644 --- a/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/SpanBo.java +++ b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/SpanBo.java @@ -58,11 +58,11 @@ public class SpanBo implements Span { private String endPoint; private int apiId; - private List annotationBoList; + private List annotationBoList = new ArrayList<>(); private short flag; // optional private int errCode; - private List spanEventBoList; + private List spanEventBoList = new ArrayList<>(); private long collectorAcceptTime; @@ -251,7 +251,12 @@ public class SpanBo implements Span { return spanId; } + @Deprecated public void setSpanID(long spanId) { + this.setSpanId(spanId); + } + + public void setSpanId(long spanId) { this.spanId = spanId; } @@ -310,9 +315,17 @@ public class SpanBo implements Span { this.annotationBoList = anoList; } - public void addSpanEvent(SpanEventBo spanEventBo) { + public void addSpanEventBoList(List spanEventBoList) { if (spanEventBoList == null) { - spanEventBoList = new ArrayList<>(); + return; + } + this.spanEventBoList.addAll(spanEventBoList); + } + + + public void addSpanEvent(SpanEventBo spanEventBo) { + if (spanEventBo == null) { + return; } spanEventBoList.add(spanEventBo); } diff --git a/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/SpanChunkBo.java b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/SpanChunkBo.java new file mode 100644 index 000000000..077d4c270 --- /dev/null +++ b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/SpanChunkBo.java @@ -0,0 +1,130 @@ +package com.navercorp.pinpoint.common.server.bo; + +import com.navercorp.pinpoint.thrift.dto.TSpanEvent; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +/** + * @author Woonduk Kang(emeroad) + */ +public class SpanChunkBo { + + private byte version = 0; + + private String agentId; + private String applicationId; + private long agentStartTime; + + private String traceAgentId; + private long traceAgentStartTime; + private long traceTransactionSequence; + + private long spanId; + + private List spanEventBoList = new ArrayList<>(); + private long collectorAcceptTime; + + public SpanChunkBo() { + } + + public byte getVersion() { + return version; + } + + public void setVersion(byte version) { + this.version = version; + } + + public String getAgentId() { + return agentId; + } + + public void setAgentId(String agentId) { + this.agentId = agentId; + } + + public String getApplicationId() { + return applicationId; + } + + public void setApplicationId(String applicationId) { + this.applicationId = applicationId; + } + + public long getAgentStartTime() { + return agentStartTime; + } + + public void setAgentStartTime(long agentStartTime) { + this.agentStartTime = agentStartTime; + } + + public String getTraceAgentId() { + return traceAgentId; + } + + public void setTraceAgentId(String traceAgentId) { + this.traceAgentId = traceAgentId; + } + + public long getTraceAgentStartTime() { + return traceAgentStartTime; + } + + public void setTraceAgentStartTime(long traceAgentStartTime) { + this.traceAgentStartTime = traceAgentStartTime; + } + + public long getTraceTransactionSequence() { + return traceTransactionSequence; + } + + public void setTraceTransactionSequence(long traceTransactionSequence) { + this.traceTransactionSequence = traceTransactionSequence; + } + + public long getSpanId() { + return spanId; + } + + public void setSpanId(long spanId) { + this.spanId = spanId; + } + + public long getCollectorAcceptTime() { + return collectorAcceptTime; + } + + public void setCollectorAcceptTime(long collectorAcceptTime) { + this.collectorAcceptTime = collectorAcceptTime; + } + + public List getSpanEventBoList() { + return spanEventBoList; + } + + public void addSpanEventBoList(List spanEventBoList) { + if (spanEventBoList == null) { + return; + } + this.spanEventBoList.addAll(spanEventBoList); + } + + @Override + public String toString() { + return "SpanChunkBo{" + + "version=" + version + + ", agentId='" + agentId + '\'' + + ", applicationId='" + applicationId + '\'' + + ", agentStartTime=" + agentStartTime + + ", traceAgentId='" + traceAgentId + '\'' + + ", traceAgentStartTime=" + traceAgentStartTime + + ", traceTransactionSequence=" + traceTransactionSequence + + ", spanId=" + spanId + + ", spanEventBoList=" + spanEventBoList + + ", collectorAcceptTime=" + collectorAcceptTime + + '}'; + } +} diff --git a/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/SpanEventBo.java b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/SpanEventBo.java index 02f921d09..b5e102ccf 100644 --- a/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/SpanEventBo.java +++ b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/SpanEventBo.java @@ -119,7 +119,7 @@ public class SpanEventBo implements Span { this.nextSpanId = tSpanEvent.getNextSpanId(); } - setAnnotationList(tSpanEvent.getAnnotations()); + this.annotationBoList = buildAnnotationList(tSpanEvent.getAnnotations()); final TIntStringValue exceptionInfo = tSpanEvent.getExceptionInfo(); if (exceptionInfo != null) { @@ -183,7 +183,7 @@ public class SpanEventBo implements Span { this.nextSpanId = spanEvent.getNextSpanId(); } - setAnnotationList(spanEvent.getAnnotations()); + this.annotationBoList = buildAnnotationList(spanEvent.getAnnotations()); final TIntStringValue exceptionInfo = spanEvent.getExceptionInfo(); if (exceptionInfo != null) { @@ -356,22 +356,23 @@ public class SpanEventBo implements Span { this.nextSpanId = nextSpanId; } - public void setAnnotationList(List annotations) { - if (annotations == null) { - return; + private List buildAnnotationList(List annotationList) { + if (annotationList == null) { + return new ArrayList<>(); } - List boList = new ArrayList(annotations.size()); - for (TAnnotation ano : annotations) { - boList.add(new AnnotationBo(ano)); + List boList = new ArrayList(annotationList.size()); + for (TAnnotation annotation : annotationList) { + AnnotationBo annotationBo = new AnnotationBo(annotation); + boList.add(annotationBo); } - this.annotationBoList = boList; + return boList; } - public void setAnnotationBoList(List anoList) { - if (anoList == null) { + public void setAnnotationBoList(List annotationList) { + if (annotationList == null) { return; } - this.annotationBoList = anoList; + this.annotationBoList = annotationList; } public boolean isAsync() { @@ -574,6 +575,8 @@ public class SpanEventBo implements Span { builder.append(asyncId); builder.append(", nextAsyncId="); builder.append(nextAsyncId); + builder.append(", asyncSequence="); + builder.append(asyncSequence); builder.append("}"); return builder.toString(); } diff --git a/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/AnnotationSerializer.java b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v1/AnnotationSerializer.java similarity index 91% rename from commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/AnnotationSerializer.java rename to commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v1/AnnotationSerializer.java index d6590c0e5..fdc67a789 100644 --- a/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/AnnotationSerializer.java +++ b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v1/AnnotationSerializer.java @@ -1,9 +1,11 @@ -package com.navercorp.pinpoint.common.server.bo.serializer; +package com.navercorp.pinpoint.common.server.bo.serializer.trace.v1; import com.navercorp.pinpoint.common.buffer.AutomaticBuffer; import com.navercorp.pinpoint.common.buffer.Buffer; import com.navercorp.pinpoint.common.server.bo.AnnotationBo; import com.navercorp.pinpoint.common.server.bo.SpanBo; +import com.navercorp.pinpoint.common.server.bo.serializer.HbaseSerializer; +import com.navercorp.pinpoint.common.server.bo.serializer.SerializationContext; import org.apache.commons.collections.CollectionUtils; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.util.Bytes; diff --git a/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/SpanEventSerializer.java b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v1/SpanEventSerializer.java similarity index 93% rename from commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/SpanEventSerializer.java rename to commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v1/SpanEventSerializer.java index 7711a9730..58811f059 100644 --- a/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/SpanEventSerializer.java +++ b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v1/SpanEventSerializer.java @@ -1,10 +1,11 @@ -package com.navercorp.pinpoint.common.server.bo.serializer; +package com.navercorp.pinpoint.common.server.bo.serializer.trace.v1; import com.navercorp.pinpoint.common.buffer.AutomaticBuffer; import com.navercorp.pinpoint.common.buffer.Buffer; import com.navercorp.pinpoint.common.server.bo.AnnotationBo; import com.navercorp.pinpoint.common.server.bo.SpanEventBo; -import com.navercorp.pinpoint.common.server.util.AcceptedTimeService; +import com.navercorp.pinpoint.common.server.bo.serializer.HbaseSerializer; +import com.navercorp.pinpoint.common.server.bo.serializer.SerializationContext; import org.apache.hadoop.hbase.client.Put; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; diff --git a/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/SpanSerializer.java b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v1/SpanSerializer.java similarity index 93% rename from commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/SpanSerializer.java rename to commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v1/SpanSerializer.java index 6b75b1431..f56978ad9 100644 --- a/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/SpanSerializer.java +++ b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v1/SpanSerializer.java @@ -1,8 +1,10 @@ -package com.navercorp.pinpoint.common.server.bo.serializer; +package com.navercorp.pinpoint.common.server.bo.serializer.trace.v1; import com.navercorp.pinpoint.common.server.bo.SpanBo; import com.navercorp.pinpoint.common.buffer.AutomaticBuffer; import com.navercorp.pinpoint.common.buffer.Buffer; +import com.navercorp.pinpoint.common.server.bo.serializer.HbaseSerializer; +import com.navercorp.pinpoint.common.server.bo.serializer.SerializationContext; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.util.Bytes; import org.springframework.stereotype.Component; diff --git a/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/SpanChunkSerializerV2.java b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/SpanChunkSerializerV2.java new file mode 100644 index 000000000..9a642f141 --- /dev/null +++ b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/SpanChunkSerializerV2.java @@ -0,0 +1,34 @@ +package com.navercorp.pinpoint.common.server.bo.serializer.trace.v2; + +import com.navercorp.pinpoint.common.server.bo.SpanChunkBo; +import com.navercorp.pinpoint.common.server.bo.serializer.HbaseSerializer; +import com.navercorp.pinpoint.common.server.bo.serializer.SerializationContext; +import org.apache.hadoop.hbase.client.Put; +import org.springframework.stereotype.Component; + +import java.nio.ByteBuffer; + +import static com.navercorp.pinpoint.common.hbase.HBaseTables.TRACE_V2_CF_SPAN; + +/** + * @author Woonduk Kang(emeroad) + */ +@Component +public class SpanChunkSerializerV2 implements HbaseSerializer { + + private final SpanEncoder spanEncoder = new SpanEncoder(); + + @Override + public void serialize(SpanChunkBo spanChunkBo, Put put, SerializationContext context) { + + SpanEncodingContext encodingContext = new SpanEncodingContext<>(spanChunkBo); + + ByteBuffer qualifier = spanEncoder.encodeSpanChunkQualifier(encodingContext); + ByteBuffer columnValue = spanEncoder.encodeSpanChunkColumnValue(encodingContext); + + long acceptedTime = put.getTimeStamp(); + put.addColumn(TRACE_V2_CF_SPAN, qualifier, acceptedTime, columnValue); + + } + +} diff --git a/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/SpanDecoder.java b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/SpanDecoder.java new file mode 100644 index 000000000..d5f6181c0 --- /dev/null +++ b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/SpanDecoder.java @@ -0,0 +1,505 @@ +package com.navercorp.pinpoint.common.server.bo.serializer.trace.v2; + +import com.navercorp.pinpoint.common.buffer.Buffer; +import com.navercorp.pinpoint.common.server.bo.AnnotationBo; +import com.navercorp.pinpoint.common.server.bo.SpanBo; +import com.navercorp.pinpoint.common.server.bo.SpanChunkBo; +import com.navercorp.pinpoint.common.server.bo.SpanEventBo; +import com.navercorp.pinpoint.common.server.bo.serializer.trace.v2.bitfield.SpanBitFiled; +import com.navercorp.pinpoint.common.server.bo.serializer.trace.v2.bitfield.SpanEventBitField; +import com.navercorp.pinpoint.common.server.bo.serializer.trace.v2.bitfield.SpanEventQualifierBitField; +import com.navercorp.pinpoint.common.server.bo.serializer.trace.v2.bitfield.StartElapsedTimeEncodingStrategy; +import com.navercorp.pinpoint.common.util.AnnotationTranscoder; +import com.navercorp.pinpoint.common.util.TransactionId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author Woonduk Kang(emeroad) + */ +public class SpanDecoder { + + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + + private static final AnnotationTranscoder transcoder = new AnnotationTranscoder(); + + public static final Object UNKNOWN = new Object(); + + public void decode(Buffer qualifier, Buffer columnValue, SpanDecodingContext decodingContext, List out) { + final byte type = qualifier.readByte(); + + if (SpanEncoder.TYPE_SPAN == type) { + + SpanBo span = readSpan(qualifier, columnValue, decodingContext); + out.add(span); + + } else if (SpanEncoder.TYPE_SPAN_CHUNK == type) { + + SpanChunkBo spanChunk = readSpanChunk(qualifier, columnValue, decodingContext); + out.add(spanChunk); + + } else { + logger.warn("Unknown span type {}", type); + out.add(UNKNOWN); + } + } + + private SpanChunkBo readSpanChunk(Buffer qualifier, Buffer columnValue, SpanDecodingContext decodingContext) { + final SpanChunkBo spanChunk = new SpanChunkBo(); + + final TransactionId transactionId = decodingContext.getTransactionId(); + + spanChunk.setTraceAgentStartTime(transactionId.getAgentStartTime()); + spanChunk.setTraceTransactionSequence(transactionId.getTransactionSequence()); + spanChunk.setCollectorAcceptTime(decodingContext.getCollectorAcceptedTime()); + + + SpanAdaptor spanAdaptor = new SpanChunkBoAdaptor(spanChunk); + SpanEventBo firstSpanEvent = readQualifier(spanAdaptor, qualifier); + + readSpanChunkValue(columnValue, spanChunk, firstSpanEvent, decodingContext); + + return spanChunk; + } + + + private SpanBo readSpan(Buffer qualifier, Buffer columnValue, SpanDecodingContext decodingContext) { + final SpanBo span = new SpanBo(); + + final TransactionId transactionId = decodingContext.getTransactionId(); + span.setTraceAgentId(transactionId.getAgentId()); + span.setTraceAgentStartTime(transactionId.getAgentStartTime()); + span.setTraceTransactionSequence(transactionId.getTransactionSequence()); + span.setCollectorAcceptTime(decodingContext.getCollectorAcceptedTime()); + + SpanAdaptor spanAdaptor = new SpanBoAdaptor(span); + SpanEventBo firstSpanEvent = readQualifier(spanAdaptor, qualifier); + + readSpanValue(columnValue, span, firstSpanEvent, decodingContext); + + return span; + } + + private void readSpanChunkValue(Buffer buffer, SpanChunkBo spanChunk, SpanEventBo firstSpanEvent, SpanDecodingContext decodingContext) { + final byte version = buffer.readByte(); + if (version != 0) { + throw new IllegalStateException("unknown version :" + version); + } + spanChunk.setVersion(version); + + List spanEventBoList = readSpanEvent(buffer, firstSpanEvent, decodingContext); + spanChunk.addSpanEventBoList(spanEventBoList); + } + + public void readSpanValue(Buffer buffer, SpanBo span, SpanEventBo firstSpanEvent, SpanDecodingContext decodingContext) { + + final byte version = buffer.readByte(); + if (version != 0) { + throw new IllegalStateException("unknown version :" + version); + } + span.setVersion(version); + + final SpanBitFiled bitFiled = new SpanBitFiled(buffer.readByte()); + + final short serviceType = buffer.readShort(); + span.setServiceType(serviceType); + + switch (bitFiled.getApplicationServiceTypeEncodingStrategy()) { + case PREV_EQUALS: + span.setApplicationServiceType(serviceType); + break; + case RAW: + span.setApplicationServiceType(buffer.readShort()); + break; + default: + throw new IllegalStateException("applicationServiceType"); + } + + if (!bitFiled.isRoot()) { + span.setParentSpanId(buffer.readLong()); + } else { + span.setParentSpanId(-1); + } + + final int startTimeDelta = buffer.readVInt(); + final long startTime = startTimeDelta + span.getCollectorAcceptTime(); + span.setStartTime(startTime); + span.setElapsed(buffer.readVInt()); + + span.setRpc(buffer.readPrefixedString()); + + span.setEndPoint(buffer.readPrefixedString()); + span.setRemoteAddr(buffer.readPrefixedString()); + span.setApiId(buffer.readSVInt()); + + if (bitFiled.isSetErrorCode()) { + span.setErrCode(buffer.readInt()); + } + if (bitFiled.isSetHasException()) { + int exceptionId = buffer.readSVInt(); + String exceptionMessage = buffer.readPrefixedString(); + span.setExceptionInfo(exceptionId, exceptionMessage); + } + + if (bitFiled.isSetFlag()) { + span.setFlag(buffer.readShort()); + } + + if (bitFiled.isSetLoggingTransactionInfo()) { + span.setLoggingTransactionInfo(buffer.readByte()); + } + + span.setAcceptorHost(buffer.readPrefixedString()); + + + if (bitFiled.isSetAnnotation()) { + List annotationBoList = readAnnotationList(buffer, decodingContext); + span.setAnnotationBoList(annotationBoList); + } + + List spanEventBoList = readSpanEvent(buffer, firstSpanEvent, decodingContext); + span.addSpanEventBoList(spanEventBoList); + + + } + + private List readSpanEvent(Buffer buffer, SpanEventBo firstSpanEvent, SpanDecodingContext decodingContext) { + final int spanEventSize = buffer.readVInt(); + if (spanEventSize <= 0) { + return new ArrayList<>(); + } + final List spanEventBoList = new ArrayList<>(); + SpanEventBo prev = null; + for (int i = 0; i < spanEventSize; i++) { + SpanEventBo spanEvent; + if (i == 0) { + spanEvent = readFirstSpanEvent(buffer, firstSpanEvent, decodingContext); + } else { + spanEvent = readNextSpanEvent(buffer, prev, decodingContext); + } + prev = spanEvent; + spanEventBoList.add(spanEvent); + } + + return spanEventBoList; + } + + private SpanEventBo readNextSpanEvent(final Buffer buffer, final SpanEventBo prev, SpanDecodingContext decodingContext) { + final SpanEventBo spanEventBo = new SpanEventBo(); + + final SpanEventBitField bitField = new SpanEventBitField(buffer.readShort()); + + switch (bitField.getStartElapsedEncodingStrategy()) { + case PREV_DELTA: + int startTimeDelta = buffer.readVInt(); + int startTime = startTimeDelta + prev.getStartElapsed(); + spanEventBo.setStartElapsed(startTime); + break; + case PREV_EQUALS: + spanEventBo.setStartElapsed(prev.getStartElapsed()); + break; + default: + throw new IllegalStateException("unsupported SequenceEncodingStrategy"); + } + spanEventBo.setEndElapsed(buffer.readVInt()); + + switch (bitField.getSequenceEncodingStrategy()) { + case PREV_DELTA: + int sequenceDelta = buffer.readVInt(); + final int sequence = sequenceDelta + prev.getSequence(); + spanEventBo.setSequence((short) sequence); + break; + case PREV_ADD1: + spanEventBo.setSequence((short) (prev.getSequence() + 1)); + break; + default: + throw new IllegalStateException("unsupported SequenceEncodingStrategy"); + } + + switch (bitField.getDepthEncodingStrategy()) { + case RAW: + spanEventBo.setDepth(buffer.readSVInt()); + break; + case PREV_EQUALS: + spanEventBo.setDepth(prev.getDepth()); + break; + default: + throw new IllegalStateException("unsupported DepthEncodingStrategy"); + } + + switch (bitField.getServiceTypeEncodingStrategy()) { + case RAW: + spanEventBo.setServiceType(buffer.readShort()); + break; + case PREV_EQUALS: + spanEventBo.setServiceType(prev.getServiceType()); + break; + default: + throw new IllegalStateException("unsupported ServiceTypeEncodingStrategy"); + } + + + spanEventBo.setApiId(buffer.readSVInt()); + + if (bitField.isSetRpc()) { + spanEventBo.setRpc(buffer.readPrefixedString()); + } + + if (bitField.isSetEndPoint()) { + spanEventBo.setEndPoint(buffer.readPrefixedString()); + } + if (bitField.isSetDestinationId()) { + spanEventBo.setDestinationId(buffer.readPrefixedString()); + } + + if (bitField.isSetNextSpanId()) { + spanEventBo.setNextSpanId(buffer.readLong()); + } + + + if (bitField.isSetHasException()) { + int exceptionId = buffer.readSVInt(); + String exceptionMessage = buffer.readPrefixedString(); + spanEventBo.setExceptionInfo(exceptionId, exceptionMessage); + } + + if (bitField.isSetAnnotation()) { + List annotationBoList = readAnnotationList(buffer, decodingContext); + spanEventBo.setAnnotationBoList(annotationBoList); + } + + if (bitField.isSetNextAsyncId()) { + spanEventBo.setNextAsyncId(buffer.readSVInt()); + } + + if (bitField.isSetAsyncId()) { + spanEventBo.setAsyncId(buffer.readInt()); + spanEventBo.setAsyncSequence((short) buffer.readVInt()); + } + + return spanEventBo; + } + + private SpanEventBo readFirstSpanEvent(Buffer buffer, SpanEventBo firstSpanEvent, SpanDecodingContext decodingContext) { + SpanEventBitField bitField = new SpanEventBitField(buffer.readByte()); + + firstSpanEvent.setStartElapsed(buffer.readVInt()); + firstSpanEvent.setEndElapsed(buffer.readVInt()); + + firstSpanEvent.setSequence(buffer.readShort()); + firstSpanEvent.setDepth(buffer.readSVInt()); + firstSpanEvent.setServiceType(buffer.readShort()); + + if (bitField.isSetRpc()) { + firstSpanEvent.setRpc(buffer.readPrefixedString()); + } + + if (bitField.isSetEndPoint()) { + firstSpanEvent.setEndPoint(buffer.readPrefixedString()); + } + if (bitField.isSetDestinationId()) { + firstSpanEvent.setDestinationId(buffer.readPrefixedString()); + } + + firstSpanEvent.setApiId(buffer.readSVInt()); + + if (bitField.isSetNextSpanId()) { + firstSpanEvent.setNextSpanId(buffer.readLong()); + } + + if (bitField.isSetHasException()) { + int exceptionId = buffer.readSVInt(); + String exceptionMessage = buffer.readPrefixedString(); + firstSpanEvent.setExceptionInfo(exceptionId, exceptionMessage); + } + + if (bitField.isSetAnnotation()) { + List annotationBoList = readAnnotationList(buffer, decodingContext); + firstSpanEvent.setAnnotationBoList(annotationBoList); + } + + if (bitField.isSetNextAsyncId()) { + firstSpanEvent.setNextAsyncId(buffer.readSVInt()); + } + +// if (bitField.isSetAsyncId()) { +// firstSpanEvent.setAsyncId(buffer.readInt()); +// firstSpanEvent.setAsyncSequence((short) buffer.readVInt()); +// } + return firstSpanEvent; + } + + private List readAnnotationList(Buffer buffer, SpanDecodingContext decodingContext) { + int annotationListSize = buffer.readVInt(); + List annotationBoList = new ArrayList<>(annotationListSize); + +// AnnotationBo prev = decodingContext.getPrevFirstAnnotationBo(); + AnnotationBo prev = null; + for (int i = 0; i < annotationListSize; i++) { + AnnotationBo current; + if (i == 0) { + current = readFirstAnnotationBo(buffer); + // save first annotation for delta bitfield +// decodingContext.setPrevFirstAnnotationBo(current); + } else { + current = readDeltaAnnotationBo(buffer, prev); + } + + prev = current; + annotationBoList.add(current); + } + return annotationBoList; + } + + private AnnotationBo readFirstAnnotationBo(Buffer buffer) { + AnnotationBo current; + current = new AnnotationBo(); + current.setKey(buffer.readSVInt()); + + byte valueType = buffer.readByte(); + byte[] valueBytes = buffer.readPrefixedBytes(); + Object value = transcoder.decode(valueType, valueBytes); + + current.setValueType(valueType); + current.setValue(value); + return current; + } + + private AnnotationBo readDeltaAnnotationBo(Buffer buffer, AnnotationBo prev) { + AnnotationBo annotation = new AnnotationBo(); + + final int prevKey = prev.getKey(); + + annotation.setKey(buffer.readSVInt() + prevKey); + + byte valueType = buffer.readByte(); + byte[] valueBytes = buffer.readPrefixedBytes(); + Object value = transcoder.decode(valueType, valueBytes); + annotation.setValueType(valueType); + annotation.setValue(value); + return annotation; + } + + + private SpanEventBo readQualifier(SpanAdaptor span, Buffer buffer) { + String applicationId = buffer.readPrefixedString(); + span.setApplicationId(applicationId); + + String agentId = buffer.readPrefixedString(); + span.setAgentId(agentId); + + long agentStartTime = buffer.readVLong(); + span.setAgentStartTime(agentStartTime); + + long spanId = buffer.readLong(); + span.setSpanId(spanId); + + int firstSpanEventSequence = buffer.readSVInt(); + if (firstSpanEventSequence == -1) { +// buffer.readByte(); + // spanEvent not exist ?? + logger.info("firstSpanEvent is null. bug!!!!"); + return null; + } else { + return readQualifierFirstSpanEvent(buffer); + } + } + + private SpanEventBo readQualifierFirstSpanEvent(Buffer buffer) { + final SpanEventBo firstSpanEvent = new SpanEventBo(); + + final byte bitField = buffer.readByte(); + if (SpanEventQualifierBitField.isSetAsync(bitField)) { + int asyncId = buffer.readInt(); + int asyncSequence = buffer.readVInt(); + firstSpanEvent.setAsyncId(asyncId); + firstSpanEvent.setAsyncSequence((short) asyncSequence); + } + return firstSpanEvent; + } + + public void next(SpanDecodingContext decodingContext) { + decodingContext.next(); + } + + public void finish(SpanDecodingContext decodingContext) { + decodingContext.finish(); + } + + + // resolve type miss match + private interface SpanAdaptor { + void setApplicationId(String applicationId); + + void setAgentId(String agentId); + + void setAgentStartTime(long agentStartTime); + + void setSpanId(long spanId); + } + + private static class SpanBoAdaptor implements SpanAdaptor { + private SpanBo spanBo; + + private SpanBoAdaptor(SpanBo spanBo) { + if (spanBo == null) { + throw new NullPointerException("spanBo must not be null"); + } + this.spanBo = spanBo; + } + + @Override + public void setApplicationId(String applicationId) { + this.spanBo.setApplicationId(applicationId); + } + + @Override + public void setAgentId(String agentId) { + this.spanBo.setAgentId(agentId); + } + + @Override + public void setAgentStartTime(long agentStartTime) { + this.spanBo.setAgentStartTime(agentStartTime); + } + + @Override + public void setSpanId(long spanId) { + this.spanBo.setSpanId(spanId); + } + } + + private static class SpanChunkBoAdaptor implements SpanAdaptor { + private SpanChunkBo spanChunkBo; + + private SpanChunkBoAdaptor(SpanChunkBo spanChunkBo) { + if (spanChunkBo == null) { + throw new NullPointerException("spanChunkBo must not be null"); + } + this.spanChunkBo = spanChunkBo; + } + + @Override + public void setApplicationId(String applicationId) { + this.spanChunkBo.setApplicationId(applicationId); + } + + @Override + public void setAgentId(String agentId) { + this.spanChunkBo.setAgentId(agentId); + } + + @Override + public void setAgentStartTime(long agentStartTime) { + this.spanChunkBo.setAgentStartTime(agentStartTime); + } + + @Override + public void setSpanId(long spanId) { + this.spanChunkBo.setSpanId(spanId); + } + } +} diff --git a/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/SpanDecodingContext.java b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/SpanDecodingContext.java new file mode 100644 index 000000000..39270dd5f --- /dev/null +++ b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/SpanDecodingContext.java @@ -0,0 +1,48 @@ +package com.navercorp.pinpoint.common.server.bo.serializer.trace.v2; + +import com.navercorp.pinpoint.common.util.TransactionId; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author Woonduk Kang(emeroad) + */ +public class SpanDecodingContext { + +// private AnnotationBo prevAnnotationBo; + private long collectorAcceptedTime; + private TransactionId transactionId; + +// public AnnotationBo getPrevFirstAnnotationBo() { +// return prevAnnotationBo; +// } +// +// public void setPrevFirstAnnotationBo(AnnotationBo prevAnnotationBo) { +// this.prevAnnotationBo = prevAnnotationBo; +// } + + + public void setCollectorAcceptedTime(long collectorAcceptedTime) { + this.collectorAcceptedTime = collectorAcceptedTime; + } + + public long getCollectorAcceptedTime() { + return collectorAcceptedTime; + } + + public void setTransactionId(TransactionId transactionId) { + this.transactionId = transactionId; + } + + public TransactionId getTransactionId() { + return transactionId; + } + + + public void next() { + } + + public void finish() { + } +} diff --git a/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/SpanEncoder.java b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/SpanEncoder.java new file mode 100644 index 000000000..0fd907964 --- /dev/null +++ b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/SpanEncoder.java @@ -0,0 +1,428 @@ +package com.navercorp.pinpoint.common.server.bo.serializer.trace.v2; + +import com.navercorp.pinpoint.common.buffer.AutomaticBuffer; +import com.navercorp.pinpoint.common.buffer.Buffer; +import com.navercorp.pinpoint.common.server.bo.AnnotationBo; +import com.navercorp.pinpoint.common.server.bo.SpanBo; +import com.navercorp.pinpoint.common.server.bo.SpanChunkBo; +import com.navercorp.pinpoint.common.server.bo.SpanEventBo; +import com.navercorp.pinpoint.common.server.bo.serializer.trace.v2.bitfield.SpanBitFiled; +import com.navercorp.pinpoint.common.server.bo.serializer.trace.v2.bitfield.SpanEventBitField; +import com.navercorp.pinpoint.common.server.bo.serializer.trace.v2.bitfield.SpanEventQualifierBitField; +import org.apache.commons.collections.CollectionUtils; + +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +/** + * @author Woonduk Kang(emeroad) + */ +public class SpanEncoder { + + public static final Comparator SPAN_EVENT_SEQUENCE_COMPARATOR = new Comparator() { + @Override + public int compare(SpanEventBo o1, SpanEventBo o2) { + final int sequenceCompare = Short.compare(o1.getSequence(), o2.getSequence()); + if (sequenceCompare != 0) { + return sequenceCompare; + } + final int asyncId1 = o1.getAsyncId(); + final int asyncId2 = o2.getAsyncId(); + final int asyncIdCompare = Integer.compare(asyncId1, asyncId2); + if (asyncIdCompare != 0) { + if (asyncId1 == -1) { + return -1; + } + if (asyncId2 == -1) { + return -1; + } + return asyncIdCompare; + } + return Integer.compare(o1.getAsyncSequence(), o2.getAsyncSequence()); + } + }; + + public static final Comparator ANNOTATION_COMPARATOR = new Comparator() { + @Override + public int compare(AnnotationBo o1, AnnotationBo o2) { + return Integer.compare(o1.getKey(), o2.getKey()); + } + }; + public static byte TYPE_SPAN = 0; + public static byte TYPE_SPAN_CHUNK = 1; + + // reserved + public static byte TYPE_INDEX = 2; + + + public ByteBuffer encodeSpanQualifier(SpanEncodingContext encodingCtx) { + final SpanBo spanBo = encodingCtx.getValue(); + final List spanEventBoList = spanBo.getSpanEventBoList(); + final SpanEventBo firstEvent = getFirstSpanEvent(spanEventBoList); + + return encodeQualifier(TYPE_SPAN, spanBo.getApplicationId(), spanBo.getAgentId(), spanBo.getAgentStartTime(), spanBo.getSpanId(), firstEvent); + } + + public ByteBuffer encodeSpanChunkQualifier(SpanEncodingContext encodingCtx) { + final SpanChunkBo spanChunkBo = encodingCtx.getValue(); + + final List spanEventBoList = spanChunkBo.getSpanEventBoList(); + final SpanEventBo firstEvent = getFirstSpanEvent(spanEventBoList); + + return encodeQualifier(TYPE_SPAN_CHUNK, spanChunkBo.getApplicationId(), spanChunkBo.getAgentId(), spanChunkBo.getAgentStartTime(), spanChunkBo.getSpanId(), firstEvent); + } + + private ByteBuffer encodeQualifier(byte type, String applicationId, String agentId, long agentStartTime, long spanId, SpanEventBo firstEvent) { + final Buffer buffer = new AutomaticBuffer(128); + buffer.putByte(type); + buffer.putPrefixedString(applicationId); + buffer.putPrefixedString(agentId); + buffer.putVLong(agentStartTime); + buffer.putLong(spanId); + + if (firstEvent != null) { + buffer.putSVInt(firstEvent.getSequence()); + + final byte bitField = SpanEventQualifierBitField.buildBitField(firstEvent); + buffer.putByte(bitField); + // case : async span + if (SpanEventQualifierBitField.isSetAsync(bitField)) { + buffer.putInt(firstEvent.getAsyncId()); + buffer.putVInt(firstEvent.getAsyncSequence()); + } + } else { + // simple trace case +// buffer.putSVInt((short) -1); + +// byte cfBitField = SpanEventQualifierBitField.setAsync((byte) 0, false); +// buffer.putByte(cfBitField); + } + + return buffer.wrapByteBuffer(); + } + + + + private SpanEventBo getFirstSpanEvent(List spanEventBoList) { + if (CollectionUtils.isEmpty(spanEventBoList)) { + return null; + } + // TODO duplicated sort + sortSpanEvent(spanEventBoList); + return spanEventBoList.get(0); + } + + public ByteBuffer encodeSpanChunkColumnValue(SpanEncodingContext encodingCtx) { + final SpanChunkBo spanChunkBo = encodingCtx.getValue(); + // TODO duplicated sort + sortSpanEvent(spanChunkBo.getSpanEventBoList()); + + final Buffer buffer = new AutomaticBuffer(256); + + final byte version = spanChunkBo.getVersion(); + buffer.putByte(version); + + + final List spanEventBoList = spanChunkBo.getSpanEventBoList(); + writeSpanEventList(buffer, spanEventBoList, encodingCtx); + + return buffer.wrapByteBuffer(); + } + + private void writeSpanEventList(Buffer buffer, List spanEventBoList, SpanEncodingContext encodingCtx) { + if (CollectionUtils.isEmpty(spanEventBoList)) { + buffer.putVInt(0); + } else { + buffer.putVInt(spanEventBoList.size()); + + SpanEventBo prevSpanEvent = null; + for (SpanEventBo spanEventBo : spanEventBoList) { + if (prevSpanEvent == null) { + writeFirstSpanEvent(buffer, spanEventBo, encodingCtx); + } else { + writeNextSpanEvent(buffer, spanEventBo, prevSpanEvent, encodingCtx); + } + prevSpanEvent = spanEventBo; + } + } + } + + public ByteBuffer encodeSpanColumnValue(SpanEncodingContext encodingCtx) { + final SpanBo span = encodingCtx.getValue(); + + sortSpanEvent(span.getSpanEventBoList()); + + final SpanBitFiled bitField = SpanBitFiled.build(span); + + final Buffer buffer = new AutomaticBuffer(256); + + final byte version = span.getRawVersion(); + buffer.putByte(version); + + // bit field + buffer.putByte(bitField.getBitField()); + + + final short serviceType = span.getServiceType(); + buffer.putShort(serviceType); + + switch (bitField.getApplicationServiceTypeEncodingStrategy()) { + case PREV_EQUALS: + break; + case RAW: + buffer.putShort(span.getApplicationServiceType()); + break; + default: + throw new IllegalStateException("applicationServiceType"); + } + + + // insert for rowkey + // buffer.put(spanID); + if (!bitField.isRoot()) { + buffer.putLong(span.getParentSpanId()); + } + + // prevSpanEvent coding + final long startTime = span.getStartTime(); + final long startTimeDelta = span.getCollectorAcceptTime() - startTime; + buffer.putVLong(startTimeDelta); + buffer.putVInt(span.getElapsed()); + + + buffer.putPrefixedString(span.getRpc()); + + buffer.putPrefixedString(span.getEndPoint()); + buffer.putPrefixedString(span.getRemoteAddr()); + buffer.putSVInt(span.getApiId()); + + + // BIT flag + if (bitField.isSetErrorCode()) { + buffer.putInt(span.getErrCode()); + } + + if (bitField.isSetHasException()) { + buffer.putSVInt(span.getExceptionId()); + buffer.putPrefixedString(span.getExceptionMessage()); + } + + if (bitField.isSetFlag()) { + buffer.putShort(span.getFlag()); + } + + + if (bitField.isSetLoggingTransactionInfo()) { + buffer.putByte(span.getLoggingTransactionInfo()); + } + + buffer.putPrefixedString(span.getAcceptorHost()); + + if (bitField.isSetAnnotation()) { + List annotationBoList = span.getAnnotationBoList(); + writeAnnotationList(buffer, annotationBoList, encodingCtx); + } + + final List spanEventBoList = span.getSpanEventBoList(); + writeSpanEventList(buffer, spanEventBoList, encodingCtx); + + return buffer.wrapByteBuffer(); + } + + private void sortSpanEvent(List spanEventBoList) { + + if (CollectionUtils.isEmpty(spanEventBoList)) { + return; + } + Collections.sort(spanEventBoList, SPAN_EVENT_SEQUENCE_COMPARATOR); + } + + public void writeFirstSpanEvent(Buffer buffer, SpanEventBo spanEventBo, SpanEncodingContext encodingCtx) { + + final SpanEventBitField bitField = SpanEventBitField.buildFirst(spanEventBo); + + final byte firstSpanBitField1 = (byte) bitField.getBitField(); + buffer.putByte(firstSpanBitField1); + + buffer.putVInt(spanEventBo.getStartElapsed()); + buffer.putVInt(spanEventBo.getEndElapsed()); + + buffer.putShort(spanEventBo.getSequence()); + buffer.putSVInt(spanEventBo.getDepth()); + buffer.putShort(spanEventBo.getServiceType()); + + + if (bitField.isSetRpc()) { + buffer.putPrefixedString(spanEventBo.getRpc()); + } + + if (bitField.isSetEndPoint()) { + buffer.putPrefixedString(spanEventBo.getEndPoint()); + } + if (bitField.isSetDestinationId()) { + buffer.putPrefixedString(spanEventBo.getDestinationId()); + } + + buffer.putSVInt(spanEventBo.getApiId()); + + if (bitField.isSetNextSpanId()) { + buffer.putLong(spanEventBo.getNextSpanId()); + } + + if (bitField.isSetHasException()) { + buffer.putSVInt(spanEventBo.getExceptionId()); + buffer.putPrefixedString(spanEventBo.getExceptionMessage()); + } + + if (bitField.isSetAnnotation()) { + final List annotationBoList = spanEventBo.getAnnotationBoList(); + writeAnnotationList(buffer, annotationBoList, encodingCtx); + } + + if (bitField.isSetNextAsyncId()) { + buffer.putSVInt(spanEventBo.getNextAsyncId()); + } + +// if (bitField.isSetAsyncId()) { +// buffer.putInt(spanEventBo.getAsyncId()); +// buffer.putVInt(spanEventBo.getAsyncSequence()); +// } + } + + public void writeNextSpanEvent(Buffer buffer, SpanEventBo spanEventBo, SpanEventBo prevSpanEvent, SpanEncodingContext encodingCtx) { + + final SpanEventBitField bitField = SpanEventBitField.build(spanEventBo, prevSpanEvent); + + buffer.putShort(bitField.getBitField()); + + switch (bitField.getStartElapsedEncodingStrategy()) { + case PREV_DELTA: + final int startTimeDelta = spanEventBo.getStartElapsed() - prevSpanEvent.getStartElapsed(); + buffer.putVInt(startTimeDelta); + break; + case PREV_EQUALS: + // skip bitfield + break; + default: + throw new IllegalStateException("unsupported SequenceEncodingStrategy"); + } + buffer.putVInt(spanEventBo.getEndElapsed()); + + switch (bitField.getSequenceEncodingStrategy()) { + case PREV_DELTA: + final int sequenceDelta = spanEventBo.getSequence() - prevSpanEvent.getSequence(); + buffer.putVInt(sequenceDelta); + break; + case PREV_ADD1: + // skip bitfield + break; + default: + throw new IllegalStateException("unsupported SequenceEncodingStrategy"); + } + + switch (bitField.getDepthEncodingStrategy()) { + case RAW: + buffer.putSVInt(spanEventBo.getDepth()); + break; + case PREV_EQUALS: + // skip bitfield + break; + default: + throw new IllegalStateException("unsupported DepthEncodingStrategy"); + } + + switch (bitField.getServiceTypeEncodingStrategy()) { + case RAW: + buffer.putShort(spanEventBo.getServiceType()); + break; + case PREV_EQUALS: + // skip bitfield + break; + default: + throw new IllegalStateException("unsupported ServiceTypeEncodingStrategy"); + } + + + buffer.putSVInt(spanEventBo.getApiId()); + + if (bitField.isSetRpc()) { + buffer.putPrefixedString(spanEventBo.getRpc()); + } + + if (bitField.isSetEndPoint()) { + buffer.putPrefixedString(spanEventBo.getEndPoint()); + } + if (bitField.isSetDestinationId()) { + buffer.putPrefixedString(spanEventBo.getDestinationId()); + } + + if (bitField.isSetNextSpanId()) { + buffer.putLong(spanEventBo.getNextSpanId()); + } + + if (bitField.isSetHasException()) { + buffer.putSVInt(spanEventBo.getExceptionId()); + buffer.putPrefixedString(spanEventBo.getExceptionMessage()); + } + + if (bitField.isSetAnnotation()) { + List annotationBoList = spanEventBo.getAnnotationBoList(); + writeAnnotationList(buffer, annotationBoList, encodingCtx); + } + + if (bitField.isSetNextAsyncId()) { + buffer.putSVInt(spanEventBo.getNextAsyncId()); + } + + if (bitField.isSetAsyncId()) { + buffer.putInt(spanEventBo.getAsyncId()); + buffer.putVInt(spanEventBo.getAsyncSequence()); + } + } + + private void writeAnnotationList(Buffer buffer, List annotationBoList, SpanEncodingContext encodingCtx) { + if (CollectionUtils.isEmpty(annotationBoList)) { + return; + } + Collections.sort(annotationBoList, ANNOTATION_COMPARATOR); + + buffer.putVInt(annotationBoList.size()); + +// AnnotationBo prev = encodingCtx.getPrevFirstAnnotationBo(); + AnnotationBo prev = null; + for (int i = 0; i < annotationBoList.size(); i++) { + final AnnotationBo current = annotationBoList.get(i); + // first row + if (i == 0) { + + // first annotation + buffer.putSVInt(current.getKey()); + buffer.putByte(current.getRawValueType()); + buffer.putPrefixedBytes(current.getByteValue()); +// else { +// writeDeltaAnnotationBo(buffer, prev, current); +// } + // save first annotation +// encodingCtx.setPrevFirstAnnotationBo(current); + } else { + writeDeltaAnnotationBo(buffer, prev, current); + } + prev = current; + } + } + + private void writeDeltaAnnotationBo(Buffer buffer, AnnotationBo prev, AnnotationBo current) { + // prev : -30 cur: -20 = -20 - - 30 = 10 + // prev : 20 cur: 100 = 100 - 20 = 80 + // prev : -40 cur: 1000 = 1000 + 40 = 10040 + final int prevKey = prev.getKey(); + final int currentKey = current.getKey(); + buffer.putSVInt(currentKey - prevKey); + buffer.putByte(current.getRawValueType()); + buffer.putPrefixedBytes(current.getByteValue()); + } + + +} diff --git a/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/SpanEncodingContext.java b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/SpanEncodingContext.java new file mode 100644 index 000000000..9689cd965 --- /dev/null +++ b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/SpanEncodingContext.java @@ -0,0 +1,30 @@ +package com.navercorp.pinpoint.common.server.bo.serializer.trace.v2; + +import com.navercorp.pinpoint.common.server.bo.AnnotationBo; +import com.navercorp.pinpoint.common.server.bo.SpanBo; + +/** + * @author Woonduk Kang(emeroad) + */ +public class SpanEncodingContext { + private T value; + + private AnnotationBo prevAnnotationBo; + + public SpanEncodingContext(T value) { + this.value = value; + } + + + public T getValue() { + return value; + } + +// public AnnotationBo getPrevFirstAnnotationBo() { +// return prevAnnotationBo; +// } +// +// public void setPrevFirstAnnotationBo(AnnotationBo prevAnnotationBo) { +// this.prevAnnotationBo = prevAnnotationBo; +// } +} diff --git a/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/SpanSerializerV2.java b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/SpanSerializerV2.java new file mode 100644 index 000000000..e8659120a --- /dev/null +++ b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/SpanSerializerV2.java @@ -0,0 +1,41 @@ +package com.navercorp.pinpoint.common.server.bo.serializer.trace.v2; + +import com.navercorp.pinpoint.common.server.bo.SpanBo; +import com.navercorp.pinpoint.common.server.bo.serializer.HbaseSerializer; +import com.navercorp.pinpoint.common.server.bo.serializer.SerializationContext; +import org.apache.hadoop.hbase.client.Put; +import org.springframework.stereotype.Component; + +import java.nio.ByteBuffer; + + +import static com.navercorp.pinpoint.common.hbase.HBaseTables.TRACE_V2_CF_SPAN; + +/** + * @author Woonduk Kang(emeroad) + */ +@Component +public class SpanSerializerV2 implements HbaseSerializer { + + + private final SpanEncoder spanEncoder = new SpanEncoder(); + + public SpanSerializerV2() { + } + + + @Override + public void serialize(SpanBo spanBo, Put put, SerializationContext context) { + + final SpanEncodingContext encodingContext = new SpanEncodingContext<>(spanBo); + + ByteBuffer qualifier = spanEncoder.encodeSpanQualifier(encodingContext); + ByteBuffer columnValue = spanEncoder.encodeSpanColumnValue(encodingContext); + + long acceptedTime = put.getTimeStamp(); + put.addColumn(TRACE_V2_CF_SPAN, qualifier, acceptedTime, columnValue); + } + + + +} diff --git a/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/DepthEncodingStrategy.java b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/DepthEncodingStrategy.java new file mode 100644 index 000000000..80db3c511 --- /dev/null +++ b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/DepthEncodingStrategy.java @@ -0,0 +1,16 @@ +package com.navercorp.pinpoint.common.server.bo.serializer.trace.v2.bitfield; + +/** + * @author Woonduk Kang(emeroad) + */ +public enum DepthEncodingStrategy { + // 1bit + PREV_EQUALS(0), + RAW(1); + + private final int code; + + DepthEncodingStrategy(int code) { + this.code = code; + } +} diff --git a/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/SequenceEncodingStrategy.java b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/SequenceEncodingStrategy.java new file mode 100644 index 000000000..7f2e9694f --- /dev/null +++ b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/SequenceEncodingStrategy.java @@ -0,0 +1,18 @@ +package com.navercorp.pinpoint.common.server.bo.serializer.trace.v2.bitfield; + +/** + * @author Woonduk Kang(emeroad) + */ +public enum SequenceEncodingStrategy { + // 1 bit + PREV_ADD1(0), + PREV_DELTA(1); + + private final int code; + + SequenceEncodingStrategy(int code) { + this.code = code; + } + + +} diff --git a/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/ServiceTypeEncodingStrategy.java b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/ServiceTypeEncodingStrategy.java new file mode 100644 index 000000000..2e4b7709a --- /dev/null +++ b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/ServiceTypeEncodingStrategy.java @@ -0,0 +1,18 @@ +package com.navercorp.pinpoint.common.server.bo.serializer.trace.v2.bitfield; + +/** + * @author Woonduk Kang(emeroad) + */ +public enum ServiceTypeEncodingStrategy { + + // 1 bit + PREV_EQUALS(0), + RAW(1); + + private final int code; + + ServiceTypeEncodingStrategy(int code) { + this.code = code; + } + +} diff --git a/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/SimpleServiceTypeEncodingStrategy.java b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/SimpleServiceTypeEncodingStrategy.java new file mode 100644 index 000000000..222ea3a99 --- /dev/null +++ b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/SimpleServiceTypeEncodingStrategy.java @@ -0,0 +1,21 @@ +package com.navercorp.pinpoint.common.server.bo.serializer.trace.v2.bitfield; + +/** + * @author Woonduk Kang(emeroad) + */ +public enum SimpleServiceTypeEncodingStrategy { + + // 1 bit + PREV_EQUALS(0), + RAW(1); + + private final int code; + + SimpleServiceTypeEncodingStrategy(int code) { + this.code = code; + } + + public int getCode() { + return code; + } +} diff --git a/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/SpanBitFiled.java b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/SpanBitFiled.java new file mode 100644 index 000000000..3a1d34437 --- /dev/null +++ b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/SpanBitFiled.java @@ -0,0 +1,178 @@ +package com.navercorp.pinpoint.common.server.bo.serializer.trace.v2.bitfield; + +import com.navercorp.pinpoint.common.server.bo.SpanBo; +import com.navercorp.pinpoint.common.trace.LoggingInfo; +import com.navercorp.pinpoint.common.util.BitFieldUtils; +import org.apache.commons.collections.CollectionUtils; + +/** + * @author Woonduk Kang(emeroad) + */ +public class SpanBitFiled { + + // 1bit + public static final int SET_APPLICATION_SERVICE_TYPE_ENCODING_STRATEGY = 0; + public static final int SET_ROOT = 1; + public static final int SET_ERROR_CODE = 2; + public static final int SET_HAS_EXCEPTION = 3; + public static final int SET_FLAG = 4; + public static final int SET_LOGGING_TRANSACTION_INFO = 5; + public static final int SET_ANNOTATION = 6; + + + private static final long ROOT_PARENT_SPAN_ID = -1; + // used : 7bit + // reserved : 1 bit + private byte bitField = 0; + + public static SpanBitFiled build(SpanBo spanBo) { + if (spanBo == null) { + throw new NullPointerException("spanBo must not be null"); + } + final SpanBitFiled spanBitFiled = new SpanBitFiled(); + + + if (spanBo.getServiceType() == spanBo.getApplicationServiceType()) { + spanBitFiled.setApplicationServiceTypeEncodingStrategy(SimpleServiceTypeEncodingStrategy.PREV_EQUALS); + } else { + spanBitFiled.setApplicationServiceTypeEncodingStrategy(SimpleServiceTypeEncodingStrategy.RAW); + } + + if (spanBo.getParentSpanId() == ROOT_PARENT_SPAN_ID) { + spanBitFiled.setRoot(true); + } + if (spanBo.getErrCode() != 0) { + spanBitFiled.setErrorCode(true); + } + + if (spanBo.hasException()) { + spanBitFiled.setHasException(true); + } + + if (spanBo.getFlag() != 0) { + spanBitFiled.setFlag(true); + } + + if (spanBo.getLoggingTransactionInfo() != LoggingInfo.NOT_LOGGED.getCode()) { + spanBitFiled.setLoggingTransactionInfo(true); + } + if (CollectionUtils.isNotEmpty(spanBo.getAnnotationBoList())) { + spanBitFiled.setAnnotation(true); + } + + return spanBitFiled; + } + + public SpanBitFiled() { + } + + public SpanBitFiled(byte bitField) { + this.bitField = bitField; + } + + public byte getBitField() { + return bitField; + } + + // for test + void maskAll() { + bitField = -1; + } + + private void setBit(int position, boolean value) { + this.bitField = BitFieldUtils.setBit(bitField, position, value); + } + + private boolean testBit(int position) { + return BitFieldUtils.testBit(bitField, position); + } + + private int getBit(int position) { + return BitFieldUtils.getBit(bitField, position); + } + + public SimpleServiceTypeEncodingStrategy getApplicationServiceTypeEncodingStrategy() { + final int set = getBit(SET_APPLICATION_SERVICE_TYPE_ENCODING_STRATEGY); + switch (set) { + case 0: + return SimpleServiceTypeEncodingStrategy.PREV_EQUALS; + case 1: + return SimpleServiceTypeEncodingStrategy.RAW; + default: + throw new IllegalArgumentException("SET_APPLICATION_SERVICE_TYPE_ENCODING_STRATEGY"); + } + } + + void setApplicationServiceTypeEncodingStrategy(SimpleServiceTypeEncodingStrategy strategy) { + switch (strategy) { + case PREV_EQUALS: + setBit(SET_APPLICATION_SERVICE_TYPE_ENCODING_STRATEGY, false); + break; + case RAW: + setBit(SET_APPLICATION_SERVICE_TYPE_ENCODING_STRATEGY, true); + break; + default: + throw new IllegalArgumentException("SET_APPLICATION_SERVICE_TYPE_ENCODING_STRATEGY"); + } + + } + + + public boolean isRoot() { + return testBit(SET_ROOT); + } + + // for test + void setRoot(boolean root) { + setBit(SET_ROOT, root); + } + + + public boolean isSetErrorCode() { + return testBit(SET_ERROR_CODE); + } + + + // for test + void setErrorCode(boolean errorCode) { + setBit(SET_ERROR_CODE, errorCode); + } + + public boolean isSetHasException() { + return testBit(SET_HAS_EXCEPTION); + } + + // for test + void setHasException(boolean hasException) { + setBit(SET_HAS_EXCEPTION, hasException); + } + + + public boolean isSetFlag() { + return testBit(SET_FLAG); + } + + // for test + void setFlag(boolean flag) { + setBit(SET_FLAG, flag); + } + + + public boolean isSetLoggingTransactionInfo() { + return testBit(SET_LOGGING_TRANSACTION_INFO); + } + + // for test + void setLoggingTransactionInfo(boolean loggingTransactionInfo) { + setBit(SET_LOGGING_TRANSACTION_INFO, loggingTransactionInfo); + } + + public boolean isSetAnnotation() { + return testBit(SET_ANNOTATION); + } + + + public void setAnnotation(boolean annotation) { + setBit(SET_ANNOTATION, annotation); + } +} diff --git a/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/SpanEventBitField.java b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/SpanEventBitField.java new file mode 100644 index 000000000..524cbcc01 --- /dev/null +++ b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/SpanEventBitField.java @@ -0,0 +1,323 @@ +package com.navercorp.pinpoint.common.server.bo.serializer.trace.v2.bitfield; + +import com.navercorp.pinpoint.common.server.bo.AnnotationBo; +import com.navercorp.pinpoint.common.server.bo.SpanEventBo; +import com.navercorp.pinpoint.common.util.BitFieldUtils; +import org.apache.commons.collections.CollectionUtils; + +import java.util.List; + +/** + * @author Woonduk Kang(emeroad) + */ +public class SpanEventBitField { + + + public static final int SET_ANNOTATION = 0; + public static final int SET_HAS_EXCEPTION = 1; + public static final int SET_NEXT_ASYNCID = 2; + public static final int SET_ASYNCID = 3; + public static final int SET_NEXT_SPANID = 4; + public static final int SET_ENDPOINT = 5; + public static final int SET_DESTINATIONID = 6; + public static final int SET_RPC = 7; +// firstSpan bitField ----------------------------------------------- + public static final int START_ELAPSED_ENCODING_STRATEGY = 8; + public static final int SERVICE_TYPE_ENCODING_STRATEGY = 9; + public static final int SEQUENCE_ENCODING_STRATEGY = 10; + public static final int DEPTH_ENCODING_STRATEGY = 11; + + public static final int API_ENCODING_STRATEGY = 13; + + + private short bitField = 0; + + + public static SpanEventBitField buildFirst(SpanEventBo spanEventBo) { + if (spanEventBo == null) { + throw new NullPointerException("spanEventBo must not be null"); + } + final SpanEventBitField bitFiled = new SpanEventBitField(); + + if (spanEventBo.getRpc() != null) { + bitFiled.setRpc(true); + } + if (spanEventBo.getEndPoint() != null) { + bitFiled.setEndPoint(true); + } + + if (spanEventBo.getDestinationId() != null) { + bitFiled.setDestinationId(true); + } + + if (spanEventBo.getNextSpanId() != -1) { + bitFiled.setNextSpanId(true); + } + + if (spanEventBo.hasException()) { + bitFiled.setHasException(true); + } + + final List annotationBoList = spanEventBo.getAnnotationBoList(); + if (CollectionUtils.isNotEmpty(annotationBoList)) { + bitFiled.setAnnotation(true); + } + + if (spanEventBo.getNextAsyncId() != -1) { + bitFiled.setNextAsyncId(true); + } + + + if (spanEventBo.getAsyncId() == -1 && spanEventBo.getAsyncSequence() == -1) { + bitFiled.setAsyncId(false); + } else { + bitFiled.setAsyncId(true); + } + return bitFiled; + } + + public static SpanEventBitField build(SpanEventBo spanEventBo, SpanEventBo prevSpanEventBo) { + if (spanEventBo == null) { + throw new NullPointerException("spanEventBo must not be null"); + } + if (prevSpanEventBo == null) { + throw new NullPointerException("prevSpanEventBo must not be null"); + } + + final SpanEventBitField bitFiled = buildFirst(spanEventBo); + + if (spanEventBo.getStartElapsed() == prevSpanEventBo.getStartElapsed()) { + bitFiled.setStartElapsedEncodingStrategy(StartElapsedTimeEncodingStrategy.PREV_EQUALS); + } else { + bitFiled.setStartElapsedEncodingStrategy(StartElapsedTimeEncodingStrategy.PREV_DELTA); + } + + // sequence prev: 5 current: 6 = 6 - 5= delta 1 + final short sequenceDelta = (short) (spanEventBo.getSequence() - prevSpanEventBo.getSequence()); + if (sequenceDelta == 1) { + bitFiled.setSequenceEncodingStrategy(SequenceEncodingStrategy.PREV_ADD1); + } else { + bitFiled.setSequenceEncodingStrategy(SequenceEncodingStrategy.PREV_DELTA); + } + + if (spanEventBo.getDepth() == prevSpanEventBo.getDepth()) { + bitFiled.setDepthEncodingStrategy(DepthEncodingStrategy.PREV_EQUALS); + } else { + bitFiled.setDepthEncodingStrategy(DepthEncodingStrategy.RAW); + } + + + if (prevSpanEventBo.getServiceType() == spanEventBo.getServiceType()) { + bitFiled.setServiceTypeEncodingStrategy(ServiceTypeEncodingStrategy.PREV_EQUALS); + } else { + bitFiled.setServiceTypeEncodingStrategy(ServiceTypeEncodingStrategy.RAW); + } + + + return bitFiled; + } + + public SpanEventBitField() { + } + + public SpanEventBitField(short bitField) { + this.bitField = bitField; + } + + // for test + void maskAll() { + bitField = -1; + } + + public short getBitField() { + return bitField; + } + + private void setBit(int position, boolean value) { + this.bitField = BitFieldUtils.setBit(bitField, position, value); + } + + private boolean testBit(int position) { + return BitFieldUtils.testBit(bitField, position); + } + + private int getBit(int position) { + return BitFieldUtils.getBit(bitField, position); + } + + + public boolean isSetHasException() { + return testBit(SET_HAS_EXCEPTION); + } + + void setHasException(boolean hasException) { + setBit(SET_HAS_EXCEPTION, hasException); + } + + + public boolean isSetAnnotation() { + return testBit(SET_ANNOTATION); + } + + void setAnnotation(boolean annotation) { + setBit(SET_ANNOTATION, annotation); + } + + + public boolean isSetNextAsyncId() { + return testBit(SET_NEXT_ASYNCID); + } + + void setNextAsyncId(boolean nextAsyncSpanId) { + setBit(SET_NEXT_ASYNCID, nextAsyncSpanId); + } + + + public boolean isSetNextSpanId() { + return testBit(SET_NEXT_SPANID); + } + + void setNextSpanId(boolean nextSpanId) { + setBit(SET_NEXT_SPANID, nextSpanId); + } + + public boolean isSetEndPoint() { + return testBit(SET_ENDPOINT); + } + + void setEndPoint(boolean endPoint) { + setBit(SET_ENDPOINT, endPoint); + } + + public boolean isSetDestinationId() { + return testBit(SET_DESTINATIONID); + } + + void setDestinationId(boolean destinationId) { + setBit(SET_DESTINATIONID, destinationId); + } + + + public boolean isSetRpc() { + return testBit(SET_RPC); + } + + void setRpc(boolean rpc) { + setBit(SET_RPC, rpc); + } + + public boolean isSetAsyncId() { + return testBit(SET_ASYNCID); + } + + void setAsyncId(boolean asyncId) { + setBit(SET_ASYNCID, asyncId); + } + + + + public StartElapsedTimeEncodingStrategy getStartElapsedEncodingStrategy() { + final int set = getBit(START_ELAPSED_ENCODING_STRATEGY); + switch (set) { + case 0: + return StartElapsedTimeEncodingStrategy.PREV_EQUALS; + case 1: + return StartElapsedTimeEncodingStrategy.PREV_DELTA; + default: + throw new IllegalArgumentException("SERVICE_TYPE_ENCODING_STRATEGY"); + } + } + + void setStartElapsedEncodingStrategy(StartElapsedTimeEncodingStrategy strategy) { + switch (strategy) { + case PREV_EQUALS: + setBit(START_ELAPSED_ENCODING_STRATEGY, false); + break; + case PREV_DELTA: + setBit(START_ELAPSED_ENCODING_STRATEGY, true); + break; + default: + throw new IllegalArgumentException("START_ELAPSED_ENCODING_STRATEGY"); + } + } + + + + public ServiceTypeEncodingStrategy getServiceTypeEncodingStrategy() { + final int set = getBit(SERVICE_TYPE_ENCODING_STRATEGY); + switch (set) { + case 0: + return ServiceTypeEncodingStrategy.PREV_EQUALS; + case 1: + return ServiceTypeEncodingStrategy.RAW; + default: + throw new IllegalArgumentException("SERVICE_TYPE_ENCODING_STRATEGY"); + } + } + + void setServiceTypeEncodingStrategy(ServiceTypeEncodingStrategy strategy) { + switch (strategy) { + case PREV_EQUALS: + setBit(SERVICE_TYPE_ENCODING_STRATEGY, false); + break; + case RAW: + setBit(SERVICE_TYPE_ENCODING_STRATEGY, true); + break; + default: + throw new IllegalArgumentException("SERVICE_TYPE_ENCODING_STRATEGY"); + } + } + + public SequenceEncodingStrategy getSequenceEncodingStrategy() { + final int set = getBit(SEQUENCE_ENCODING_STRATEGY); + switch (set) { + case 0: + return SequenceEncodingStrategy.PREV_ADD1; + case 1: + return SequenceEncodingStrategy.PREV_DELTA; + default: + throw new IllegalArgumentException("SEQUENCE_ENCODING_STRATEGY"); + } + } + + + void setSequenceEncodingStrategy(SequenceEncodingStrategy strategy) { + switch (strategy) { + case PREV_ADD1: + setBit(SEQUENCE_ENCODING_STRATEGY, false); + break; + case PREV_DELTA: + setBit(SEQUENCE_ENCODING_STRATEGY, true); + break; + default: + throw new IllegalArgumentException("SEQUENCE_ENCODING_STRATEGY"); + } + } + + public DepthEncodingStrategy getDepthEncodingStrategy() { + final int set = getBit(DEPTH_ENCODING_STRATEGY); + switch (set) { + case 0: + return DepthEncodingStrategy.PREV_EQUALS; + case 1: + return DepthEncodingStrategy.RAW; + default: + throw new IllegalArgumentException("DEPTH_ENCODING_STRATEGY"); + } + } + + void setDepthEncodingStrategy(DepthEncodingStrategy strategy) { + switch (strategy) { + case PREV_EQUALS: + setBit(DEPTH_ENCODING_STRATEGY, false); + break; + case RAW: + setBit(DEPTH_ENCODING_STRATEGY, true); + break; + default: + throw new IllegalArgumentException("SEQUENCE_ENCODING_STRATEGY"); + } + } + + +} diff --git a/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/SpanEventQualifierBitField.java b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/SpanEventQualifierBitField.java new file mode 100644 index 000000000..44a2ba575 --- /dev/null +++ b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/SpanEventQualifierBitField.java @@ -0,0 +1,45 @@ +package com.navercorp.pinpoint.common.server.bo.serializer.trace.v2.bitfield; + +import com.navercorp.pinpoint.common.server.bo.SpanEventBo; +import com.navercorp.pinpoint.common.util.BitFieldUtils; + +/** + * @author Woonduk Kang(emeroad) + */ +public class SpanEventQualifierBitField { + + public static final int SET_ASYNC = 0; + + public static byte buildBitField(SpanEventBo firstSpanEvent) { + if (firstSpanEvent == null) { + // no async bit field + return 0; + } + + byte bitField = 0; + + final int asyncId = firstSpanEvent.getAsyncId(); + final short asyncSequence = firstSpanEvent.getAsyncSequence(); + if (asyncId == -1 && asyncSequence == -1) { + bitField = setAsync(bitField, false); + } else { + bitField = setAsync(bitField, true); + } + + return bitField; + } + + private SpanEventQualifierBitField() { + } + + + public static boolean isSetAsync(byte bitField) { + return BitFieldUtils.testBit(bitField, SET_ASYNC); + } + + public static byte setAsync(byte bitField, boolean async) { + return BitFieldUtils.setBit(bitField, SET_ASYNC, async); + } + + +} diff --git a/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/StartElapsedTimeEncodingStrategy.java b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/StartElapsedTimeEncodingStrategy.java new file mode 100644 index 000000000..88724c90a --- /dev/null +++ b/commons-server/src/main/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/StartElapsedTimeEncodingStrategy.java @@ -0,0 +1,16 @@ +package com.navercorp.pinpoint.common.server.bo.serializer.trace.v2.bitfield; + +/** + * @author Woonduk Kang(emeroad) + */ +public enum StartElapsedTimeEncodingStrategy { + // 1 bit + PREV_EQUALS(0), + PREV_DELTA(1); + + private final int code; + + StartElapsedTimeEncodingStrategy(int code) { + this.code = code; + } +} diff --git a/commons-server/src/test/java/com/navercorp/pinpoint/common/server/bo/AnnotationBoTest.java b/commons-server/src/test/java/com/navercorp/pinpoint/common/server/bo/AnnotationBoTest.java index 67dbc32a3..a62d051f8 100644 --- a/commons-server/src/test/java/com/navercorp/pinpoint/common/server/bo/AnnotationBoTest.java +++ b/commons-server/src/test/java/com/navercorp/pinpoint/common/server/bo/AnnotationBoTest.java @@ -18,7 +18,7 @@ package com.navercorp.pinpoint.common.server.bo; import com.navercorp.pinpoint.common.buffer.AutomaticBuffer; import com.navercorp.pinpoint.common.buffer.Buffer; -import com.navercorp.pinpoint.common.server.bo.serializer.AnnotationSerializer; +import com.navercorp.pinpoint.common.server.bo.serializer.trace.v1.AnnotationSerializer; import com.navercorp.pinpoint.common.trace.AnnotationKey; import org.apache.commons.lang.RandomStringUtils; diff --git a/commons-server/src/test/java/com/navercorp/pinpoint/common/server/bo/SpanEventBoTest.java b/commons-server/src/test/java/com/navercorp/pinpoint/common/server/bo/SpanEventBoTest.java index d817b2019..a3130a638 100644 --- a/commons-server/src/test/java/com/navercorp/pinpoint/common/server/bo/SpanEventBoTest.java +++ b/commons-server/src/test/java/com/navercorp/pinpoint/common/server/bo/SpanEventBoTest.java @@ -16,8 +16,8 @@ package com.navercorp.pinpoint.common.server.bo; -import com.navercorp.pinpoint.common.server.bo.serializer.AnnotationSerializer; -import com.navercorp.pinpoint.common.server.bo.serializer.SpanEventSerializer; +import com.navercorp.pinpoint.common.server.bo.serializer.trace.v1.AnnotationSerializer; +import com.navercorp.pinpoint.common.server.bo.serializer.trace.v1.SpanEventSerializer; import com.navercorp.pinpoint.common.trace.ServiceType; import org.junit.Assert; diff --git a/commons-server/src/test/java/com/navercorp/pinpoint/common/server/bo/serializer/SpanBoTest.java b/commons-server/src/test/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v1/SpanBoTest.java similarity index 97% rename from commons-server/src/test/java/com/navercorp/pinpoint/common/server/bo/serializer/SpanBoTest.java rename to commons-server/src/test/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v1/SpanBoTest.java index b515c3c5c..7dc239b89 100644 --- a/commons-server/src/test/java/com/navercorp/pinpoint/common/server/bo/serializer/SpanBoTest.java +++ b/commons-server/src/test/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v1/SpanBoTest.java @@ -14,10 +14,11 @@ * limitations under the License. */ -package com.navercorp.pinpoint.common.server.bo.serializer; +package com.navercorp.pinpoint.common.server.bo.serializer.trace.v1; import com.navercorp.pinpoint.common.server.bo.SpanBo; +import com.navercorp.pinpoint.common.server.bo.serializer.trace.v1.SpanSerializer; import com.navercorp.pinpoint.common.trace.LoggingInfo; import org.apache.commons.lang3.RandomStringUtils; import org.junit.Assert; diff --git a/commons-server/src/test/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/SpanEncoderTest.java b/commons-server/src/test/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/SpanEncoderTest.java new file mode 100644 index 000000000..cd9635159 --- /dev/null +++ b/commons-server/src/test/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/SpanEncoderTest.java @@ -0,0 +1,20 @@ +package com.navercorp.pinpoint.common.server.bo.serializer.trace.v2; + +import com.navercorp.pinpoint.common.server.bo.SpanBo; +import org.junit.Test; + +import java.nio.ByteBuffer; + +import static org.junit.Assert.*; + +/** + * @author Woonduk Kang(emeroad) + */ +public class SpanEncoderTest { + @Test + public void encodeSpanColumnValue() throws Exception { + + + } + +} \ No newline at end of file diff --git a/commons-server/src/test/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/SpanSerializerV2Test.java b/commons-server/src/test/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/SpanSerializerV2Test.java new file mode 100644 index 000000000..d6e1b199c --- /dev/null +++ b/commons-server/src/test/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/SpanSerializerV2Test.java @@ -0,0 +1,26 @@ +package com.navercorp.pinpoint.common.server.bo.serializer.trace.v2; + +import com.navercorp.pinpoint.common.util.BytesUtils; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author Woonduk Kang(emeroad) + */ +public class SpanSerializerV2Test { + + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + + + + @Test + public void testEstimationDelta() { + + } + + + + + +} \ No newline at end of file diff --git a/commons-server/src/test/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/SpanBitFiledTest.java b/commons-server/src/test/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/SpanBitFiledTest.java new file mode 100644 index 000000000..85b946147 --- /dev/null +++ b/commons-server/src/test/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/SpanBitFiledTest.java @@ -0,0 +1,151 @@ +package com.navercorp.pinpoint.common.server.bo.serializer.trace.v2.bitfield; + +import com.navercorp.pinpoint.common.server.bo.SpanBo; +import com.navercorp.pinpoint.common.server.bo.serializer.trace.v2.bitfield.SpanBitFiled; +import com.navercorp.pinpoint.common.server.bo.serializer.trace.v2.bitfield.SimpleServiceTypeEncodingStrategy; +import com.navercorp.pinpoint.common.trace.LoggingInfo; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author Woonduk Kang(emeroad) + */ +public class SpanBitFiledTest { + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + + @Test + public void testRoot_1() throws Exception { + SpanBo spanBo = new SpanBo(); + spanBo.setParentSpanId(-1); + + SpanBitFiled spanBitFiled = SpanBitFiled.build(spanBo); + Assert.assertTrue(spanBitFiled.isRoot()); + + spanBitFiled.setRoot(false); + Assert.assertFalse(spanBitFiled.isRoot()); + + } + + @Test + public void testRoot_2() throws Exception { + SpanBo spanBo = new SpanBo(); + spanBo.setParentSpanId(0); + + SpanBitFiled spanBitFiled = SpanBitFiled.build(spanBo); + Assert.assertFalse(spanBitFiled.isRoot()); + + spanBitFiled.maskAll(); + spanBitFiled.setRoot(false); + Assert.assertFalse(spanBitFiled.isRoot()); + } + + @Test + public void testErrorCode_1() { + SpanBo spanBo = new SpanBo(); + spanBo.setErrCode(1); + SpanBitFiled spanBitFiled = SpanBitFiled.build(spanBo); + + Assert.assertTrue(spanBitFiled.isSetErrorCode()); + + spanBitFiled.setErrorCode(false); + Assert.assertFalse(spanBitFiled.isSetErrorCode()); + } + + @Test + public void testErrorCode_2() { + SpanBo spanBo = new SpanBo(); + spanBo.setErrCode(0); + SpanBitFiled spanBitFiled = SpanBitFiled.build(spanBo); + + Assert.assertFalse(spanBitFiled.isSetErrorCode()); + + spanBitFiled.maskAll(); + spanBitFiled.setErrorCode(false); + Assert.assertFalse(spanBitFiled.isSetErrorCode()); + } + + @Test + public void testApplicationServiceTypeEncodingStrategy_PREV_TYPE_EQUALS() { + SpanBo spanBo = new SpanBo(); + spanBo.setServiceType((short) 1000); + spanBo.setApplicationServiceType((short) 1000); + + SpanBitFiled spanBitFiled = SpanBitFiled.build(spanBo); + + Assert.assertEquals(spanBitFiled.getApplicationServiceTypeEncodingStrategy(), SimpleServiceTypeEncodingStrategy.PREV_EQUALS); + } + + @Test + public void testApplicationServiceTypeEncodingStrategy_RAW() { + SpanBo spanBo = new SpanBo(); + spanBo.setServiceType((short) 1000); + spanBo.setApplicationServiceType((short) 2000); + + SpanBitFiled spanBitFiled = SpanBitFiled.build(spanBo); + + Assert.assertEquals(spanBitFiled.getApplicationServiceTypeEncodingStrategy(), SimpleServiceTypeEncodingStrategy.RAW); + + spanBitFiled.maskAll(); + Assert.assertEquals(spanBitFiled.getApplicationServiceTypeEncodingStrategy(), SimpleServiceTypeEncodingStrategy.RAW); + spanBitFiled.setApplicationServiceTypeEncodingStrategy(SimpleServiceTypeEncodingStrategy.PREV_EQUALS); + Assert.assertEquals(spanBitFiled.getApplicationServiceTypeEncodingStrategy(), SimpleServiceTypeEncodingStrategy.PREV_EQUALS); + } + + + @Test + public void testHasException_1() { + SpanBo spanBo = new SpanBo(); + spanBo.setExceptionInfo(1, "error"); + + SpanBitFiled spanBitFiled = SpanBitFiled.build(spanBo); + + Assert.assertTrue(spanBitFiled.isSetHasException()); + + spanBitFiled.setHasException(false); + Assert.assertFalse(spanBitFiled.isSetHasException()); + } + + @Test + public void testHasException_2() { + SpanBo spanBo = new SpanBo(); + + SpanBitFiled spanBitFiled = SpanBitFiled.build(spanBo); + + Assert.assertFalse(spanBitFiled.isSetHasException()); + + spanBitFiled.maskAll(); + spanBitFiled.setHasException(false); + Assert.assertFalse(spanBitFiled.isSetHasException()); + } + + + + @Test + public void testLoggingTransactionInfo_1() { + SpanBo spanBo = new SpanBo(); + spanBo.setLoggingTransactionInfo(LoggingInfo.INFO.getCode()); + + SpanBitFiled spanBitFiled = SpanBitFiled.build(spanBo); + + Assert.assertTrue(spanBitFiled.isSetLoggingTransactionInfo()); + + spanBitFiled.setLoggingTransactionInfo(false); + Assert.assertFalse(spanBitFiled.isSetLoggingTransactionInfo()); + } + + @Test + public void testLoggingTransactionInfo_2() { + SpanBo spanBo = new SpanBo(); + + SpanBitFiled spanBitFiled = SpanBitFiled.build(spanBo); + + Assert.assertFalse(spanBitFiled.isSetLoggingTransactionInfo()); + + spanBitFiled.maskAll(); + spanBitFiled.setLoggingTransactionInfo(false); + Assert.assertFalse(spanBitFiled.isSetLoggingTransactionInfo()); + } + +} \ No newline at end of file diff --git a/commons-server/src/test/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/SpanEventBitFieldTest.java b/commons-server/src/test/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/SpanEventBitFieldTest.java new file mode 100644 index 000000000..b54871896 --- /dev/null +++ b/commons-server/src/test/java/com/navercorp/pinpoint/common/server/bo/serializer/trace/v2/bitfield/SpanEventBitFieldTest.java @@ -0,0 +1,29 @@ +package com.navercorp.pinpoint.common.server.bo.serializer.trace.v2.bitfield; + +import com.navercorp.pinpoint.common.server.bo.serializer.trace.v2.bitfield.SpanEventBitField; +import org.junit.Assert; +import org.junit.Test; + +/** + * @author Woonduk Kang(emeroad) + */ +public class SpanEventBitFieldTest { + @Test + public void isSetHasException() throws Exception { + + } + + @Test + public void setHasException_shortToByteCasting() throws Exception { + SpanEventBitField field = new SpanEventBitField(); + field.setHasException(true); + + byte byteField = (byte) field.getBitField(); + + SpanEventBitField byteCastField = new SpanEventBitField(byteField); + Assert.assertTrue(byteCastField.isSetHasException()); + + + } + +} \ No newline at end of file diff --git a/hbase/scripts/hbase-create-snappy.hbase b/hbase/scripts/hbase-create-snappy.hbase index c45e81177..ae21e213a 100644 --- a/hbase/scripts/hbase-create-snappy.hbase +++ b/hbase/scripts/hbase-create-snappy.hbase @@ -11,6 +11,8 @@ create 'ApiMetaData', { NAME => 'Api', COMPRESSION => 'SNAPPY', TTL => 31536000, create 'SqlMetaData_Ver2', { NAME => 'Sql', COMPRESSION => 'SNAPPY', TTL => 15552000, DATA_BLOCK_ENCODING => 'PREFIX' }, {SPLITS=>["\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x0a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x0c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x0e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x12\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x14\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x16\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x18\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x1a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x1c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x1e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"]} create 'Traces', { NAME => 'S', TTL => 5184000, COMPRESSION => 'SNAPPY', DATA_BLOCK_ENCODING => 'PREFIX' }, { NAME => 'A', TTL => 5184000, COMPRESSION => 'SNAPPY', DATA_BLOCK_ENCODING => 'PREFIX' }, { NAME => 'T', TTL => 5184000, COMPRESSION => 'SNAPPY', DATA_BLOCK_ENCODING => 'PREFIX' }, {SPLITS=>["\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x07\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x09\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x0a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x0c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x0d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x0e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x11\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x12\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x13\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x14\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x15\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x16\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x17\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x18\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x19\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x1a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x1b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x1c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x1d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x1e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x1f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x20\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x21\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x22\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x23\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x24\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x25\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x26\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x27\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x28\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x29\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x2a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x2b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x2c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x2d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x2e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x2f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x30\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x31\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x32\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x33\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x34\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x35\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x36\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x37\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x38\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x39\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x3a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x3b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x3c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x3d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x3e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x3f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"]} +create 'TraceV2', { NAME => 'S', TTL => 5184000, COMPRESSION => 'SNAPPY', DATA_BLOCK_ENCODING => 'PREFIX' }, {NUMREGIONS => 256, SPLITALGO => 'UniformSplit'} + create 'ApplicationTraceIndex', { NAME => 'I', TTL => 5184000, COMPRESSION => 'SNAPPY', DATA_BLOCK_ENCODING => 'PREFIX' }, {SPLITS=>["\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x0a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x0c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x0e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x12\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x14\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x16\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x18\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x1a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x1c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x1e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"]} create 'ApplicationMapStatisticsCaller_Ver2', { NAME => 'C', TTL => 5184000, VERSIONS => 1, COMPRESSION => 'SNAPPY', DATA_BLOCK_ENCODING => 'PREFIX' }, {SPLITS=>["\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x0a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x0c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x0e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x12\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x14\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x16\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x18\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x1a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x1c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x1e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"]} diff --git a/hbase/scripts/hbase-create.hbase b/hbase/scripts/hbase-create.hbase index 767bdbc40..d97ad6354 100644 --- a/hbase/scripts/hbase-create.hbase +++ b/hbase/scripts/hbase-create.hbase @@ -11,6 +11,8 @@ create 'ApiMetaData', { NAME => 'Api', TTL => 31536000, DATA_BLOCK_ENCODING => ' create 'SqlMetaData_Ver2', { NAME => 'Sql', TTL => 15552000, DATA_BLOCK_ENCODING => 'PREFIX' }, {SPLITS=>["\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x0a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x0c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x0e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x12\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x14\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x16\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x18\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x1a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x1c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x1e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"]} create 'Traces', { NAME => 'S', TTL => 5184000, DATA_BLOCK_ENCODING => 'PREFIX' }, { NAME => 'A', TTL => 5184000, DATA_BLOCK_ENCODING => 'PREFIX' }, { NAME => 'T', TTL => 5184000, DATA_BLOCK_ENCODING => 'PREFIX' }, {SPLITS=>["\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x07\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x09\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x0a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x0b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x0c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x0d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x0e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x0f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x11\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x12\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x13\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x14\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x15\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x16\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x17\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x18\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x19\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x1a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x1b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x1c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x1d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x1e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x1f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x20\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x21\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x22\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x23\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x24\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x25\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x26\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x27\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x28\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x29\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x2a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x2b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x2c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x2d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x2e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x2f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x30\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x31\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x32\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x33\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x34\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x35\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x36\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x37\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x38\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x39\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x3a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x3b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x3c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x3d\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x3e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x3f\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"]} +create 'TraceV2', { NAME => 'S', TTL => 5184000, DATA_BLOCK_ENCODING => 'PREFIX' }, {NUMREGIONS => 256, SPLITALGO => 'UniformSplit'} + create 'ApplicationTraceIndex', { NAME => 'I', TTL => 5184000, DATA_BLOCK_ENCODING => 'PREFIX' }, {SPLITS=>["\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x0a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x0c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x0e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x12\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x14\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x16\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x18\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x1a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x1c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x1e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"]} create 'ApplicationMapStatisticsCaller_Ver2', { NAME => 'C', TTL => 5184000, VERSIONS => 1, DATA_BLOCK_ENCODING => 'PREFIX' }, {SPLITS=>["\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x06\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x0a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x0c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x0e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x10\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x12\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x14\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x16\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x18\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x1a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x1c\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00","\x1e\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"]} diff --git a/hbase/scripts/hbase-drop.hbase b/hbase/scripts/hbase-drop.hbase index 638f28612..73da8b718 100644 --- a/hbase/scripts/hbase-drop.hbase +++ b/hbase/scripts/hbase-drop.hbase @@ -12,6 +12,7 @@ disable 'SqlMetaData_Ver2' disable 'ApplicationTraceIndex' disable 'Traces' +disable 'TraceV2' disable 'ApplicationMapStatisticsCaller_Ver2' diff --git a/web/src/main/java/com/navercorp/pinpoint/web/dao/TraceDao.java b/web/src/main/java/com/navercorp/pinpoint/web/dao/TraceDao.java index f0d5efa68..c0cfe1214 100644 --- a/web/src/main/java/com/navercorp/pinpoint/web/dao/TraceDao.java +++ b/web/src/main/java/com/navercorp/pinpoint/web/dao/TraceDao.java @@ -30,12 +30,14 @@ public interface TraceDao { List selectSpan(TransactionId transactionId); + @Deprecated List selectSpanAndAnnotation(TransactionId transactionId); List> selectSpans(List transactionIdList); List> selectAllSpans(Collection transactionIdList); + @Deprecated List selectSpans(TransactionId transactionId); } diff --git a/web/src/main/java/com/navercorp/pinpoint/web/dao/hbase/HbaseDualReadDao.java b/web/src/main/java/com/navercorp/pinpoint/web/dao/hbase/HbaseDualReadDao.java new file mode 100644 index 000000000..bcd6e0ecd --- /dev/null +++ b/web/src/main/java/com/navercorp/pinpoint/web/dao/hbase/HbaseDualReadDao.java @@ -0,0 +1,148 @@ +package com.navercorp.pinpoint.web.dao.hbase; + +import com.google.common.annotations.Beta; +import com.navercorp.pinpoint.common.server.bo.SpanBo; +import com.navercorp.pinpoint.web.dao.TraceDao; +import com.navercorp.pinpoint.web.vo.TransactionId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collection; +import java.util.List; + +/** + * @author Woonduk Kang(emeroad) + */ +@Beta +public class HbaseDualReadDao implements TraceDao { + + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + + private final TraceDao master; + private final TraceDao slave; + + public HbaseDualReadDao(TraceDao v2, TraceDao slave) { + if (v2 == null) { + throw new NullPointerException("master must not be null"); + } + if (slave == null) { + throw new NullPointerException("v1 must not be null"); + } + + this.master = v2; + this.slave = slave; + } + + + @Override + public List selectSpan(TransactionId transactionId) { + Throwable masterThrowable = null; + List result = null; + try { + result = master.selectSpan(transactionId); + } catch (Throwable th) { + masterThrowable = th; + } + try { + slave.selectSpan(transactionId); + } catch (Throwable th) { + logger.debug("slave error :{}", th.getMessage(), th); + } + + rethrowRuntimeException(masterThrowable); + + return result; + } + + @Override + public List selectSpanAndAnnotation(TransactionId transactionId) { + Throwable masterThrowable = null; + List result = null; + try { + result = master.selectSpanAndAnnotation(transactionId); + } catch (Throwable th) { + masterThrowable = th; + } + try { + slave.selectSpanAndAnnotation(transactionId); + } catch (Throwable th) { + logger.debug("slave error :{}", th.getMessage(), th); + } + + rethrowRuntimeException(masterThrowable); + + return result; + } + + @Override + public List> selectSpans(List transactionIdList) { + Throwable masterThrowable = null; + List> result = null; + try { + result = master.selectSpans(transactionIdList); + } catch (Throwable th) { + masterThrowable = th; + } + try { + slave.selectSpans(transactionIdList); + } catch (Throwable th) { + logger.debug("slave error :{}", th.getMessage(), th); + } + + rethrowRuntimeException(masterThrowable); + + return result; + } + + @Override + public List> selectAllSpans(Collection transactionIdList) { + Throwable masterThrowable = null; + List> result = null; + try { + result = master.selectAllSpans(transactionIdList); + } catch (Throwable th) { + masterThrowable = th; + } + try { + slave.selectAllSpans(transactionIdList); + } catch (Throwable th) { + logger.debug("slave error :{}", th.getMessage(), th); + } + + rethrowRuntimeException(masterThrowable); + + return result; + } + + @Override + public List selectSpans(TransactionId transactionId) { + Throwable masterThrowable = null; + List result = null; + try { + result = master.selectSpans(transactionId); + } catch (Throwable th) { + masterThrowable = th; + } + try { + slave.selectSpans(transactionId); + } catch (Throwable th) { + logger.debug("slave error :{}", th.getMessage(), th); + } + + rethrowRuntimeException(masterThrowable); + + return result; + } + + + private void rethrowRuntimeException(Throwable exception) { + if (exception != null) { + this.rethrowException(exception); + } + } + + @SuppressWarnings("unchecked") + private void rethrowException(final Throwable exception) throws T { + throw (T) exception; + } +} diff --git a/web/src/main/java/com/navercorp/pinpoint/web/dao/hbase/HbaseTraceCompatibilityDao.java b/web/src/main/java/com/navercorp/pinpoint/web/dao/hbase/HbaseTraceCompatibilityDao.java new file mode 100644 index 000000000..84115ac83 --- /dev/null +++ b/web/src/main/java/com/navercorp/pinpoint/web/dao/hbase/HbaseTraceCompatibilityDao.java @@ -0,0 +1,80 @@ +package com.navercorp.pinpoint.web.dao.hbase; + +import com.navercorp.pinpoint.common.server.bo.SpanBo; +import com.navercorp.pinpoint.web.dao.TraceDao; +import com.navercorp.pinpoint.web.vo.TransactionId; +import org.apache.commons.collections.CollectionUtils; + +import java.util.Collection; +import java.util.List; + +/** + * @author Woonduk Kang(emeroad) + */ +public class HbaseTraceCompatibilityDao implements TraceDao { + + private final TraceDao master; + private final TraceDao slave; + + public HbaseTraceCompatibilityDao(TraceDao master, TraceDao slave) { + if (master == null) { + throw new NullPointerException("master must not be null"); + } + if (slave == null) { + throw new NullPointerException("slave must not be null"); + } + + this.master = master; + this.slave = slave; + } + + @Override + public List selectSpan(TransactionId transactionId) { + List spanBos = this.master.selectSpan(transactionId); + if (CollectionUtils.isNotEmpty(spanBos)) { + return spanBos; + } + + return slave.selectSpan(transactionId); + } + + @Override + public List selectSpanAndAnnotation(TransactionId transactionId) { + List spanBos = this.master.selectSpanAndAnnotation(transactionId); + if (CollectionUtils.isNotEmpty(spanBos)) { + return spanBos; + } + + return slave.selectSpanAndAnnotation(transactionId); + } + + @Override + public List> selectSpans(List transactionIdList) { + List> spanBos = this.master.selectSpans(transactionIdList); + if (CollectionUtils.isNotEmpty(spanBos)) { + return spanBos; + } + + return slave.selectSpans(transactionIdList); + } + + @Override + public List> selectAllSpans(Collection transactionIdList) { + List> spanBos = this.master.selectAllSpans(transactionIdList); + if (CollectionUtils.isNotEmpty(spanBos)) { + return spanBos; + } + + return slave.selectAllSpans(transactionIdList); + } + + @Override + public List selectSpans(TransactionId transactionId) { + List spanBos = this.master.selectSpans(transactionId); + if (CollectionUtils.isNotEmpty(spanBos)) { + return spanBos; + } + + return slave.selectSpans(transactionId); + } +} diff --git a/web/src/main/java/com/navercorp/pinpoint/web/dao/hbase/HbaseTraceDao.java b/web/src/main/java/com/navercorp/pinpoint/web/dao/hbase/HbaseTraceDao.java index 4f830d948..72339567f 100644 --- a/web/src/main/java/com/navercorp/pinpoint/web/dao/hbase/HbaseTraceDao.java +++ b/web/src/main/java/com/navercorp/pinpoint/web/dao/hbase/HbaseTraceDao.java @@ -21,9 +21,12 @@ import com.navercorp.pinpoint.common.hbase.HBaseTables; import com.navercorp.pinpoint.common.hbase.HbaseOperations2; import com.navercorp.pinpoint.common.hbase.RowMapper; import com.navercorp.pinpoint.web.dao.TraceDao; +import com.navercorp.pinpoint.web.mapper.CellTraceMapper; import com.navercorp.pinpoint.web.vo.TransactionId; import com.sematext.hbase.wd.AbstractRowKeyDistributor; import org.apache.hadoop.hbase.client.Get; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; @@ -41,6 +44,8 @@ import java.util.List; @Repository public class HbaseTraceDao implements TraceDao { + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + @Autowired private HbaseOperations2 template2; @@ -48,8 +53,7 @@ public class HbaseTraceDao implements TraceDao { @Qualifier("traceDistributor") private AbstractRowKeyDistributor rowKeyDistributor; - @Autowired - @Qualifier("spanMapper") + private RowMapper> spanMapper; @Autowired @@ -62,6 +66,16 @@ public class HbaseTraceDao implements TraceDao { @Value("#{pinpointWebProps['web.hbase.selectAllSpans.limit'] ?: 500}") private int selectAllSpansLimit; + @Autowired + @Qualifier("spanMapper") + public void setSpanMapper(RowMapper> spanMapper) { + final Logger logger = LoggerFactory.getLogger(spanMapper.getClass()); + if (logger.isDebugEnabled()) { + spanMapper = CellTraceMapper.wrap(spanMapper); + } + this.spanMapper = spanMapper; + } + @Override public List selectSpan(TransactionId transactionId) { if (transactionId == null) { diff --git a/web/src/main/java/com/navercorp/pinpoint/web/dao/hbase/HbaseTraceDaoFactory.java b/web/src/main/java/com/navercorp/pinpoint/web/dao/hbase/HbaseTraceDaoFactory.java new file mode 100644 index 000000000..097cb10f0 --- /dev/null +++ b/web/src/main/java/com/navercorp/pinpoint/web/dao/hbase/HbaseTraceDaoFactory.java @@ -0,0 +1,62 @@ +package com.navercorp.pinpoint.web.dao.hbase; + +import com.navercorp.pinpoint.web.dao.TraceDao; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.FactoryBean; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Repository; + +/** + * TraceDao Factory for compatibility + * @author Woonduk Kang(emeroad) + */ +@Repository +public class HbaseTraceDaoFactory implements FactoryBean { + + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + + @Autowired + @Qualifier("hbaseTraceDao") + private TraceDao v1; + + @Autowired + @Qualifier("hbaseTraceDaoV2") + private TraceDao v2; + + + @Value("#{pinpointWebProps['web.experimental.span.format.compatibility.version'] ?: 'v1'}") + private String mode = "v1"; + + @Override + public TraceDao getObject() throws Exception { + + logger.info("TraceDao Compatibility {}", mode); + + if (mode.equalsIgnoreCase("v1")) { + return v1; + } + else if (mode.equalsIgnoreCase("v2")) { + return v2; + } + else if (mode.equalsIgnoreCase("compatibilityMode")) { + return new HbaseTraceCompatibilityDao(v2, v1); + } + else if (mode.equalsIgnoreCase("dualRead")) { + return new HbaseDualReadDao(v2, v1); + } + return v1; + } + + @Override + public Class getObjectType() { + return TraceDao.class; + } + + @Override + public boolean isSingleton() { + return true; + } +} diff --git a/web/src/main/java/com/navercorp/pinpoint/web/dao/hbase/HbaseTraceDaoV2.java b/web/src/main/java/com/navercorp/pinpoint/web/dao/hbase/HbaseTraceDaoV2.java new file mode 100644 index 000000000..00bad4dc9 --- /dev/null +++ b/web/src/main/java/com/navercorp/pinpoint/web/dao/hbase/HbaseTraceDaoV2.java @@ -0,0 +1,170 @@ +package com.navercorp.pinpoint.web.dao.hbase; + +import com.google.common.annotations.Beta; +import com.google.common.collect.Lists; +import com.navercorp.pinpoint.common.hbase.HBaseTables; +import com.navercorp.pinpoint.common.hbase.HbaseOperations2; +import com.navercorp.pinpoint.common.hbase.RowMapper; +import com.navercorp.pinpoint.common.server.bo.SpanBo; +import com.navercorp.pinpoint.web.dao.TraceDao; +import com.navercorp.pinpoint.web.mapper.CellTraceMapper; +import com.navercorp.pinpoint.web.vo.TransactionId; +import com.sematext.hbase.wd.AbstractRowKeyDistributor; +import org.apache.hadoop.hbase.client.Get; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Repository; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +/** + * @author Woonduk Kang(emeroad) + */ +@Beta +@Repository +public class HbaseTraceDaoV2 implements TraceDao { + + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + + @Autowired + private HbaseOperations2 template2; + + @Autowired + @Qualifier("traceV2Distributor") + private AbstractRowKeyDistributor rowKeyDistributor; + + + private RowMapper> spanMapperV2; + + + @Value("#{pinpointWebProps['web.hbase.selectSpans.limit'] ?: 500}") + private int selectSpansLimit; + + @Value("#{pinpointWebProps['web.hbase.selectAllSpans.limit'] ?: 500}") + private int selectAllSpansLimit; + + @Autowired + @Qualifier("spanMapperV2") + public void setSpanMapperV2(RowMapper> spanMapperV2) { + final Logger logger = LoggerFactory.getLogger(spanMapperV2.getClass()); + if (logger.isDebugEnabled()) { + spanMapperV2 = CellTraceMapper.wrap(spanMapperV2); + } + this.spanMapperV2 = spanMapperV2; + } + + @Override + public List selectSpan(TransactionId transactionId) { + if (transactionId == null) { + throw new NullPointerException("transactionId must not be null"); + } + + byte[] traceIdBytes = rowKeyDistributor.getDistributedKey(transactionId.getBytes()); + return template2.get(HBaseTables.TRACE_V2, traceIdBytes, HBaseTables.TRACE_V2_CF_SPAN, spanMapperV2); + } + + @Deprecated + @Override + public List selectSpanAndAnnotation(TransactionId transactionId) { + if (transactionId == null) { + throw new NullPointerException("transactionId must not be null"); + } + + return selectSpan(transactionId); + } + + + @Override + public List> selectSpans(List transactionIdList) { + return selectSpans(transactionIdList, selectSpansLimit); + } + + public List> selectSpans(List transactionIdList, int hBaseGetLimitSize) { + if (transactionIdList == null) { + throw new NullPointerException("transactionIdList must not be null"); + } + + List> splitTransactionIdList = splitTransactionIdList(transactionIdList, hBaseGetLimitSize); + + return getSpans(splitTransactionIdList, HBaseTables.TRACE_V2_CF_SPAN); + } + + @Override + public List> selectAllSpans(Collection transactionIdList) { + return selectAllSpans(transactionIdList, selectAllSpansLimit); + } + + public List> selectAllSpans(Collection transactionIdList, int hBaseGetLimitSize) { + if (transactionIdList == null) { + throw new NullPointerException("transactionIdList must not be null"); + } + + List> splitTransactionIdList = splitTransactionIdList(Lists.newArrayList(transactionIdList), hBaseGetLimitSize); + + + return getSpans(splitTransactionIdList, HBaseTables.TRACE_V2_CF_SPAN); + } + + + private List> splitTransactionIdList(List transactionIdList, int maxTransactionIdListSize) { + if (transactionIdList == null || transactionIdList.isEmpty()) { + return Collections.emptyList(); + } + + List> splitTransactionIdList = new ArrayList<>(); + + int index = 0; + int endIndex = transactionIdList.size(); + while (index < endIndex) { + int subListEndIndex = Math.min(index + maxTransactionIdListSize, endIndex); + splitTransactionIdList.add(transactionIdList.subList(index, subListEndIndex)); + index = subListEndIndex; + } + + return splitTransactionIdList; + } + + private List> getSpans(List> splitTransactionIdList, byte[] columnFamily) { + if (splitTransactionIdList == null || splitTransactionIdList.isEmpty()) { + return Collections.emptyList(); + } + + List> spanBoList = new ArrayList<>(); + for (List transactionIdList : splitTransactionIdList) { + spanBoList.addAll(getSpans0(transactionIdList, columnFamily)); + } + return spanBoList; + } + + private List> getSpans0(List transactionIdList, byte[] columnFamily) { + if (transactionIdList == null || transactionIdList.isEmpty()) { + return Collections.emptyList(); + } + + if (columnFamily == null) { + throw new NullPointerException("columnFamily may not be null."); + } + + final List getList = new ArrayList<>(transactionIdList.size()); + for (TransactionId transactionId : transactionIdList) { + + final byte[] transactionIdBytes = rowKeyDistributor.getDistributedKey(transactionId.getBytes()); + final Get get = new Get(transactionIdBytes); + get.addFamily(columnFamily); + + getList.add(get); + } + return template2.get(HBaseTables.TRACE_V2, getList, spanMapperV2); + } + + @Override + public List selectSpans(TransactionId transactionId) { + return selectSpan(transactionId); + } +} diff --git a/web/src/main/java/com/navercorp/pinpoint/web/mapper/CellTraceMapper.java b/web/src/main/java/com/navercorp/pinpoint/web/mapper/CellTraceMapper.java new file mode 100644 index 000000000..b88dfba07 --- /dev/null +++ b/web/src/main/java/com/navercorp/pinpoint/web/mapper/CellTraceMapper.java @@ -0,0 +1,46 @@ +package com.navercorp.pinpoint.web.mapper; + +import com.navercorp.pinpoint.common.hbase.RowMapper; +import com.navercorp.pinpoint.web.util.CellTracker; +import com.navercorp.pinpoint.web.util.DefaultCellTracker; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.client.Result; + +/** + * @author Woonduk Kang(emeroad) + */ +public class CellTraceMapper implements RowMapper { + private RowMapper delegate; + + public static RowMapper wrap(RowMapper deleagate) { + + return new CellTraceMapper(deleagate); + } + + + private CellTraceMapper(RowMapper delegate) { + if (delegate == null) { + throw new NullPointerException("delegate must not be null"); + } + this.delegate = delegate; + } + + @Override + public T mapRow(Result result, int rowNum) throws Exception { + final T returnValue = this.delegate.mapRow(result, rowNum); + + if (!result.isEmpty()) { + + final Cell[] rawCells = result.rawCells(); + + final CellTracker cellTracker = new DefaultCellTracker(delegate.getClass().getSimpleName()); + for (Cell cell : rawCells) { + cellTracker.trace(cell); + } + cellTracker.log(); + } + + return returnValue; + } + +} diff --git a/web/src/main/java/com/navercorp/pinpoint/web/mapper/SpanMapper.java b/web/src/main/java/com/navercorp/pinpoint/web/mapper/SpanMapper.java index 4b92a38ef..635770eea 100644 --- a/web/src/main/java/com/navercorp/pinpoint/web/mapper/SpanMapper.java +++ b/web/src/main/java/com/navercorp/pinpoint/web/mapper/SpanMapper.java @@ -43,7 +43,6 @@ import java.util.*; public class SpanMapper implements RowMapper> { private final Logger logger = LoggerFactory.getLogger(this.getClass()); - private AnnotationMapper annotationMapper; private final AnnotationBoDecoder annotationBoDecoder = new AnnotationBoDecoder(); @@ -79,7 +78,7 @@ public class SpanMapper implements RowMapper> { spanBo.setTraceTransactionSequence(transactionId.getTransactionSequence()); spanBo.setCollectorAcceptTime(cell.getTimestamp()); - spanBo.setSpanID(Bytes.toLong(cell.getQualifierArray(), cell.getQualifierOffset())); + spanBo.setSpanId(Bytes.toLong(cell.getQualifierArray(), cell.getQualifierOffset())); readSpan(spanBo, cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()); if (logger.isDebugEnabled()) { logger.debug("read span :{}", spanBo); @@ -118,9 +117,14 @@ public class SpanMapper implements RowMapper> { } } for (SpanEventBo spanEventBo : spanEventBoList) { - SpanBo spanBo = spanMap.get(spanEventBo.getSpanId()); + final Long spanId = spanEventBo.getSpanId(); + SpanBo spanBo = spanMap.get(spanId); if (spanBo != null) { spanBo.addSpanEvent(spanEventBo); + } else { + if (logger.isInfoEnabled()) { + logger.info("Span not exist spanId:{} spanEvent:{}", spanEventBo); + } } } if (annotationMapper != null) { diff --git a/web/src/main/java/com/navercorp/pinpoint/web/mapper/SpanMapperV2.java b/web/src/main/java/com/navercorp/pinpoint/web/mapper/SpanMapperV2.java new file mode 100644 index 000000000..f731e2c36 --- /dev/null +++ b/web/src/main/java/com/navercorp/pinpoint/web/mapper/SpanMapperV2.java @@ -0,0 +1,194 @@ +/* + * Copyright 2014 NAVER Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.navercorp.pinpoint.web.mapper; + +import com.google.common.annotations.Beta; +import com.google.common.collect.LinkedHashMultimap; +import com.google.common.collect.Lists; +import com.navercorp.pinpoint.common.PinpointConstants; +import com.navercorp.pinpoint.common.buffer.Buffer; +import com.navercorp.pinpoint.common.buffer.OffsetFixedBuffer; +import com.navercorp.pinpoint.common.hbase.HBaseTables; +import com.navercorp.pinpoint.common.hbase.RowMapper; +import com.navercorp.pinpoint.common.server.bo.SpanBo; +import com.navercorp.pinpoint.common.server.bo.SpanChunkBo; +import com.navercorp.pinpoint.common.server.bo.SpanEventBo; +import com.navercorp.pinpoint.common.server.bo.serializer.trace.v2.SpanDecoder; +import com.navercorp.pinpoint.common.server.bo.serializer.trace.v2.SpanDecodingContext; +import com.navercorp.pinpoint.common.server.bo.serializer.trace.v2.SpanEncoder; +import com.navercorp.pinpoint.common.util.BytesUtils; +import com.navercorp.pinpoint.common.util.TransactionId; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.util.Bytes; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Set; + +/** + * @author emeroad + */ +@Beta +@Component +public class SpanMapperV2 implements RowMapper> { + + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + + public static final int AGENT_NAME_MAX_LEN = PinpointConstants.AGENT_NAME_MAX_LEN; + public static final int DISTRIBUTE_HASH_SIZE = 1; + + private final SpanDecoder spanDecoder = new SpanDecoder(); + + + @Override + public List mapRow(Result result, int rowNum) throws Exception { + if (result.isEmpty()) { + return Collections.emptyList(); + } + + + byte[] rowKey = result.getRow(); + final TransactionId transactionId = newTransactionId(rowKey, DISTRIBUTE_HASH_SIZE); + + final Cell[] rawCells = result.rawCells(); + + final List out = new ArrayList<>(rawCells.length); + final SpanDecodingContext decodingContext = new SpanDecodingContext(); + decodingContext.setTransactionId(transactionId); + + for (Cell cell : rawCells) { + // only if family name is "span" + if (CellUtil.matchingFamily(cell, HBaseTables.TRACE_V2_CF_SPAN)) { + + decodingContext.setCollectorAcceptedTime(cell.getTimestamp()); + + final Buffer qualifier = new OffsetFixedBuffer(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength()); + final Buffer columnValue = new OffsetFixedBuffer(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()); + + this.spanDecoder.decode(qualifier, columnValue, decodingContext, out); + + } else { + logger.warn("Unknown ColumnFamily :{}", Bytes.toStringBinary(CellUtil.cloneFamily(cell))); + } + this.spanDecoder.next(decodingContext); + } + this.spanDecoder.finish(decodingContext); + + + return buildSpanBoList(out); + + } + + private TransactionId newTransactionId(byte[] rowKey, int offset) { + + String agentId = BytesUtils.toStringAndRightTrim(rowKey, offset, AGENT_NAME_MAX_LEN); + long agentStartTime = BytesUtils.bytesToLong(rowKey, offset + AGENT_NAME_MAX_LEN); + long transactionSequence = BytesUtils.bytesToLong(rowKey, offset + BytesUtils.LONG_BYTE_LENGTH + AGENT_NAME_MAX_LEN); + + return new TransactionId(agentId, agentStartTime, transactionSequence); + } + + private List buildSpanBoList(List out) { + + LinkedHashMultimap spanMap = LinkedHashMultimap.create(); + List spanChunkList = new ArrayList<>(); + for (Object decodedSpan : out) { + if (decodedSpan instanceof SpanBo) { + SpanBo span = (SpanBo) decodedSpan; + + spanMap.put(span.getSpanId(), span); + + } else if (decodedSpan instanceof SpanChunkBo) { + + SpanChunkBo spanChunk = (SpanChunkBo) decodedSpan; + spanChunkList.add(spanChunk); + } else { + + logger.warn("Unknown span type {}", decodedSpan); + } + } + + + List spanBoList = bindSpanChunk(spanMap, spanChunkList); + bindAgentInfo(spanBoList); + return spanBoList; + } + + private void sortSpanEvent(List spanEventBoList) { + if (CollectionUtils.isEmpty(spanEventBoList)) { + return; + } + Collections.sort(spanEventBoList, SpanEncoder.SPAN_EVENT_SEQUENCE_COMPARATOR); + } + + private void bindAgentInfo(List spanBoList) { + // TODO workaround. fix class dependency + for (SpanBo spanBo : spanBoList) { + + List spanEventBoList = spanBo.getSpanEventBoList(); + sortSpanEvent(spanEventBoList); + for (SpanEventBo spanEventBo : spanEventBoList) { + spanEventBo.setAgentId(spanBo.getAgentId()); + spanEventBo.setApplicationId(spanBo.getApplicationId()); + spanEventBo.setAgentStartTime(spanBo.getAgentStartTime()); + + spanEventBo.setTraceAgentId(spanBo.getTraceAgentId()); + spanEventBo.setTraceAgentStartTime(spanBo.getTraceAgentStartTime()); + spanEventBo.setTraceTransactionSequence(spanBo.getTraceTransactionSequence()); + } + } + } + + private List bindSpanChunk(LinkedHashMultimap spanMap, List spanChunkList) { + for (SpanChunkBo spanChunkBo : spanChunkList) { + final Long spanId = spanChunkBo.getSpanId(); + Set matchedSpanBoList = spanMap.get(spanId); + if (matchedSpanBoList != null) { + final int spanIdCollisionSize = matchedSpanBoList.size(); + if (spanIdCollisionSize > 1) { + // exceptional case dump + logger.warn("spanIdCollision {}", matchedSpanBoList); + } + + int agentLevelCollisionCount = 0; + for (SpanBo spanBo : matchedSpanBoList) { + if (StringUtils.equals(spanBo.getAgentId(), spanChunkBo.getAgentId())) { + spanBo.addSpanEventBoList(spanChunkBo.getSpanEventBoList()); + agentLevelCollisionCount++; + } + } + if (agentLevelCollisionCount > 1) { + // exceptional case dump + logger.warn("agentLevelCollision {}", matchedSpanBoList); + } + } else { + if (logger.isInfoEnabled()) { + logger.info("Span not exist spanId:{} spanChunk:{}", spanId, spanChunkBo); + } + } + } + return Lists.newArrayList(spanMap.values()); + } +} diff --git a/web/src/main/java/com/navercorp/pinpoint/web/service/FilteredMapServiceImpl.java b/web/src/main/java/com/navercorp/pinpoint/web/service/FilteredMapServiceImpl.java index e62c14f8e..45a817d8e 100755 --- a/web/src/main/java/com/navercorp/pinpoint/web/service/FilteredMapServiceImpl.java +++ b/web/src/main/java/com/navercorp/pinpoint/web/service/FilteredMapServiceImpl.java @@ -46,6 +46,7 @@ import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; import org.springframework.util.StopWatch; @@ -71,6 +72,7 @@ public class FilteredMapServiceImpl implements FilteredMapService { private AgentInfoService agentInfoService; @Autowired + @Qualifier("hbaseTraceDaoFactory") private TraceDao traceDao; @Autowired diff --git a/web/src/main/java/com/navercorp/pinpoint/web/service/ScatterChartServiceImpl.java b/web/src/main/java/com/navercorp/pinpoint/web/service/ScatterChartServiceImpl.java index 87d4f121f..065ec94a4 100644 --- a/web/src/main/java/com/navercorp/pinpoint/web/service/ScatterChartServiceImpl.java +++ b/web/src/main/java/com/navercorp/pinpoint/web/service/ScatterChartServiceImpl.java @@ -29,6 +29,7 @@ import com.navercorp.pinpoint.web.vo.scatter.Dot; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; import java.util.ArrayList; @@ -48,6 +49,7 @@ public class ScatterChartServiceImpl implements ScatterChartService { private ApplicationTraceIndexDao applicationTraceIndexDao; @Autowired + @Qualifier("hbaseTraceDaoFactory") private TraceDao traceDao; @Override diff --git a/web/src/main/java/com/navercorp/pinpoint/web/service/SpanServiceImpl.java b/web/src/main/java/com/navercorp/pinpoint/web/service/SpanServiceImpl.java index 081279279..d9a06ec7d 100644 --- a/web/src/main/java/com/navercorp/pinpoint/web/service/SpanServiceImpl.java +++ b/web/src/main/java/com/navercorp/pinpoint/web/service/SpanServiceImpl.java @@ -43,10 +43,12 @@ import com.navercorp.pinpoint.web.security.MetaDataFilter; import com.navercorp.pinpoint.web.security.MetaDataFilter.MetaData; import com.navercorp.pinpoint.web.vo.TransactionId; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; /** * @author emeroad @@ -59,6 +61,7 @@ public class SpanServiceImpl implements SpanService { private Logger logger = LoggerFactory.getLogger(this.getClass()); @Autowired + @Qualifier("hbaseTraceDaoFactory") private TraceDao traceDao; // @Autowired @@ -83,7 +86,7 @@ public class SpanServiceImpl implements SpanService { } final List spans = traceDao.selectSpanAndAnnotation(transactionId); - if (spans == null || spans.isEmpty()) { + if (CollectionUtils.isEmpty(spans)) { return new SpanResult(SpanAligner2.FAIL_MATCH, new CallTreeIterator(null)); } diff --git a/web/src/main/java/com/navercorp/pinpoint/web/service/TransactionInfoServiceImpl.java b/web/src/main/java/com/navercorp/pinpoint/web/service/TransactionInfoServiceImpl.java index 139012219..9f9a237df 100644 --- a/web/src/main/java/com/navercorp/pinpoint/web/service/TransactionInfoServiceImpl.java +++ b/web/src/main/java/com/navercorp/pinpoint/web/service/TransactionInfoServiceImpl.java @@ -45,6 +45,7 @@ import com.navercorp.pinpoint.web.vo.callstacks.RecordSet; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; /** @@ -57,6 +58,7 @@ public class TransactionInfoServiceImpl implements TransactionInfoService { private final Logger logger = LoggerFactory.getLogger(this.getClass()); @Autowired + @Qualifier("hbaseTraceDaoFactory") private TraceDao traceDao; @Autowired diff --git a/web/src/main/java/com/navercorp/pinpoint/web/util/CellTracker.java b/web/src/main/java/com/navercorp/pinpoint/web/util/CellTracker.java new file mode 100644 index 000000000..0e1b2e935 --- /dev/null +++ b/web/src/main/java/com/navercorp/pinpoint/web/util/CellTracker.java @@ -0,0 +1,14 @@ +package com.navercorp.pinpoint.web.util; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; + +/** + * @author Woonduk Kang(emeroad) + */ +public interface CellTracker { + + void trace(Cell cell); + + void log(); +} diff --git a/web/src/main/java/com/navercorp/pinpoint/web/util/DefaultCellTracker.java b/web/src/main/java/com/navercorp/pinpoint/web/util/DefaultCellTracker.java new file mode 100644 index 000000000..75950147d --- /dev/null +++ b/web/src/main/java/com/navercorp/pinpoint/web/util/DefaultCellTracker.java @@ -0,0 +1,38 @@ +package com.navercorp.pinpoint.web.util; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author Woonduk Kang(emeroad) + */ +public class DefaultCellTracker implements CellTracker { + + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + + private final String message; + + private int serializedSize = 0; + private int serializedKeySize = 0; + private int cellCount = 0; + + public DefaultCellTracker(String message) { + this.message = message; + } + + @Override + public void trace(Cell cell) { + this.serializedSize += CellUtil.estimatedSerializedSizeOf(cell); + this.serializedKeySize += CellUtil.estimatedSerializedSizeOfKey(cell); + this.cellCount++; + } + + @Override + public void log() { + final int nonKey = serializedSize - serializedKeySize; + logger.debug("{} cellCount:{} serializedSize:{} key:{} nonKey:{}", message, cellCount, serializedSize, serializedKeySize, nonKey); + } + +} diff --git a/web/src/main/java/com/navercorp/pinpoint/web/util/EmptyCellTracker.java b/web/src/main/java/com/navercorp/pinpoint/web/util/EmptyCellTracker.java new file mode 100644 index 000000000..79f371a18 --- /dev/null +++ b/web/src/main/java/com/navercorp/pinpoint/web/util/EmptyCellTracker.java @@ -0,0 +1,24 @@ +package com.navercorp.pinpoint.web.util; + +import org.apache.hadoop.hbase.Cell; + +/** + * @author Woonduk Kang(emeroad) + */ +public final class EmptyCellTracker implements CellTracker { + + public static final EmptyCellTracker EMPTY_CELL_TRACER = new EmptyCellTracker(); + + private EmptyCellTracker() { + } + + @Override + public void trace(Cell cell) { + // skip + } + + @Override + public void log() { + // skip + } +} diff --git a/web/src/main/resources/applicationContext-hbase.xml b/web/src/main/resources/applicationContext-hbase.xml index dfcc2f33f..4cc22ce84 100644 --- a/web/src/main/resources/applicationContext-hbase.xml +++ b/web/src/main/resources/applicationContext-hbase.xml @@ -71,6 +71,15 @@ + + + + + + + + + diff --git a/web/src/main/resources/pinpoint-web.properties b/web/src/main/resources/pinpoint-web.properties index 40e04fb3d..c5fb27483 100644 --- a/web/src/main/resources/pinpoint-web.properties +++ b/web/src/main/resources/pinpoint-web.properties @@ -34,4 +34,8 @@ config.openSource=true web.hbase.selectSpans.limit=500 web.hbase.selectAllSpans.limit=500 -web.activethread.activeAgent.duration.days=7 \ No newline at end of file +web.activethread.activeAgent.duration.days=7 + +# span.binary format compatibility = v1 or v2 (WARNING : experimental feature) +# experimental feature span format v2 : https://github.com/naver/pinpoint/issues/1819 +web.experimental.span.format.compatibility.version=v1 \ No newline at end of file diff --git a/web/src/test/java/com/navercorp/pinpoint/web/filter/LinkFilterTest.java b/web/src/test/java/com/navercorp/pinpoint/web/filter/LinkFilterTest.java index 2a1cf1f18..c410611d6 100644 --- a/web/src/test/java/com/navercorp/pinpoint/web/filter/LinkFilterTest.java +++ b/web/src/test/java/com/navercorp/pinpoint/web/filter/LinkFilterTest.java @@ -44,7 +44,7 @@ public class LinkFilterTest { fromSpanBo.setServiceType(tomcatServiceType); fromSpanBo.setAgentId("AGENT_A"); - fromSpanBo.setSpanID(100); + fromSpanBo.setSpanId(100); SpanBo toSpanBO = new SpanBo(); toSpanBO.setApplicationId("APP_B"); @@ -86,7 +86,7 @@ public class LinkFilterTest { fromSpanBo.setServiceType(tomcatServiceType); fromSpanBo.setAgentId("AGENT_A"); - fromSpanBo.setSpanID(100); + fromSpanBo.setSpanId(100); SpanBo toSpanBO = new SpanBo(); toSpanBO.setApplicationId("APP_B"); diff --git a/web/src/test/java/com/navercorp/pinpoint/web/mapper/SpanMapperV2Test.java b/web/src/test/java/com/navercorp/pinpoint/web/mapper/SpanMapperV2Test.java new file mode 100644 index 000000000..9b6354766 --- /dev/null +++ b/web/src/test/java/com/navercorp/pinpoint/web/mapper/SpanMapperV2Test.java @@ -0,0 +1,106 @@ +package com.navercorp.pinpoint.web.mapper; + +import com.google.common.collect.Lists; +import com.navercorp.pinpoint.common.buffer.Buffer; +import com.navercorp.pinpoint.common.buffer.OffsetFixedBuffer; +import com.navercorp.pinpoint.common.server.bo.AnnotationBo; +import com.navercorp.pinpoint.common.server.bo.SpanBo; +import com.navercorp.pinpoint.common.server.bo.SpanEventBo; +import com.navercorp.pinpoint.common.server.bo.serializer.trace.v2.SpanDecoder; +import com.navercorp.pinpoint.common.server.bo.serializer.trace.v2.SpanDecodingContext; +import com.navercorp.pinpoint.common.server.bo.serializer.trace.v2.SpanEncoder; +import com.navercorp.pinpoint.common.server.bo.serializer.trace.v2.SpanEncodingContext; +import com.navercorp.pinpoint.common.util.AnnotationTranscoder; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.nio.ByteBuffer; +import java.util.List; + +/** + * @author Woonduk Kang(emeroad) + */ +public class SpanMapperV2Test { + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + + private final AnnotationTranscoder transcoder = new AnnotationTranscoder(); + private final SpanDecoder decoder = new SpanDecoder(); + + @Test + public void test() { + + SpanBo span = new SpanBo(); + span.setServiceType((short) 1000); + span.setExceptionInfo(1, "spanException"); + + SpanEventBo firstSpanEventBo = new SpanEventBo(); + firstSpanEventBo.setExceptionInfo(2, "first"); + firstSpanEventBo.setEndElapsed(100); + + AnnotationBo annotationBo = newAnnotation(200, "annotation"); + firstSpanEventBo.setAnnotationBoList(Lists.newArrayList(annotationBo)); + firstSpanEventBo.setServiceType((short) 1003); + firstSpanEventBo.setSequence((short) 0); + + span.addSpanEvent(firstSpanEventBo); + + //// next + SpanEventBo nextSpanEventBo = new SpanEventBo(); + nextSpanEventBo.setEndElapsed(200); + nextSpanEventBo.setServiceType((short) 2003); + nextSpanEventBo.setSequence((short) 1); + + + span.addSpanEvent(nextSpanEventBo); + + SpanEncodingContext encodingContext = new SpanEncodingContext<>(span); + SpanEncoder encoder = new SpanEncoder(); + ByteBuffer byteBuffer = encoder.encodeSpanColumnValue(encodingContext); + + Buffer buffer = new OffsetFixedBuffer(byteBuffer.array(), byteBuffer.arrayOffset(), byteBuffer.remaining()); + + SpanBo readSpan = new SpanBo(); + SpanDecodingContext decodingContext = new SpanDecodingContext(); + decoder.readSpanValue(buffer, readSpan, new SpanEventBo(), decodingContext); + + Assert.assertEquals(readSpan.getSpanEventBoList().size(), 2); + + + // span + Assert.assertEquals(readSpan.getServiceType(), 1000); + Assert.assertEquals(readSpan.hasException(), true); + Assert.assertEquals(readSpan.getExceptionId(), 1); + Assert.assertEquals(readSpan.getExceptionMessage(), "spanException"); + + List spanEventBoList = readSpan.getSpanEventBoList(); + SpanEventBo readFirst = spanEventBoList.get(0); + SpanEventBo readNext = spanEventBoList.get(1); + + Assert.assertEquals(readFirst.getEndElapsed(), 100); + Assert.assertEquals(readNext.getEndElapsed(), 200); + + Assert.assertEquals(readFirst.getExceptionId(), 2); + Assert.assertEquals(readNext.hasException(), false); + + Assert.assertEquals(readFirst.getServiceType(), 1003); + Assert.assertEquals(readNext.getServiceType(), 2003); + + Assert.assertEquals(readFirst.getSequence(), 0); + Assert.assertEquals(readNext.getSequence(), 1); + + } + + private AnnotationBo newAnnotation(int key, Object value) { + AnnotationBo annotationBo = new AnnotationBo(); + annotationBo.setKey(key); + + byte typeCode = transcoder.getTypeCode(value); + byte[] encode = transcoder.encode(value, typeCode); + annotationBo.setValue(value); + annotationBo.setByteValue(encode); + return annotationBo; + } + +} \ No newline at end of file