From 72e939ab0856e811c4c15105b45b61c74a0800f3 Mon Sep 17 00:00:00 2001 From: Woonduk Kang Date: Mon, 3 Aug 2015 15:56:59 +0900 Subject: [PATCH] #798 improve sql trace performance --- .../bootstrap/context}/ParsingResult.java | 4 +- .../bootstrap/context/RecordableTrace.java | 1 - .../bootstrap/context/SpanEventRecorder.java | 1 - .../bootstrap/context/TraceContext.java | 1 - .../tracevalue/ParsingResultTraceValue.java | 2 +- .../bootstrap/interceptor/MockTrace.java | 3 - .../interceptor/MockTraceContext.java | 3 +- .../common/util/DefaultNormalizedSql.java | 41 ++ .../common/util/DefaultParsingResult.java | 108 ---- .../common/util/DefaultSqlParser.java | 467 ++++++++++++++++++ .../pinpoint/common/util/NormalizedSql.java | 26 + .../common/util/OutputParameterParser.java | 2 +- .../pinpoint/common/util/SqlParser.java | 408 +-------------- ...serTest.java => DefaultSqlParserTest.java} | 50 +- .../PreparedStatementCreateInterceptor.java | 2 +- ...paredStatementExecuteQueryInterceptor.java | 2 +- .../context/CachingSqlNormalizer.java | 28 ++ .../context/DefaultCachingSqlNormalizer.java | 94 ++++ .../context/DefaultParsingResult.java | 93 ++++ .../profiler/context/DefaultTraceContext.java | 31 +- .../context/ParsingResultInternal.java | 35 ++ .../context/WrappedSpanEventRecorder.java | 2 +- .../PreparedStatementCreateInterceptor.java | 3 +- ...paredStatementExecuteQueryInterceptor.java | 2 +- .../context}/DefaultParsingResultTest.java | 7 +- .../pinpoint/test/mock/MockTraceContext.java | 2 +- .../pinpoint/web/service/SpanServiceImpl.java | 3 +- 27 files changed, 838 insertions(+), 583 deletions(-) rename {commons/src/main/java/com/navercorp/pinpoint/common/util => bootstrap-core/src/main/java/com/navercorp/pinpoint/bootstrap/context}/ParsingResult.java (91%) create mode 100644 commons/src/main/java/com/navercorp/pinpoint/common/util/DefaultNormalizedSql.java delete mode 100644 commons/src/main/java/com/navercorp/pinpoint/common/util/DefaultParsingResult.java create mode 100644 commons/src/main/java/com/navercorp/pinpoint/common/util/DefaultSqlParser.java create mode 100644 commons/src/main/java/com/navercorp/pinpoint/common/util/NormalizedSql.java rename commons/src/test/java/com/navercorp/pinpoint/common/util/{SqlParserTest.java => DefaultSqlParserTest.java} (87%) create mode 100644 profiler/src/main/java/com/navercorp/pinpoint/profiler/context/CachingSqlNormalizer.java create mode 100644 profiler/src/main/java/com/navercorp/pinpoint/profiler/context/DefaultCachingSqlNormalizer.java create mode 100644 profiler/src/main/java/com/navercorp/pinpoint/profiler/context/DefaultParsingResult.java create mode 100644 profiler/src/main/java/com/navercorp/pinpoint/profiler/context/ParsingResultInternal.java rename {commons/src/test/java/com/navercorp/pinpoint/common/util => profiler/src/test/java/com/navercorp/pinpoint/profiler/context}/DefaultParsingResultTest.java (84%) diff --git a/commons/src/main/java/com/navercorp/pinpoint/common/util/ParsingResult.java b/bootstrap-core/src/main/java/com/navercorp/pinpoint/bootstrap/context/ParsingResult.java similarity index 91% rename from commons/src/main/java/com/navercorp/pinpoint/common/util/ParsingResult.java rename to bootstrap-core/src/main/java/com/navercorp/pinpoint/bootstrap/context/ParsingResult.java index 789ab7897..d5990f56c 100644 --- a/commons/src/main/java/com/navercorp/pinpoint/common/util/ParsingResult.java +++ b/bootstrap-core/src/main/java/com/navercorp/pinpoint/bootstrap/context/ParsingResult.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.navercorp.pinpoint.common.util; +package com.navercorp.pinpoint.bootstrap.context; /** * @author emeroad @@ -28,6 +28,4 @@ public interface ParsingResult { String getOutput(); int getId(); - - boolean setId(int id); } diff --git a/bootstrap-core/src/main/java/com/navercorp/pinpoint/bootstrap/context/RecordableTrace.java b/bootstrap-core/src/main/java/com/navercorp/pinpoint/bootstrap/context/RecordableTrace.java index cb479ee81..89c877419 100644 --- a/bootstrap-core/src/main/java/com/navercorp/pinpoint/bootstrap/context/RecordableTrace.java +++ b/bootstrap-core/src/main/java/com/navercorp/pinpoint/bootstrap/context/RecordableTrace.java @@ -19,7 +19,6 @@ package com.navercorp.pinpoint.bootstrap.context; import com.navercorp.pinpoint.bootstrap.interceptor.MethodDescriptor; import com.navercorp.pinpoint.common.trace.AnnotationKey; import com.navercorp.pinpoint.common.trace.ServiceType; -import com.navercorp.pinpoint.common.util.ParsingResult; /** * @author emeroad diff --git a/bootstrap-core/src/main/java/com/navercorp/pinpoint/bootstrap/context/SpanEventRecorder.java b/bootstrap-core/src/main/java/com/navercorp/pinpoint/bootstrap/context/SpanEventRecorder.java index ecc045a60..eb36e24a2 100644 --- a/bootstrap-core/src/main/java/com/navercorp/pinpoint/bootstrap/context/SpanEventRecorder.java +++ b/bootstrap-core/src/main/java/com/navercorp/pinpoint/bootstrap/context/SpanEventRecorder.java @@ -3,7 +3,6 @@ package com.navercorp.pinpoint.bootstrap.context; import com.navercorp.pinpoint.bootstrap.interceptor.MethodDescriptor; import com.navercorp.pinpoint.common.trace.AnnotationKey; import com.navercorp.pinpoint.common.trace.ServiceType; -import com.navercorp.pinpoint.common.util.ParsingResult; public interface SpanEventRecorder extends FrameAttachment { diff --git a/bootstrap-core/src/main/java/com/navercorp/pinpoint/bootstrap/context/TraceContext.java b/bootstrap-core/src/main/java/com/navercorp/pinpoint/bootstrap/context/TraceContext.java index 5a44bb9fa..3cef3e98e 100644 --- a/bootstrap-core/src/main/java/com/navercorp/pinpoint/bootstrap/context/TraceContext.java +++ b/bootstrap-core/src/main/java/com/navercorp/pinpoint/bootstrap/context/TraceContext.java @@ -19,7 +19,6 @@ package com.navercorp.pinpoint.bootstrap.context; import com.navercorp.pinpoint.bootstrap.config.ProfilerConfig; import com.navercorp.pinpoint.bootstrap.interceptor.MethodDescriptor; import com.navercorp.pinpoint.common.trace.ServiceType; -import com.navercorp.pinpoint.common.util.ParsingResult; /** * @author emeroad diff --git a/bootstrap-core/src/main/java/com/navercorp/pinpoint/bootstrap/interceptor/tracevalue/ParsingResultTraceValue.java b/bootstrap-core/src/main/java/com/navercorp/pinpoint/bootstrap/interceptor/tracevalue/ParsingResultTraceValue.java index 8baf1020d..0342d4741 100644 --- a/bootstrap-core/src/main/java/com/navercorp/pinpoint/bootstrap/interceptor/tracevalue/ParsingResultTraceValue.java +++ b/bootstrap-core/src/main/java/com/navercorp/pinpoint/bootstrap/interceptor/tracevalue/ParsingResultTraceValue.java @@ -16,7 +16,7 @@ package com.navercorp.pinpoint.bootstrap.interceptor.tracevalue; -import com.navercorp.pinpoint.common.util.ParsingResult; +import com.navercorp.pinpoint.bootstrap.context.ParsingResult; /** * @author emeroad diff --git a/bootstrap/src/test/java/com/navercorp/pinpoint/bootstrap/interceptor/MockTrace.java b/bootstrap/src/test/java/com/navercorp/pinpoint/bootstrap/interceptor/MockTrace.java index 2373f4108..263c52444 100644 --- a/bootstrap/src/test/java/com/navercorp/pinpoint/bootstrap/interceptor/MockTrace.java +++ b/bootstrap/src/test/java/com/navercorp/pinpoint/bootstrap/interceptor/MockTrace.java @@ -23,10 +23,7 @@ import com.navercorp.pinpoint.bootstrap.context.SpanRecorder; import com.navercorp.pinpoint.bootstrap.context.Trace; import com.navercorp.pinpoint.bootstrap.context.TraceId; import com.navercorp.pinpoint.bootstrap.context.TraceType; -import com.navercorp.pinpoint.common.trace.AnnotationKey; -import com.navercorp.pinpoint.common.trace.ServiceType; import com.navercorp.pinpoint.common.util.Clock; -import com.navercorp.pinpoint.common.util.ParsingResult; import com.navercorp.pinpoint.common.util.SystemClock; /** diff --git a/bootstrap/src/test/java/com/navercorp/pinpoint/bootstrap/interceptor/MockTraceContext.java b/bootstrap/src/test/java/com/navercorp/pinpoint/bootstrap/interceptor/MockTraceContext.java index 4eccdfc12..9c28913ea 100644 --- a/bootstrap/src/test/java/com/navercorp/pinpoint/bootstrap/interceptor/MockTraceContext.java +++ b/bootstrap/src/test/java/com/navercorp/pinpoint/bootstrap/interceptor/MockTraceContext.java @@ -18,9 +18,8 @@ package com.navercorp.pinpoint.bootstrap.interceptor; import com.navercorp.pinpoint.bootstrap.config.ProfilerConfig; import com.navercorp.pinpoint.bootstrap.context.*; -import com.navercorp.pinpoint.bootstrap.interceptor.MethodDescriptor; import com.navercorp.pinpoint.common.trace.ServiceType; -import com.navercorp.pinpoint.common.util.ParsingResult; +import com.navercorp.pinpoint.bootstrap.context.ParsingResult; /** * @author emeroad diff --git a/commons/src/main/java/com/navercorp/pinpoint/common/util/DefaultNormalizedSql.java b/commons/src/main/java/com/navercorp/pinpoint/common/util/DefaultNormalizedSql.java new file mode 100644 index 000000000..4b1cc23b5 --- /dev/null +++ b/commons/src/main/java/com/navercorp/pinpoint/common/util/DefaultNormalizedSql.java @@ -0,0 +1,41 @@ +/* + * Copyright 2014 NAVER Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.navercorp.pinpoint.common.util; + +/** + * @author emeroad + */ +public class DefaultNormalizedSql implements NormalizedSql { + + private final String normalizedSql; + private final String parseParameter; + + public DefaultNormalizedSql(String normalizedSql, String parseParameter) { + this.normalizedSql = normalizedSql; + this.parseParameter = parseParameter; + } + + @Override + public String getNormalizedSql() { + return normalizedSql; + } + + @Override + public String getParseParameter() { + return parseParameter; + } +} diff --git a/commons/src/main/java/com/navercorp/pinpoint/common/util/DefaultParsingResult.java b/commons/src/main/java/com/navercorp/pinpoint/common/util/DefaultParsingResult.java deleted file mode 100644 index e76b4b79b..000000000 --- a/commons/src/main/java/com/navercorp/pinpoint/common/util/DefaultParsingResult.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright 2014 NAVER Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.navercorp.pinpoint.common.util; - - -import java.util.concurrent.atomic.AtomicIntegerFieldUpdater; - -/** - * @author emeroad - */ -public class DefaultParsingResult implements ParsingResult { - - public static final char SEPARATOR = ','; - - private static final AtomicIntegerFieldUpdater ID_UPDATER = AtomicIntegerFieldUpdater.newUpdater(DefaultParsingResult.class, "id"); - - private String sql; - private StringBuilder output; - private volatile int id = ParsingResult.ID_NOT_EXIST; - - - public DefaultParsingResult() { - - } - - public DefaultParsingResult(String sql, StringBuilder output) { - this.output = output; - this.sql = sql; - } - - @Override - public String getSql() { - return sql; - } - - public void setSql(String sql) { - this.sql = sql; - } - - @Override - public int getId() { - return id; - } - - public boolean setId(int id) { - return ID_UPDATER.compareAndSet(this, ID_NOT_EXIST, id); - } - - @Override - public String getOutput() { - if (output == null) { - return ""; - } - return output.toString(); - } - - /** - * Must be invoked at least once. If not, generates a NullPointerException upon invoking appendOutputParam. - */ - void appendOutputSeparator() { - if (output == null) { - this.output = new StringBuilder(); - } else { - this.output.append(SEPARATOR); - } - } - - void appendOutputParam(String str) { - this.output.append(str); - } - - void appendSeparatorCheckOutputParam(char ch) { - if (ch == ',') { - this.output.append(",,"); - } else { - this.output.append(ch); - } - } - - void appendOutputParam(char ch) { - this.output.append(ch); - } - - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder("DefaultParsingResult{"); - sb.append("sql='").append(sql).append('\''); - sb.append(", output=").append(output); - sb.append(", id=").append(id); - sb.append('}'); - return sb.toString(); - } -} diff --git a/commons/src/main/java/com/navercorp/pinpoint/common/util/DefaultSqlParser.java b/commons/src/main/java/com/navercorp/pinpoint/common/util/DefaultSqlParser.java new file mode 100644 index 000000000..a4dc3ff2c --- /dev/null +++ b/commons/src/main/java/com/navercorp/pinpoint/common/util/DefaultSqlParser.java @@ -0,0 +1,467 @@ +/* + * Copyright 2014 NAVER Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.navercorp.pinpoint.common.util; + +import java.util.List; + +/** + * @author emeroad + */ +public class DefaultSqlParser implements SqlParser { + + public static final char SEPARATOR = ','; + public static final char SYMBOL_REPLACE = '$'; + public static final char NUMBER_REPLACE = '#'; + + + private static final int NEXT_TOKEN_NOT_EXIST = -1; + private static final int NORMALIZED_SQL_BUFFER = 32; + + private static final NormalizedSql NULL_OBJECT = new DefaultNormalizedSql("", ""); + + public DefaultSqlParser() { + } + + + @Override + public NormalizedSql normalizedSql(final String sql) { + if (sql == null) { + return NULL_OBJECT; + } + + final int length = sql.length(); + final StringBuilder normalized = new StringBuilder(length + NORMALIZED_SQL_BUFFER); + final StringBuilder parsedParameter = new StringBuilder(32); + boolean change = false; + int replaceIndex = 0; + boolean numberTokenStartEnable = true; + for (int i = 0; i < length; i++) { + final char ch = sql.charAt(i); + switch (ch) { + // COMMENT start check + case '/': + // comment state + final int lookAhead1Char = lookAhead1(sql, i); + // multi line comment and oracle hint /*+ */ + if (lookAhead1Char == '*') { + normalized.append("/*"); + i += 2; + for (; i < length; i++) { + char stateCh = sql.charAt(i); + if (stateCh == '*') { + if (lookAhead1(sql, i) == '/') { + normalized.append("*/"); + i++; + break; + } + } + normalized.append(stateCh); + } + break; + // single line comment + } else if (lookAhead1Char == '/') { + normalized.append("//"); + i += 2; + i = readLine(sql, normalized, i); + break; + + } else { + // unary operator + numberTokenStartEnable = true; + normalized.append(ch); + break; + } +// case '#' +// # is a single line comment in mysql + case '-': + // single line comment state + if (lookAhead1(sql, i) == '-') { + normalized.append("--"); + i += 2; + i = readLine(sql, normalized, i); + break; + } else { + // unary operator + numberTokenStartEnable = true; + normalized.append(ch); + break; + } + + // SYMBOL start check + case '\'': + // empty symbol + if (lookAhead1(sql, i) == '\'') { + normalized.append("''"); + // no need to add parameter to output as $ is not converted + i += 2; + break; + } else { + change = true; + normalized.append('\''); + i++; + appendOutputSeparator(parsedParameter); + for (; i < length; i++) { + char stateCh = sql.charAt(i); + if (stateCh == '\'') { + // a consecutive ' is the same as \' + if (lookAhead1(sql, i) == '\'') { + i++; + appendOutputParam(parsedParameter, "''"); + continue; + } else { + normalized.append(replaceIndex++); + normalized.append(SYMBOL_REPLACE); + normalized.append('\''); +// outputParam.append(','); + break; + } + } + appendSeparatorCheckOutputParam(parsedParameter, stateCh); + } + break; + } + + // number start check + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + // http://www.h2database.com/html/grammar.html look at the state machine more + if (numberTokenStartEnable) { + change = true; + normalized.append(replaceIndex++); + normalized.append(NUMBER_REPLACE); + // number token start + appendOutputSeparator(parsedParameter); + appendOutputParam(parsedParameter, ch); + i++; + tokenEnd: + for (; i < length; i++) { + char stateCh = sql.charAt(i); + switch (stateCh) { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + case '.': + case 'E': + case 'e': + appendOutputParam(parsedParameter, stateCh); + break; + default: + // should look at the token outside the loop - not here +// outputParam.append(SEPARATOR); + i--; + break tokenEnd; + } + } + break; + } else { + normalized.append(ch); + break; + } + + // empty space + case ' ': + case '\t': + case '\n': + case '\r': + numberTokenStartEnable = true; + normalized.append(ch); + break; + // http://msdn.microsoft.com/en-us/library/ms174986.aspx + case '*': + case '+': + case '%': + case '=': + case '<': + case '>': + case '&': + case '|': + case '^': + case '~': + case '!': + numberTokenStartEnable = true; + normalized.append(ch); + break; + + case '(': + case ')': + case ',': + case ';': + numberTokenStartEnable = true; + normalized.append(ch); + break; + + case '.': + case '_': + case '@': // Assignment Operator + case ':': // Oracle's bind variable is possible with :bindvalue + numberTokenStartEnable = false; + normalized.append(ch); + break; + + default: + // what if it's in a different language?? + if (ch >= 'a' && ch <= 'z' || ch >= 'A' && ch <= 'Z') { + numberTokenStartEnable = false; + } else { + numberTokenStartEnable = true; + } + normalized.append(ch); + break; + } + } + if (change) { + String parsedParameterString; + if (parsedParameter.length() > 0) { + parsedParameterString = parsedParameter.toString(); + } else { + parsedParameterString = ""; + } + + return new DefaultNormalizedSql(normalized.toString(), parsedParameterString); + } else { + // Reuse if not modified. + // 1. new strings are not generated + // 2. reuse hashcodes + return new DefaultNormalizedSql(sql, ""); + } + } + + private int readLine(String sql, StringBuilder normalized, int index) { + final int length = sql.length(); + for (; index < length; index++) { + char ch = sql.charAt(index); + normalized.append(ch); + if (ch == '\n') { + break; + } + } + return index; + } + + private void appendOutputSeparator(StringBuilder output) { + if (output.length() == 0) { + // first parameter + return; + } + output.append(SEPARATOR); + } + + private void appendOutputParam(StringBuilder output, String str) { + output.append(str); + } + + private void appendSeparatorCheckOutputParam(StringBuilder output, char ch) { + if (ch == ',') { + output.append(",,"); + } else { + output.append(ch); + } + } + + private void appendOutputParam(StringBuilder output, char ch) { + output.append(ch); + } + + /** + * look up the next character in a string + * + * @param sql + * @param index + * @return + */ + private int lookAhead1(String sql, int index) { + index++; + if (index < sql.length()) { + return sql.charAt(index); + } else { + return NEXT_TOKEN_NOT_EXIST; + } + } + + @Override + public String combineOutputParams(String sql, List outputParams) { + + final int length = sql.length(); + final StringBuilder normalized = new StringBuilder(length + 16); + for (int i = 0; i < length; i++) { + final char ch = sql.charAt(i); + switch (ch) { + // COMMENT start check + case '/': + // comment state + int lookAhead1Char = lookAhead1(sql, i); + // multi line comment and oracle hint /*+ */ + if (lookAhead1Char == '*') { + normalized.append("/*"); + i += 2; + for (; i < length; i++) { + char stateCh = sql.charAt(i); + if (stateCh == '*') { + if (lookAhead1(sql, i) == '/') { + normalized.append("*/"); + i++; + break; + } + } + normalized.append(stateCh); + } + break; + // single line comment + } else if (lookAhead1Char == '/') { + normalized.append("//"); + i += 2; + i = readLine(sql, normalized, i); + break; + + } else { + // unary operator +// numberTokenStartEnable = true; + normalized.append(ch); + break; + } +// case '#' +// # is a single line comment in mysql + case '-': + // single line comment state + if (lookAhead1(sql, i) == '-') { + normalized.append("--"); + i += 2; + i = readLine(sql, normalized, i); + break; + } else { + // unary operator +// numberTokenStartEnable = true; + normalized.append(ch); + break; + } + + // number start check + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + // http://www.h2database.com/html/grammar.html look at the state machine more + if (lookAhead1(sql, i) == NEXT_TOKEN_NOT_EXIST) { + normalized.append(ch); + break; + } + StringBuilder outputIndex = new StringBuilder(); + outputIndex.append(ch); + // number token start + i++; + tokenEnd: + for (; i < length; i++) { + final char stateCh = sql.charAt(i); + switch (stateCh) { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + if (lookAhead1(sql, i) == NEXT_TOKEN_NOT_EXIST) { + outputIndex.append(stateCh); + normalized.append(outputIndex.toString()); + break tokenEnd; + } + outputIndex.append(stateCh); + break; + case NUMBER_REPLACE: + int numberIndex = 0; + try { + numberIndex = Integer.parseInt(outputIndex.toString()); + } catch (NumberFormatException e) { + // just append for invalid parameters + normalized.append(outputIndex.toString()); + normalized.append(NUMBER_REPLACE); + break tokenEnd; + } + try { + String replaceNumber = outputParams.get(numberIndex); + normalized.append(replaceNumber); + } catch (IndexOutOfBoundsException e) { + // just append for invalid parameters + normalized.append(outputIndex.toString()); + normalized.append(NUMBER_REPLACE); + break tokenEnd; + } + break tokenEnd; + + case SYMBOL_REPLACE: + int symbolIndex = 0; + try { + symbolIndex = Integer.parseInt(outputIndex.toString()); + } catch (NumberFormatException e) { + // just append for invalid parameters + normalized.append(outputIndex.toString()); + normalized.append(SYMBOL_REPLACE); + } + try { + String replaceSymbol = outputParams.get(symbolIndex); + normalized.append(replaceSymbol); + } catch (IndexOutOfBoundsException e) { + normalized.append(outputIndex.toString()); + normalized.append(SYMBOL_REPLACE); + } + break tokenEnd; + + default: + // should look at the token outside the loop - not here +// outputParam.append(SEPARATOR); + normalized.append(outputIndex.toString()); + i--; + break tokenEnd; + } + } + break; + + default: + normalized.append(ch); + break; + } + } + + return normalized.toString(); + } + + +} diff --git a/commons/src/main/java/com/navercorp/pinpoint/common/util/NormalizedSql.java b/commons/src/main/java/com/navercorp/pinpoint/common/util/NormalizedSql.java new file mode 100644 index 000000000..6fff1abd2 --- /dev/null +++ b/commons/src/main/java/com/navercorp/pinpoint/common/util/NormalizedSql.java @@ -0,0 +1,26 @@ +/* + * Copyright 2014 NAVER Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.navercorp.pinpoint.common.util; + +/** + * @author emeroad + */ +public interface NormalizedSql { + String getNormalizedSql(); + + String getParseParameter(); +} diff --git a/commons/src/main/java/com/navercorp/pinpoint/common/util/OutputParameterParser.java b/commons/src/main/java/com/navercorp/pinpoint/common/util/OutputParameterParser.java index 31d763956..9ac228994 100644 --- a/commons/src/main/java/com/navercorp/pinpoint/common/util/OutputParameterParser.java +++ b/commons/src/main/java/com/navercorp/pinpoint/common/util/OutputParameterParser.java @@ -25,7 +25,7 @@ import java.util.List; */ public class OutputParameterParser { - public static final char SEPARATOR = DefaultParsingResult.SEPARATOR; + public static final char SEPARATOR = DefaultSqlParser.SEPARATOR; public List parseOutputParameter(String outputParams) { // may also need to know about the parsing result diff --git a/commons/src/main/java/com/navercorp/pinpoint/common/util/SqlParser.java b/commons/src/main/java/com/navercorp/pinpoint/common/util/SqlParser.java index 993e72e07..c4fc16cfc 100644 --- a/commons/src/main/java/com/navercorp/pinpoint/common/util/SqlParser.java +++ b/commons/src/main/java/com/navercorp/pinpoint/common/util/SqlParser.java @@ -21,411 +21,9 @@ import java.util.List; /** * @author emeroad */ -public class SqlParser { +public interface SqlParser { - public static final char SYMBOL_REPLACE = '$'; - public static final char NUMBER_REPLACE = '#'; - - private static final DefaultParsingResult NULL = new DefaultParsingResult("", new StringBuilder()); - private static final int NEXT_TOKEN_NOT_EXIST = -1; - - - public SqlParser() { - } - - public DefaultParsingResult normalizedSql(String sql) { - if (sql == null) { - return NULL; - } - - DefaultParsingResult parsingResult = new DefaultParsingResult(); - final int length = sql.length(); - final StringBuilder normalized = new StringBuilder(length + 16); - boolean change = false; - int replaceIndex = 0; - boolean numberTokenStartEnable = true; - for (int i = 0; i < length; i++) { - final char ch = sql.charAt(i); - switch (ch) { - // COMMENT start check - case '/': - // comment state - int lookAhead1Char = lookAhead1(sql, i); - // multi line comment and oracle hint /*+ */ - if (lookAhead1Char == '*') { - normalized.append("/*"); - i += 2; - for (; i < length; i++) { - char stateCh = sql.charAt(i); - if (stateCh == '*') { - if (lookAhead1(sql, i) == '/') { - normalized.append("*/"); - i++; - break; - } - } - normalized.append(stateCh); - } - break; - // single line comment - } else if (lookAhead1Char == '/') { - normalized.append("//"); - i += 2; - i = readLine(sql, normalized, i); - break; - - } else { - // unary operator - numberTokenStartEnable = true; - normalized.append(ch); - break; - } -// case '#' -// # is a single line comment in mysql - case '-': - // single line comment state - if (lookAhead1(sql, i) == '-') { - normalized.append("--"); - i += 2; - i = readLine(sql, normalized, i); - break; - } else { - // unary operator - numberTokenStartEnable = true; - normalized.append(ch); - break; - } - - // SYMBOL start check - case '\'': - // empty symbol - if (lookAhead1(sql, i) == '\'') { - normalized.append("''"); - // no need to add parameter to output as $ is not converted - i += 2; - break; - } else { - change = true; - normalized.append('\''); - i++; - parsingResult.appendOutputSeparator(); - for (; i < length; i++) { - char stateCh = sql.charAt(i); - if (stateCh == '\'') { - // a consecutive ' is the same as \' - if (lookAhead1(sql, i) == '\'') { - i++; - parsingResult.appendOutputParam("''"); - continue; - } else { - normalized.append(replaceIndex++); - normalized.append(SYMBOL_REPLACE); - normalized.append('\''); -// outputParam.append(','); - break; - } - } - parsingResult.appendSeparatorCheckOutputParam(stateCh); - } - break; - } - - // number start check - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - // http://www.h2database.com/html/grammar.html look at the state machine more - if (numberTokenStartEnable) { - change = true; - normalized.append(replaceIndex++); - normalized.append(NUMBER_REPLACE); - // number token start - parsingResult.appendOutputSeparator(); - parsingResult.appendOutputParam(ch); - i++; - tokenEnd: - for (; i < length; i++) { - char stateCh = sql.charAt(i); - switch (stateCh) { - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - case '.': - case 'E': - case 'e': - parsingResult.appendOutputParam(stateCh); - break; - default: - // should look at the token outside the loop - not here -// outputParam.append(SEPARATOR); - i--; - break tokenEnd; - } - } - break; - } else { - normalized.append(ch); - break; - } - - // empty space - case ' ': - case '\t': - case '\n': - case '\r': - numberTokenStartEnable = true; - normalized.append(ch); - break; - // http://msdn.microsoft.com/en-us/library/ms174986.aspx - case '*': - case '+': - case '%': - case '=': - case '<': - case '>': - case '&': - case '|': - case '^': - case '~': - case '!': - numberTokenStartEnable = true; - normalized.append(ch); - break; - - case '(': - case ')': - case ',': - case ';': - numberTokenStartEnable = true; - normalized.append(ch); - break; - - case '.': - case '_': - case '@': // Assignment Operator - case ':': // Oracle's bind variable is possible with :bindvalue - numberTokenStartEnable = false; - normalized.append(ch); - break; - - default: - // what if it's in a different language?? - if (ch >= 'a' && ch <= 'z' || ch >= 'A' && ch <= 'Z') { - numberTokenStartEnable = false; - } else { - numberTokenStartEnable = true; - } - normalized.append(ch); - break; - } - } - if (change) { - parsingResult.setSql(normalized.toString()); - return parsingResult; - } else { - // Reuse if not modified. - // 1. new strings are not generated - // 2. reuse hashcodes - parsingResult.setSql(sql); - return parsingResult; - } - } - - private int readLine(String sql, StringBuilder normalized, int index) { - final int length = sql.length(); - for (; index < length; index++) { - char ch = sql.charAt(index); - normalized.append(ch); - if (ch == '\n') { - break; - } - } - return index; - } - - /** - * look up the next character in a string - * - * @param sql - * @param index - * @return - */ - private int lookAhead1(String sql, int index) { - index++; - if (index < sql.length()) { - return sql.charAt(index); - } else { - return NEXT_TOKEN_NOT_EXIST; - } - } - - public String combineOutputParams(String sql, List outputParams) { - - final int length = sql.length(); - final StringBuilder normalized = new StringBuilder(length + 16); - for (int i = 0; i < length; i++) { - final char ch = sql.charAt(i); - switch (ch) { - // COMMENT start check - case '/': - // comment state - int lookAhead1Char = lookAhead1(sql, i); - // multi line comment and oracle hint /*+ */ - if (lookAhead1Char == '*') { - normalized.append("/*"); - i += 2; - for (; i < length; i++) { - char stateCh = sql.charAt(i); - if (stateCh == '*') { - if (lookAhead1(sql, i) == '/') { - normalized.append("*/"); - i++; - break; - } - } - normalized.append(stateCh); - } - break; - // single line comment - } else if (lookAhead1Char == '/') { - normalized.append("//"); - i += 2; - i = readLine(sql, normalized, i); - break; - - } else { - // unary operator -// numberTokenStartEnable = true; - normalized.append(ch); - break; - } -// case '#' -// # is a single line comment in mysql - case '-': - // single line comment state - if (lookAhead1(sql, i) == '-') { - normalized.append("--"); - i += 2; - i = readLine(sql, normalized, i); - break; - } else { - // unary operator -// numberTokenStartEnable = true; - normalized.append(ch); - break; - } - - // number start check - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - // http://www.h2database.com/html/grammar.html look at the state machine more - if (lookAhead1(sql, i) == NEXT_TOKEN_NOT_EXIST) { - normalized.append(ch); - break; - } - StringBuilder outputIndex = new StringBuilder(); - outputIndex.append(ch); - // number token start - i++; - tokenEnd: - for (; i < length; i++) { - char stateCh = sql.charAt(i); - switch (stateCh) { - case '0': - case '1': - case '2': - case '3': - case '4': - case '5': - case '6': - case '7': - case '8': - case '9': - if (lookAhead1(sql, i) == NEXT_TOKEN_NOT_EXIST) { - outputIndex.append(stateCh); - normalized.append(outputIndex.toString()); - break tokenEnd; - } - outputIndex.append(stateCh); - break; - case NUMBER_REPLACE: - int numberIndex = 0; - try { - numberIndex = Integer.parseInt(outputIndex.toString()); - } catch (NumberFormatException e) { - // just append for invalid parameters - normalized.append(outputIndex.toString()); - normalized.append(NUMBER_REPLACE); - break tokenEnd; - } - try { - String replaceNumber = outputParams.get(numberIndex); - normalized.append(replaceNumber); - } catch (IndexOutOfBoundsException e) { - // just append for invalid parameters - normalized.append(outputIndex.toString()); - normalized.append(NUMBER_REPLACE); - break tokenEnd; - } - break tokenEnd; - - case SYMBOL_REPLACE: - int symbolIndex = 0; - try { - symbolIndex = Integer.parseInt(outputIndex.toString()); - } catch (NumberFormatException e) { - // just append for invalid parameters - normalized.append(outputIndex.toString()); - normalized.append(SYMBOL_REPLACE); - } - try { - String replaceSymbol = outputParams.get(symbolIndex); - normalized.append(replaceSymbol); - } catch (IndexOutOfBoundsException e) { - normalized.append(outputIndex.toString()); - normalized.append(SYMBOL_REPLACE); - } - break tokenEnd; - - default: - // should look at the token outside the loop - not here -// outputParam.append(SEPARATOR); - normalized.append(outputIndex.toString()); - i--; - break tokenEnd; - } - } - break; - - default: - normalized.append(ch); - break; - } - } - - return normalized.toString(); - } + NormalizedSql normalizedSql(String sql); + String combineOutputParams(String sql, List outputParams); } diff --git a/commons/src/test/java/com/navercorp/pinpoint/common/util/SqlParserTest.java b/commons/src/test/java/com/navercorp/pinpoint/common/util/DefaultSqlParserTest.java similarity index 87% rename from commons/src/test/java/com/navercorp/pinpoint/common/util/SqlParserTest.java rename to commons/src/test/java/com/navercorp/pinpoint/common/util/DefaultSqlParserTest.java index 03ce0813a..56dd11baf 100644 --- a/commons/src/test/java/com/navercorp/pinpoint/common/util/SqlParserTest.java +++ b/commons/src/test/java/com/navercorp/pinpoint/common/util/DefaultSqlParserTest.java @@ -22,33 +22,29 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.navercorp.pinpoint.common.util.OutputParameterParser; -import com.navercorp.pinpoint.common.util.ParsingResult; -import com.navercorp.pinpoint.common.util.SqlParser; - import java.util.List; /** * @author emeroad */ -public class SqlParserTest { +public class DefaultSqlParserTest { private final Logger logger = LoggerFactory.getLogger(this.getClass()); - private SqlParser sqlParser = new SqlParser(); + private SqlParser sqlParser = new DefaultSqlParser(); private OutputParameterParser outputParameterParser = new OutputParameterParser(); @Test public void normalizedSql() { - ParsingResult parsingResult = sqlParser.normalizedSql("select * from table a = 1 and b=50 and c=? and d='11'"); - String s = parsingResult.getSql(); + NormalizedSql parsingResult = sqlParser.normalizedSql("select * from table a = 1 and b=50 and c=? and d='11'"); + String s = parsingResult.getNormalizedSql(); logger.debug(s); - logger.debug(parsingResult.getOutput()); + logger.debug(parsingResult.getParseParameter()); - ParsingResult parsingResult2 = sqlParser.normalizedSql(" "); - String s2 = parsingResult2.getSql(); + NormalizedSql parsingResult2 = sqlParser.normalizedSql(" "); + String s2 = parsingResult2.getNormalizedSql(); logger.debug(s2); logger.debug("{}", (char) -1); @@ -60,10 +56,10 @@ public class SqlParserTest { logger.debug("{}", (int) Character.MIN_LOW_SURROGATE); logger.debug("{}", (int) Character.MAX_HIGH_SURROGATE); - ParsingResult parsingResult3 = sqlParser.normalizedSql("''"); - String s3 = parsingResult3.getSql(); + NormalizedSql parsingResult3 = sqlParser.normalizedSql("''"); + String s3 = parsingResult3.getNormalizedSql(); logger.debug("s3:{}", s3); - logger.debug("sb3:{}", parsingResult3.getOutput()); + logger.debug("sb3:{}", parsingResult3.getParseParameter()); } @Test @@ -226,7 +222,7 @@ public class SqlParserTest { assertEqual("'1234 456,7'", "'0$'", "1234 456,,7"); assertEqual("'1234''456,7'", "'0$'", "1234''456,,7"); - ParsingResult parsingResult2 = this.sqlParser.normalizedSql("'1234''456,7'"); + NormalizedSql parsingResult2 = this.sqlParser.normalizedSql("'1234''456,7'"); logger.debug("{}", parsingResult2); // for string token @@ -258,8 +254,8 @@ public class SqlParserTest { private void assertCombine(String result, String sql, String outputParams) { List output = this.outputParameterParser.parseOutputParameter(outputParams); - ParsingResult parsingResult = this.sqlParser.normalizedSql(result); - Assert.assertEquals("sql", parsingResult.getSql(), sql); + NormalizedSql parsingResult = this.sqlParser.normalizedSql(result); + Assert.assertEquals("sql", parsingResult.getNormalizedSql(), sql); String combine = this.sqlParser.combineOutputParams(sql, output); Assert.assertEquals("combine", result, combine); } @@ -273,8 +269,8 @@ public class SqlParserTest { private void assertEqual(String expected) { - ParsingResult parsingResult = sqlParser.normalizedSql(expected); - String normalizedSql = parsingResult.getSql(); + NormalizedSql parsingResult = sqlParser.normalizedSql(expected); + String normalizedSql = parsingResult.getNormalizedSql(); try { Assert.assertEquals(expected, normalizedSql); } catch (AssertionError e) { @@ -284,8 +280,8 @@ public class SqlParserTest { } private void assertEqual(String expected, String actual) { - ParsingResult parsingResult = sqlParser.normalizedSql(expected); - String normalizedSql = parsingResult.getSql(); + NormalizedSql parsingResult = sqlParser.normalizedSql(expected); + String normalizedSql = parsingResult.getNormalizedSql(); try { Assert.assertEquals(actual, normalizedSql); } catch (AssertionError e) { @@ -295,9 +291,9 @@ public class SqlParserTest { } private void assertEqual(String expected, String actual, String outputExpected) { - ParsingResult parsingResult = sqlParser.normalizedSql(expected); - String normalizedSql = parsingResult.getSql(); - String output = parsingResult.getOutput(); + NormalizedSql parsingResult = sqlParser.normalizedSql(expected); + String normalizedSql = parsingResult.getNormalizedSql(); + String output = parsingResult.getParseParameter(); List outputParams = outputParameterParser.parseOutputParameter(output); String s = sqlParser.combineOutputParams(normalizedSql, outputParams); logger.debug("combine:" + s); @@ -308,12 +304,12 @@ public class SqlParserTest { throw e; } - Assert.assertEquals("outputParam check", outputExpected, parsingResult.getOutput()); + Assert.assertEquals("outputParam check", outputExpected, parsingResult.getParseParameter()); } private void assertEqualObject(String expected) { - ParsingResult parsingResult = sqlParser.normalizedSql(expected); - String normalizedSql = parsingResult.getSql(); + NormalizedSql parsingResult = sqlParser.normalizedSql(expected); + String normalizedSql = parsingResult.getNormalizedSql(); try { Assert.assertEquals("normalizedSql check", expected, normalizedSql); Assert.assertSame(expected, normalizedSql); diff --git a/plugins/jdbc-driver/src/main/java/com/navercorp/pinpoint/plugin/jdbc/common/interceptor/PreparedStatementCreateInterceptor.java b/plugins/jdbc-driver/src/main/java/com/navercorp/pinpoint/plugin/jdbc/common/interceptor/PreparedStatementCreateInterceptor.java index 781a18ab7..b8b9a8314 100644 --- a/plugins/jdbc-driver/src/main/java/com/navercorp/pinpoint/plugin/jdbc/common/interceptor/PreparedStatementCreateInterceptor.java +++ b/plugins/jdbc-driver/src/main/java/com/navercorp/pinpoint/plugin/jdbc/common/interceptor/PreparedStatementCreateInterceptor.java @@ -26,7 +26,7 @@ import com.navercorp.pinpoint.bootstrap.plugin.annotation.Name; import com.navercorp.pinpoint.bootstrap.plugin.annotation.TargetMethod; import com.navercorp.pinpoint.bootstrap.plugin.annotation.Targets; import com.navercorp.pinpoint.bootstrap.util.InterceptorUtils; -import com.navercorp.pinpoint.common.util.ParsingResult; +import com.navercorp.pinpoint.bootstrap.context.ParsingResult; import com.navercorp.pinpoint.plugin.jdbc.common.JdbcDriverConstants; import com.navercorp.pinpoint.plugin.jdbc.common.UnKnownDatabaseInfo; diff --git a/plugins/jdbc-driver/src/main/java/com/navercorp/pinpoint/plugin/jdbc/common/interceptor/PreparedStatementExecuteQueryInterceptor.java b/plugins/jdbc-driver/src/main/java/com/navercorp/pinpoint/plugin/jdbc/common/interceptor/PreparedStatementExecuteQueryInterceptor.java index ee7c6bcb7..f6fbb0445 100644 --- a/plugins/jdbc-driver/src/main/java/com/navercorp/pinpoint/plugin/jdbc/common/interceptor/PreparedStatementExecuteQueryInterceptor.java +++ b/plugins/jdbc-driver/src/main/java/com/navercorp/pinpoint/plugin/jdbc/common/interceptor/PreparedStatementExecuteQueryInterceptor.java @@ -31,7 +31,7 @@ import com.navercorp.pinpoint.bootstrap.logging.PLoggerFactory; import com.navercorp.pinpoint.bootstrap.plugin.annotation.Name; import com.navercorp.pinpoint.bootstrap.plugin.annotation.TargetMethod; import com.navercorp.pinpoint.bootstrap.plugin.annotation.Targets; -import com.navercorp.pinpoint.common.util.ParsingResult; +import com.navercorp.pinpoint.bootstrap.context.ParsingResult; import com.navercorp.pinpoint.plugin.jdbc.common.JdbcDriverConstants; import com.navercorp.pinpoint.plugin.jdbc.common.UnKnownDatabaseInfo; import com.navercorp.pinpoint.plugin.jdbc.common.bindvalue.BindValueUtils; diff --git a/profiler/src/main/java/com/navercorp/pinpoint/profiler/context/CachingSqlNormalizer.java b/profiler/src/main/java/com/navercorp/pinpoint/profiler/context/CachingSqlNormalizer.java new file mode 100644 index 000000000..ec0c70a87 --- /dev/null +++ b/profiler/src/main/java/com/navercorp/pinpoint/profiler/context/CachingSqlNormalizer.java @@ -0,0 +1,28 @@ +/* + * Copyright 2014 NAVER Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.navercorp.pinpoint.profiler.context; + +import com.navercorp.pinpoint.bootstrap.context.ParsingResult; + +/** + * @author emeroad + */ +public interface CachingSqlNormalizer { + ParsingResult wrapSql(String sql); + + boolean normalizedSql(ParsingResult sql); +} diff --git a/profiler/src/main/java/com/navercorp/pinpoint/profiler/context/DefaultCachingSqlNormalizer.java b/profiler/src/main/java/com/navercorp/pinpoint/profiler/context/DefaultCachingSqlNormalizer.java new file mode 100644 index 000000000..5986fcc0f --- /dev/null +++ b/profiler/src/main/java/com/navercorp/pinpoint/profiler/context/DefaultCachingSqlNormalizer.java @@ -0,0 +1,94 @@ +/* + * Copyright 2014 NAVER Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.navercorp.pinpoint.profiler.context; + +import com.navercorp.pinpoint.bootstrap.context.ParsingResult; +import com.navercorp.pinpoint.common.util.*; +import com.navercorp.pinpoint.profiler.metadata.Result; +import com.navercorp.pinpoint.profiler.metadata.SimpleCache; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author emeroad + */ +public class DefaultCachingSqlNormalizer implements CachingSqlNormalizer { + + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + + private static final DefaultParsingResult EMPTY_OBJECT = new DefaultParsingResult(""); + + private final SimpleCache sqlCache; + private final SqlParser sqlParser; + + public DefaultCachingSqlNormalizer(int cacheSize) { + this.sqlCache = new SimpleCache(cacheSize); + this.sqlParser = new DefaultSqlParser(); + } + + @Override + public ParsingResult wrapSql(String sql) { + if (sql == null) { + return EMPTY_OBJECT; + } + return new DefaultParsingResult(sql); + } + + @Override + public boolean normalizedSql(ParsingResult parsingResult) { + if (parsingResult == null) { + return false; + } + if (parsingResult == EMPTY_OBJECT) { + return false; + } + if (parsingResult.getId() != ParsingResult.ID_NOT_EXIST) { + // already cached + return false; + } + + if (!(parsingResult instanceof ParsingResultInternal)) { + if (logger.isWarnEnabled()) { + logger.warn("unsupported ParsingResult Type type {}"); + } + throw new IllegalArgumentException("unsupported ParsingResult Type"); + } + + final ParsingResultInternal parsingResultInternal = (ParsingResultInternal) parsingResult; + + final String originalSql = parsingResultInternal.getOriginalSql(); + final NormalizedSql normalizedSql = this.sqlParser.normalizedSql(originalSql); + + final Result cachingResult = this.sqlCache.put(normalizedSql.getNormalizedSql()); + + // set normalizedSql + // set sqlId + final boolean success = parsingResultInternal.setId(cachingResult.getId()); + if (!success) { + if (logger.isWarnEnabled()) { + logger.warn("invalid state. setSqlId fail setId:{}, ParsingResultInternal:{}", cachingResult.getId(), parsingResultInternal); + } + } + + parsingResultInternal.setSql(normalizedSql.getNormalizedSql()); + parsingResultInternal.setOutput(normalizedSql.getParseParameter()); + + return cachingResult.isNewValue(); + } + + +} diff --git a/profiler/src/main/java/com/navercorp/pinpoint/profiler/context/DefaultParsingResult.java b/profiler/src/main/java/com/navercorp/pinpoint/profiler/context/DefaultParsingResult.java new file mode 100644 index 000000000..6d8d89789 --- /dev/null +++ b/profiler/src/main/java/com/navercorp/pinpoint/profiler/context/DefaultParsingResult.java @@ -0,0 +1,93 @@ +/* + * Copyright 2014 NAVER Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.navercorp.pinpoint.profiler.context; + + +import com.navercorp.pinpoint.bootstrap.context.ParsingResult; + +/** + * @author emeroad + */ +public class DefaultParsingResult implements ParsingResultInternal { + + private String originalSql; + private String sql ; + private String output; + + private int id = ParsingResult.ID_NOT_EXIST; + + + public DefaultParsingResult(String originalSql) { + this.originalSql = originalSql; + } + + + public String getOriginalSql() { + return originalSql; + } + + @Override + public String getSql() { + return sql; + } + + @Override + public void setSql(String sql) { + this.sql = sql; + } + + @Override + public void setOutput(String output) { + this.output = output; + } + + @Override + public int getId() { + return id; + } + + + public boolean setId(int id) { + // clear originalSql reference + this.originalSql = null; + + if (this.id == ID_NOT_EXIST) { + this.id = id; + return true; + } + return false; + } + + @Override + public String getOutput() { + if (this.output == null) { + return ""; + } + return this.output; + } + + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("DefaultParsingResult{"); + sb.append("sql='").append(sql).append('\''); + sb.append(", output=").append(output); + sb.append(", id=").append(id); + sb.append('}'); + return sb.toString(); + } +} diff --git a/profiler/src/main/java/com/navercorp/pinpoint/profiler/context/DefaultTraceContext.java b/profiler/src/main/java/com/navercorp/pinpoint/profiler/context/DefaultTraceContext.java index f3b038c4a..74be7832f 100644 --- a/profiler/src/main/java/com/navercorp/pinpoint/profiler/context/DefaultTraceContext.java +++ b/profiler/src/main/java/com/navercorp/pinpoint/profiler/context/DefaultTraceContext.java @@ -22,8 +22,7 @@ import com.navercorp.pinpoint.bootstrap.context.*; import com.navercorp.pinpoint.bootstrap.interceptor.MethodDescriptor; import com.navercorp.pinpoint.bootstrap.sampler.Sampler; import com.navercorp.pinpoint.common.trace.ServiceType; -import com.navercorp.pinpoint.common.util.DefaultParsingResult; -import com.navercorp.pinpoint.common.util.ParsingResult; +import com.navercorp.pinpoint.bootstrap.context.ParsingResult; import com.navercorp.pinpoint.common.util.SqlParser; import com.navercorp.pinpoint.profiler.AgentInformation; import com.navercorp.pinpoint.profiler.context.active.ActiveTraceFactory; @@ -66,8 +65,7 @@ public class DefaultTraceContext implements TraceContext { private EnhancedDataSender priorityDataSender; - private final SimpleCache sqlCache; - private final SqlParser sqlParser = new SqlParser(); + private final CachingSqlNormalizer cachingSqlNormalizer; private final SimpleCache apiCache = new SimpleCache(); private final SimpleCache stringCache = new SimpleCache(); @@ -96,7 +94,8 @@ public class DefaultTraceContext implements TraceContext { throw new NullPointerException("sampler must not be null"); } this.agentInformation = agentInformation; - this.sqlCache = new SimpleCache(sqlCacheSize); + + this.cachingSqlNormalizer = new DefaultCachingSqlNormalizer(sqlCacheSize); this.traceFactory = createTraceFactory(storageFactory, sampler, traceActiveThread); @@ -264,9 +263,8 @@ public class DefaultTraceContext implements TraceContext { @Override public ParsingResult parseSql(final String sql) { - - final DefaultParsingResult parsingResult = this.sqlParser.normalizedSql(sql); - return parsingResult; + // lazy sql normalization + return this.cachingSqlNormalizer.wrapSql(sql); } @Override @@ -274,14 +272,9 @@ public class DefaultTraceContext implements TraceContext { if (parsingResult == null) { return false; } - if (parsingResult.getId() != ParsingResult.ID_NOT_EXIST) { - // already cached - return false; - } - final String normalizedSql = parsingResult.getSql(); - - final Result cachingResult = this.sqlCache.put(normalizedSql); - if (cachingResult.isNewValue()) { + // lazy sql parsing + boolean isNewValue = this.cachingSqlNormalizer.normalizedSql(parsingResult); + if (isNewValue) { if (isDebug) { // TODO logging hit ratio could help debugging logger.debug("NewSQLParsingResult:{}", parsingResult); @@ -293,12 +286,12 @@ public class DefaultTraceContext implements TraceContext { sqlMetaData.setAgentId(getAgentId()); sqlMetaData.setAgentStartTime(getAgentStartTime()); - sqlMetaData.setSqlId(cachingResult.getId()); - sqlMetaData.setSql(normalizedSql); + sqlMetaData.setSqlId(parsingResult.getId()); + sqlMetaData.setSql(parsingResult.getSql()); this.priorityDataSender.request(sqlMetaData); } - return parsingResult.setId(cachingResult.getId()); + return isNewValue; } @Override diff --git a/profiler/src/main/java/com/navercorp/pinpoint/profiler/context/ParsingResultInternal.java b/profiler/src/main/java/com/navercorp/pinpoint/profiler/context/ParsingResultInternal.java new file mode 100644 index 000000000..ed241104e --- /dev/null +++ b/profiler/src/main/java/com/navercorp/pinpoint/profiler/context/ParsingResultInternal.java @@ -0,0 +1,35 @@ +/* + * Copyright 2014 NAVER Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.navercorp.pinpoint.profiler.context; + +import com.navercorp.pinpoint.bootstrap.context.ParsingResult; + +/** + * @author emeroad + */ +interface ParsingResultInternal extends ParsingResult { + + + String getOriginalSql(); + + boolean setId(int id); + + void setSql(String sql); + + void setOutput(String output); + +} diff --git a/profiler/src/main/java/com/navercorp/pinpoint/profiler/context/WrappedSpanEventRecorder.java b/profiler/src/main/java/com/navercorp/pinpoint/profiler/context/WrappedSpanEventRecorder.java index 3ecbdc5a8..67725b8b8 100644 --- a/profiler/src/main/java/com/navercorp/pinpoint/profiler/context/WrappedSpanEventRecorder.java +++ b/profiler/src/main/java/com/navercorp/pinpoint/profiler/context/WrappedSpanEventRecorder.java @@ -22,7 +22,7 @@ import com.navercorp.pinpoint.bootstrap.context.SpanEventRecorder; import com.navercorp.pinpoint.bootstrap.context.TraceContext; import com.navercorp.pinpoint.common.trace.AnnotationKey; import com.navercorp.pinpoint.common.trace.ServiceType; -import com.navercorp.pinpoint.common.util.ParsingResult; +import com.navercorp.pinpoint.bootstrap.context.ParsingResult; import com.navercorp.pinpoint.thrift.dto.TIntStringStringValue; /** diff --git a/profiler/src/main/java/com/navercorp/pinpoint/profiler/modifier/db/interceptor/PreparedStatementCreateInterceptor.java b/profiler/src/main/java/com/navercorp/pinpoint/profiler/modifier/db/interceptor/PreparedStatementCreateInterceptor.java index 5fdbdd2d8..b8e3fe95e 100644 --- a/profiler/src/main/java/com/navercorp/pinpoint/profiler/modifier/db/interceptor/PreparedStatementCreateInterceptor.java +++ b/profiler/src/main/java/com/navercorp/pinpoint/profiler/modifier/db/interceptor/PreparedStatementCreateInterceptor.java @@ -17,14 +17,13 @@ package com.navercorp.pinpoint.profiler.modifier.db.interceptor; import com.navercorp.pinpoint.bootstrap.context.DatabaseInfo; -import com.navercorp.pinpoint.bootstrap.context.RecordableTrace; import com.navercorp.pinpoint.bootstrap.context.SpanEventRecorder; import com.navercorp.pinpoint.bootstrap.interceptor.*; import com.navercorp.pinpoint.bootstrap.interceptor.tracevalue.DatabaseInfoTraceValue; import com.navercorp.pinpoint.bootstrap.interceptor.tracevalue.DatabaseInfoTraceValueUtils; import com.navercorp.pinpoint.bootstrap.interceptor.tracevalue.ParsingResultTraceValue; import com.navercorp.pinpoint.bootstrap.util.InterceptorUtils; -import com.navercorp.pinpoint.common.util.ParsingResult; +import com.navercorp.pinpoint.bootstrap.context.ParsingResult; /** * @author emeroad diff --git a/profiler/src/main/java/com/navercorp/pinpoint/profiler/modifier/db/interceptor/PreparedStatementExecuteQueryInterceptor.java b/profiler/src/main/java/com/navercorp/pinpoint/profiler/modifier/db/interceptor/PreparedStatementExecuteQueryInterceptor.java index 4ea56ffcf..e2490e783 100644 --- a/profiler/src/main/java/com/navercorp/pinpoint/profiler/modifier/db/interceptor/PreparedStatementExecuteQueryInterceptor.java +++ b/profiler/src/main/java/com/navercorp/pinpoint/profiler/modifier/db/interceptor/PreparedStatementExecuteQueryInterceptor.java @@ -32,7 +32,7 @@ import com.navercorp.pinpoint.bootstrap.interceptor.tracevalue.DatabaseInfoTrace import com.navercorp.pinpoint.bootstrap.interceptor.tracevalue.ParsingResultTraceValue; import com.navercorp.pinpoint.bootstrap.logging.PLogger; import com.navercorp.pinpoint.bootstrap.logging.PLoggerFactory; -import com.navercorp.pinpoint.common.util.ParsingResult; +import com.navercorp.pinpoint.bootstrap.context.ParsingResult; /** * @author emeroad diff --git a/commons/src/test/java/com/navercorp/pinpoint/common/util/DefaultParsingResultTest.java b/profiler/src/test/java/com/navercorp/pinpoint/profiler/context/DefaultParsingResultTest.java similarity index 84% rename from commons/src/test/java/com/navercorp/pinpoint/common/util/DefaultParsingResultTest.java rename to profiler/src/test/java/com/navercorp/pinpoint/profiler/context/DefaultParsingResultTest.java index 723ea5ce0..df1bf5b52 100644 --- a/commons/src/test/java/com/navercorp/pinpoint/common/util/DefaultParsingResultTest.java +++ b/profiler/src/test/java/com/navercorp/pinpoint/profiler/context/DefaultParsingResultTest.java @@ -14,8 +14,9 @@ * limitations under the License. */ -package com.navercorp.pinpoint.common.util; +package com.navercorp.pinpoint.profiler.context; +import com.navercorp.pinpoint.bootstrap.context.ParsingResult; import org.junit.Assert; import org.junit.Test; @@ -23,7 +24,7 @@ public class DefaultParsingResultTest { @Test public void testId() throws Exception { - DefaultParsingResult result = new DefaultParsingResult(); + DefaultParsingResult result = new DefaultParsingResult(""); Assert.assertEquals(ParsingResult.ID_NOT_EXIST, result.getId()); // update @@ -32,5 +33,5 @@ public class DefaultParsingResultTest { // already updated Assert.assertFalse(result.setId(1)); } +} -} \ No newline at end of file diff --git a/test/src/main/java/com/navercorp/pinpoint/test/mock/MockTraceContext.java b/test/src/main/java/com/navercorp/pinpoint/test/mock/MockTraceContext.java index e0e19ed48..b04c266f2 100644 --- a/test/src/main/java/com/navercorp/pinpoint/test/mock/MockTraceContext.java +++ b/test/src/main/java/com/navercorp/pinpoint/test/mock/MockTraceContext.java @@ -26,7 +26,7 @@ import com.navercorp.pinpoint.bootstrap.context.TraceId; import com.navercorp.pinpoint.bootstrap.context.TraceType; import com.navercorp.pinpoint.bootstrap.interceptor.MethodDescriptor; import com.navercorp.pinpoint.common.trace.ServiceType; -import com.navercorp.pinpoint.common.util.ParsingResult; +import com.navercorp.pinpoint.bootstrap.context.ParsingResult; /** * @author emeroad diff --git a/web/src/main/java/com/navercorp/pinpoint/web/service/SpanServiceImpl.java b/web/src/main/java/com/navercorp/pinpoint/web/service/SpanServiceImpl.java index 822c306bc..e241ad5a4 100644 --- a/web/src/main/java/com/navercorp/pinpoint/web/service/SpanServiceImpl.java +++ b/web/src/main/java/com/navercorp/pinpoint/web/service/SpanServiceImpl.java @@ -22,6 +22,7 @@ import java.util.List; import com.navercorp.pinpoint.common.bo.*; import com.navercorp.pinpoint.common.trace.AnnotationKey; +import com.navercorp.pinpoint.common.util.DefaultSqlParser; import com.navercorp.pinpoint.common.util.OutputParameterParser; import com.navercorp.pinpoint.common.util.SqlParser; import com.navercorp.pinpoint.web.calltree.span.CallTree; @@ -62,7 +63,7 @@ public class SpanServiceImpl implements SpanService { @Autowired private StringMetaDataDao stringMetaDataDao; - private final SqlParser sqlParser = new SqlParser(); + private final SqlParser sqlParser = new DefaultSqlParser(); private final OutputParameterParser outputParameterParser = new OutputParameterParser(); @Override