Merge pull request #803 from emeroad/#798_lazy_sql_normalization

#798 improve sql trace performance
This commit is contained in:
Woonduk Kang
2015-08-03 15:58:52 +09:00
27 changed files with 838 additions and 583 deletions
@@ -14,7 +14,7 @@
* limitations under the License.
*/
package com.navercorp.pinpoint.common.util;
package com.navercorp.pinpoint.bootstrap.context;
/**
* @author emeroad
@@ -28,6 +28,4 @@ public interface ParsingResult {
String getOutput();
int getId();
boolean setId(int id);
}
@@ -19,7 +19,6 @@ package com.navercorp.pinpoint.bootstrap.context;
import com.navercorp.pinpoint.bootstrap.interceptor.MethodDescriptor;
import com.navercorp.pinpoint.common.trace.AnnotationKey;
import com.navercorp.pinpoint.common.trace.ServiceType;
import com.navercorp.pinpoint.common.util.ParsingResult;
/**
* @author emeroad
@@ -3,7 +3,6 @@ package com.navercorp.pinpoint.bootstrap.context;
import com.navercorp.pinpoint.bootstrap.interceptor.MethodDescriptor;
import com.navercorp.pinpoint.common.trace.AnnotationKey;
import com.navercorp.pinpoint.common.trace.ServiceType;
import com.navercorp.pinpoint.common.util.ParsingResult;
public interface SpanEventRecorder extends FrameAttachment {
@@ -19,7 +19,6 @@ package com.navercorp.pinpoint.bootstrap.context;
import com.navercorp.pinpoint.bootstrap.config.ProfilerConfig;
import com.navercorp.pinpoint.bootstrap.interceptor.MethodDescriptor;
import com.navercorp.pinpoint.common.trace.ServiceType;
import com.navercorp.pinpoint.common.util.ParsingResult;
/**
* @author emeroad
@@ -16,7 +16,7 @@
package com.navercorp.pinpoint.bootstrap.interceptor.tracevalue;
import com.navercorp.pinpoint.common.util.ParsingResult;
import com.navercorp.pinpoint.bootstrap.context.ParsingResult;
/**
* @author emeroad
@@ -23,10 +23,7 @@ import com.navercorp.pinpoint.bootstrap.context.SpanRecorder;
import com.navercorp.pinpoint.bootstrap.context.Trace;
import com.navercorp.pinpoint.bootstrap.context.TraceId;
import com.navercorp.pinpoint.bootstrap.context.TraceType;
import com.navercorp.pinpoint.common.trace.AnnotationKey;
import com.navercorp.pinpoint.common.trace.ServiceType;
import com.navercorp.pinpoint.common.util.Clock;
import com.navercorp.pinpoint.common.util.ParsingResult;
import com.navercorp.pinpoint.common.util.SystemClock;
/**
@@ -18,9 +18,8 @@ package com.navercorp.pinpoint.bootstrap.interceptor;
import com.navercorp.pinpoint.bootstrap.config.ProfilerConfig;
import com.navercorp.pinpoint.bootstrap.context.*;
import com.navercorp.pinpoint.bootstrap.interceptor.MethodDescriptor;
import com.navercorp.pinpoint.common.trace.ServiceType;
import com.navercorp.pinpoint.common.util.ParsingResult;
import com.navercorp.pinpoint.bootstrap.context.ParsingResult;
/**
* @author emeroad
@@ -0,0 +1,41 @@
/*
* Copyright 2014 NAVER Corp.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.navercorp.pinpoint.common.util;
/**
* @author emeroad
*/
public class DefaultNormalizedSql implements NormalizedSql {
private final String normalizedSql;
private final String parseParameter;
public DefaultNormalizedSql(String normalizedSql, String parseParameter) {
this.normalizedSql = normalizedSql;
this.parseParameter = parseParameter;
}
@Override
public String getNormalizedSql() {
return normalizedSql;
}
@Override
public String getParseParameter() {
return parseParameter;
}
}
@@ -1,108 +0,0 @@
/*
* Copyright 2014 NAVER Corp.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.navercorp.pinpoint.common.util;
import java.util.concurrent.atomic.AtomicIntegerFieldUpdater;
/**
* @author emeroad
*/
public class DefaultParsingResult implements ParsingResult {
public static final char SEPARATOR = ',';
private static final AtomicIntegerFieldUpdater<DefaultParsingResult> ID_UPDATER = AtomicIntegerFieldUpdater.newUpdater(DefaultParsingResult.class, "id");
private String sql;
private StringBuilder output;
private volatile int id = ParsingResult.ID_NOT_EXIST;
public DefaultParsingResult() {
}
public DefaultParsingResult(String sql, StringBuilder output) {
this.output = output;
this.sql = sql;
}
@Override
public String getSql() {
return sql;
}
public void setSql(String sql) {
this.sql = sql;
}
@Override
public int getId() {
return id;
}
public boolean setId(int id) {
return ID_UPDATER.compareAndSet(this, ID_NOT_EXIST, id);
}
@Override
public String getOutput() {
if (output == null) {
return "";
}
return output.toString();
}
/**
* Must be invoked at least once. If not, generates a NullPointerException upon invoking appendOutputParam.
*/
void appendOutputSeparator() {
if (output == null) {
this.output = new StringBuilder();
} else {
this.output.append(SEPARATOR);
}
}
void appendOutputParam(String str) {
this.output.append(str);
}
void appendSeparatorCheckOutputParam(char ch) {
if (ch == ',') {
this.output.append(",,");
} else {
this.output.append(ch);
}
}
void appendOutputParam(char ch) {
this.output.append(ch);
}
@Override
public String toString() {
final StringBuilder sb = new StringBuilder("DefaultParsingResult{");
sb.append("sql='").append(sql).append('\'');
sb.append(", output=").append(output);
sb.append(", id=").append(id);
sb.append('}');
return sb.toString();
}
}
@@ -0,0 +1,467 @@
/*
* Copyright 2014 NAVER Corp.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.navercorp.pinpoint.common.util;
import java.util.List;
/**
* @author emeroad
*/
public class DefaultSqlParser implements SqlParser {
public static final char SEPARATOR = ',';
public static final char SYMBOL_REPLACE = '$';
public static final char NUMBER_REPLACE = '#';
private static final int NEXT_TOKEN_NOT_EXIST = -1;
private static final int NORMALIZED_SQL_BUFFER = 32;
private static final NormalizedSql NULL_OBJECT = new DefaultNormalizedSql("", "");
public DefaultSqlParser() {
}
@Override
public NormalizedSql normalizedSql(final String sql) {
if (sql == null) {
return NULL_OBJECT;
}
final int length = sql.length();
final StringBuilder normalized = new StringBuilder(length + NORMALIZED_SQL_BUFFER);
final StringBuilder parsedParameter = new StringBuilder(32);
boolean change = false;
int replaceIndex = 0;
boolean numberTokenStartEnable = true;
for (int i = 0; i < length; i++) {
final char ch = sql.charAt(i);
switch (ch) {
// COMMENT start check
case '/':
// comment state
final int lookAhead1Char = lookAhead1(sql, i);
// multi line comment and oracle hint /*+ */
if (lookAhead1Char == '*') {
normalized.append("/*");
i += 2;
for (; i < length; i++) {
char stateCh = sql.charAt(i);
if (stateCh == '*') {
if (lookAhead1(sql, i) == '/') {
normalized.append("*/");
i++;
break;
}
}
normalized.append(stateCh);
}
break;
// single line comment
} else if (lookAhead1Char == '/') {
normalized.append("//");
i += 2;
i = readLine(sql, normalized, i);
break;
} else {
// unary operator
numberTokenStartEnable = true;
normalized.append(ch);
break;
}
// case '#'
// # is a single line comment in mysql
case '-':
// single line comment state
if (lookAhead1(sql, i) == '-') {
normalized.append("--");
i += 2;
i = readLine(sql, normalized, i);
break;
} else {
// unary operator
numberTokenStartEnable = true;
normalized.append(ch);
break;
}
// SYMBOL start check
case '\'':
// empty symbol
if (lookAhead1(sql, i) == '\'') {
normalized.append("''");
// no need to add parameter to output as $ is not converted
i += 2;
break;
} else {
change = true;
normalized.append('\'');
i++;
appendOutputSeparator(parsedParameter);
for (; i < length; i++) {
char stateCh = sql.charAt(i);
if (stateCh == '\'') {
// a consecutive ' is the same as \'
if (lookAhead1(sql, i) == '\'') {
i++;
appendOutputParam(parsedParameter, "''");
continue;
} else {
normalized.append(replaceIndex++);
normalized.append(SYMBOL_REPLACE);
normalized.append('\'');
// outputParam.append(',');
break;
}
}
appendSeparatorCheckOutputParam(parsedParameter, stateCh);
}
break;
}
// number start check
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
// http://www.h2database.com/html/grammar.html look at the state machine more
if (numberTokenStartEnable) {
change = true;
normalized.append(replaceIndex++);
normalized.append(NUMBER_REPLACE);
// number token start
appendOutputSeparator(parsedParameter);
appendOutputParam(parsedParameter, ch);
i++;
tokenEnd:
for (; i < length; i++) {
char stateCh = sql.charAt(i);
switch (stateCh) {
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
case '.':
case 'E':
case 'e':
appendOutputParam(parsedParameter, stateCh);
break;
default:
// should look at the token outside the loop - not here
// outputParam.append(SEPARATOR);
i--;
break tokenEnd;
}
}
break;
} else {
normalized.append(ch);
break;
}
// empty space
case ' ':
case '\t':
case '\n':
case '\r':
numberTokenStartEnable = true;
normalized.append(ch);
break;
// http://msdn.microsoft.com/en-us/library/ms174986.aspx
case '*':
case '+':
case '%':
case '=':
case '<':
case '>':
case '&':
case '|':
case '^':
case '~':
case '!':
numberTokenStartEnable = true;
normalized.append(ch);
break;
case '(':
case ')':
case ',':
case ';':
numberTokenStartEnable = true;
normalized.append(ch);
break;
case '.':
case '_':
case '@': // Assignment Operator
case ':': // Oracle's bind variable is possible with :bindvalue
numberTokenStartEnable = false;
normalized.append(ch);
break;
default:
// what if it's in a different language??
if (ch >= 'a' && ch <= 'z' || ch >= 'A' && ch <= 'Z') {
numberTokenStartEnable = false;
} else {
numberTokenStartEnable = true;
}
normalized.append(ch);
break;
}
}
if (change) {
String parsedParameterString;
if (parsedParameter.length() > 0) {
parsedParameterString = parsedParameter.toString();
} else {
parsedParameterString = "";
}
return new DefaultNormalizedSql(normalized.toString(), parsedParameterString);
} else {
// Reuse if not modified.
// 1. new strings are not generated
// 2. reuse hashcodes
return new DefaultNormalizedSql(sql, "");
}
}
private int readLine(String sql, StringBuilder normalized, int index) {
final int length = sql.length();
for (; index < length; index++) {
char ch = sql.charAt(index);
normalized.append(ch);
if (ch == '\n') {
break;
}
}
return index;
}
private void appendOutputSeparator(StringBuilder output) {
if (output.length() == 0) {
// first parameter
return;
}
output.append(SEPARATOR);
}
private void appendOutputParam(StringBuilder output, String str) {
output.append(str);
}
private void appendSeparatorCheckOutputParam(StringBuilder output, char ch) {
if (ch == ',') {
output.append(",,");
} else {
output.append(ch);
}
}
private void appendOutputParam(StringBuilder output, char ch) {
output.append(ch);
}
/**
* look up the next character in a string
*
* @param sql
* @param index
* @return
*/
private int lookAhead1(String sql, int index) {
index++;
if (index < sql.length()) {
return sql.charAt(index);
} else {
return NEXT_TOKEN_NOT_EXIST;
}
}
@Override
public String combineOutputParams(String sql, List<String> outputParams) {
final int length = sql.length();
final StringBuilder normalized = new StringBuilder(length + 16);
for (int i = 0; i < length; i++) {
final char ch = sql.charAt(i);
switch (ch) {
// COMMENT start check
case '/':
// comment state
int lookAhead1Char = lookAhead1(sql, i);
// multi line comment and oracle hint /*+ */
if (lookAhead1Char == '*') {
normalized.append("/*");
i += 2;
for (; i < length; i++) {
char stateCh = sql.charAt(i);
if (stateCh == '*') {
if (lookAhead1(sql, i) == '/') {
normalized.append("*/");
i++;
break;
}
}
normalized.append(stateCh);
}
break;
// single line comment
} else if (lookAhead1Char == '/') {
normalized.append("//");
i += 2;
i = readLine(sql, normalized, i);
break;
} else {
// unary operator
// numberTokenStartEnable = true;
normalized.append(ch);
break;
}
// case '#'
// # is a single line comment in mysql
case '-':
// single line comment state
if (lookAhead1(sql, i) == '-') {
normalized.append("--");
i += 2;
i = readLine(sql, normalized, i);
break;
} else {
// unary operator
// numberTokenStartEnable = true;
normalized.append(ch);
break;
}
// number start check
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
// http://www.h2database.com/html/grammar.html look at the state machine more
if (lookAhead1(sql, i) == NEXT_TOKEN_NOT_EXIST) {
normalized.append(ch);
break;
}
StringBuilder outputIndex = new StringBuilder();
outputIndex.append(ch);
// number token start
i++;
tokenEnd:
for (; i < length; i++) {
final char stateCh = sql.charAt(i);
switch (stateCh) {
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
if (lookAhead1(sql, i) == NEXT_TOKEN_NOT_EXIST) {
outputIndex.append(stateCh);
normalized.append(outputIndex.toString());
break tokenEnd;
}
outputIndex.append(stateCh);
break;
case NUMBER_REPLACE:
int numberIndex = 0;
try {
numberIndex = Integer.parseInt(outputIndex.toString());
} catch (NumberFormatException e) {
// just append for invalid parameters
normalized.append(outputIndex.toString());
normalized.append(NUMBER_REPLACE);
break tokenEnd;
}
try {
String replaceNumber = outputParams.get(numberIndex);
normalized.append(replaceNumber);
} catch (IndexOutOfBoundsException e) {
// just append for invalid parameters
normalized.append(outputIndex.toString());
normalized.append(NUMBER_REPLACE);
break tokenEnd;
}
break tokenEnd;
case SYMBOL_REPLACE:
int symbolIndex = 0;
try {
symbolIndex = Integer.parseInt(outputIndex.toString());
} catch (NumberFormatException e) {
// just append for invalid parameters
normalized.append(outputIndex.toString());
normalized.append(SYMBOL_REPLACE);
}
try {
String replaceSymbol = outputParams.get(symbolIndex);
normalized.append(replaceSymbol);
} catch (IndexOutOfBoundsException e) {
normalized.append(outputIndex.toString());
normalized.append(SYMBOL_REPLACE);
}
break tokenEnd;
default:
// should look at the token outside the loop - not here
// outputParam.append(SEPARATOR);
normalized.append(outputIndex.toString());
i--;
break tokenEnd;
}
}
break;
default:
normalized.append(ch);
break;
}
}
return normalized.toString();
}
}
@@ -0,0 +1,26 @@
/*
* Copyright 2014 NAVER Corp.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.navercorp.pinpoint.common.util;
/**
* @author emeroad
*/
public interface NormalizedSql {
String getNormalizedSql();
String getParseParameter();
}
@@ -25,7 +25,7 @@ import java.util.List;
*/
public class OutputParameterParser {
public static final char SEPARATOR = DefaultParsingResult.SEPARATOR;
public static final char SEPARATOR = DefaultSqlParser.SEPARATOR;
public List<String> parseOutputParameter(String outputParams) {
// may also need to know about the parsing result
@@ -21,411 +21,9 @@ import java.util.List;
/**
* @author emeroad
*/
public class SqlParser {
public interface SqlParser {
public static final char SYMBOL_REPLACE = '$';
public static final char NUMBER_REPLACE = '#';
private static final DefaultParsingResult NULL = new DefaultParsingResult("", new StringBuilder());
private static final int NEXT_TOKEN_NOT_EXIST = -1;
public SqlParser() {
}
public DefaultParsingResult normalizedSql(String sql) {
if (sql == null) {
return NULL;
}
DefaultParsingResult parsingResult = new DefaultParsingResult();
final int length = sql.length();
final StringBuilder normalized = new StringBuilder(length + 16);
boolean change = false;
int replaceIndex = 0;
boolean numberTokenStartEnable = true;
for (int i = 0; i < length; i++) {
final char ch = sql.charAt(i);
switch (ch) {
// COMMENT start check
case '/':
// comment state
int lookAhead1Char = lookAhead1(sql, i);
// multi line comment and oracle hint /*+ */
if (lookAhead1Char == '*') {
normalized.append("/*");
i += 2;
for (; i < length; i++) {
char stateCh = sql.charAt(i);
if (stateCh == '*') {
if (lookAhead1(sql, i) == '/') {
normalized.append("*/");
i++;
break;
}
}
normalized.append(stateCh);
}
break;
// single line comment
} else if (lookAhead1Char == '/') {
normalized.append("//");
i += 2;
i = readLine(sql, normalized, i);
break;
} else {
// unary operator
numberTokenStartEnable = true;
normalized.append(ch);
break;
}
// case '#'
// # is a single line comment in mysql
case '-':
// single line comment state
if (lookAhead1(sql, i) == '-') {
normalized.append("--");
i += 2;
i = readLine(sql, normalized, i);
break;
} else {
// unary operator
numberTokenStartEnable = true;
normalized.append(ch);
break;
}
// SYMBOL start check
case '\'':
// empty symbol
if (lookAhead1(sql, i) == '\'') {
normalized.append("''");
// no need to add parameter to output as $ is not converted
i += 2;
break;
} else {
change = true;
normalized.append('\'');
i++;
parsingResult.appendOutputSeparator();
for (; i < length; i++) {
char stateCh = sql.charAt(i);
if (stateCh == '\'') {
// a consecutive ' is the same as \'
if (lookAhead1(sql, i) == '\'') {
i++;
parsingResult.appendOutputParam("''");
continue;
} else {
normalized.append(replaceIndex++);
normalized.append(SYMBOL_REPLACE);
normalized.append('\'');
// outputParam.append(',');
break;
}
}
parsingResult.appendSeparatorCheckOutputParam(stateCh);
}
break;
}
// number start check
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
// http://www.h2database.com/html/grammar.html look at the state machine more
if (numberTokenStartEnable) {
change = true;
normalized.append(replaceIndex++);
normalized.append(NUMBER_REPLACE);
// number token start
parsingResult.appendOutputSeparator();
parsingResult.appendOutputParam(ch);
i++;
tokenEnd:
for (; i < length; i++) {
char stateCh = sql.charAt(i);
switch (stateCh) {
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
case '.':
case 'E':
case 'e':
parsingResult.appendOutputParam(stateCh);
break;
default:
// should look at the token outside the loop - not here
// outputParam.append(SEPARATOR);
i--;
break tokenEnd;
}
}
break;
} else {
normalized.append(ch);
break;
}
// empty space
case ' ':
case '\t':
case '\n':
case '\r':
numberTokenStartEnable = true;
normalized.append(ch);
break;
// http://msdn.microsoft.com/en-us/library/ms174986.aspx
case '*':
case '+':
case '%':
case '=':
case '<':
case '>':
case '&':
case '|':
case '^':
case '~':
case '!':
numberTokenStartEnable = true;
normalized.append(ch);
break;
case '(':
case ')':
case ',':
case ';':
numberTokenStartEnable = true;
normalized.append(ch);
break;
case '.':
case '_':
case '@': // Assignment Operator
case ':': // Oracle's bind variable is possible with :bindvalue
numberTokenStartEnable = false;
normalized.append(ch);
break;
default:
// what if it's in a different language??
if (ch >= 'a' && ch <= 'z' || ch >= 'A' && ch <= 'Z') {
numberTokenStartEnable = false;
} else {
numberTokenStartEnable = true;
}
normalized.append(ch);
break;
}
}
if (change) {
parsingResult.setSql(normalized.toString());
return parsingResult;
} else {
// Reuse if not modified.
// 1. new strings are not generated
// 2. reuse hashcodes
parsingResult.setSql(sql);
return parsingResult;
}
}
private int readLine(String sql, StringBuilder normalized, int index) {
final int length = sql.length();
for (; index < length; index++) {
char ch = sql.charAt(index);
normalized.append(ch);
if (ch == '\n') {
break;
}
}
return index;
}
/**
* look up the next character in a string
*
* @param sql
* @param index
* @return
*/
private int lookAhead1(String sql, int index) {
index++;
if (index < sql.length()) {
return sql.charAt(index);
} else {
return NEXT_TOKEN_NOT_EXIST;
}
}
public String combineOutputParams(String sql, List<String> outputParams) {
final int length = sql.length();
final StringBuilder normalized = new StringBuilder(length + 16);
for (int i = 0; i < length; i++) {
final char ch = sql.charAt(i);
switch (ch) {
// COMMENT start check
case '/':
// comment state
int lookAhead1Char = lookAhead1(sql, i);
// multi line comment and oracle hint /*+ */
if (lookAhead1Char == '*') {
normalized.append("/*");
i += 2;
for (; i < length; i++) {
char stateCh = sql.charAt(i);
if (stateCh == '*') {
if (lookAhead1(sql, i) == '/') {
normalized.append("*/");
i++;
break;
}
}
normalized.append(stateCh);
}
break;
// single line comment
} else if (lookAhead1Char == '/') {
normalized.append("//");
i += 2;
i = readLine(sql, normalized, i);
break;
} else {
// unary operator
// numberTokenStartEnable = true;
normalized.append(ch);
break;
}
// case '#'
// # is a single line comment in mysql
case '-':
// single line comment state
if (lookAhead1(sql, i) == '-') {
normalized.append("--");
i += 2;
i = readLine(sql, normalized, i);
break;
} else {
// unary operator
// numberTokenStartEnable = true;
normalized.append(ch);
break;
}
// number start check
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
// http://www.h2database.com/html/grammar.html look at the state machine more
if (lookAhead1(sql, i) == NEXT_TOKEN_NOT_EXIST) {
normalized.append(ch);
break;
}
StringBuilder outputIndex = new StringBuilder();
outputIndex.append(ch);
// number token start
i++;
tokenEnd:
for (; i < length; i++) {
char stateCh = sql.charAt(i);
switch (stateCh) {
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
if (lookAhead1(sql, i) == NEXT_TOKEN_NOT_EXIST) {
outputIndex.append(stateCh);
normalized.append(outputIndex.toString());
break tokenEnd;
}
outputIndex.append(stateCh);
break;
case NUMBER_REPLACE:
int numberIndex = 0;
try {
numberIndex = Integer.parseInt(outputIndex.toString());
} catch (NumberFormatException e) {
// just append for invalid parameters
normalized.append(outputIndex.toString());
normalized.append(NUMBER_REPLACE);
break tokenEnd;
}
try {
String replaceNumber = outputParams.get(numberIndex);
normalized.append(replaceNumber);
} catch (IndexOutOfBoundsException e) {
// just append for invalid parameters
normalized.append(outputIndex.toString());
normalized.append(NUMBER_REPLACE);
break tokenEnd;
}
break tokenEnd;
case SYMBOL_REPLACE:
int symbolIndex = 0;
try {
symbolIndex = Integer.parseInt(outputIndex.toString());
} catch (NumberFormatException e) {
// just append for invalid parameters
normalized.append(outputIndex.toString());
normalized.append(SYMBOL_REPLACE);
}
try {
String replaceSymbol = outputParams.get(symbolIndex);
normalized.append(replaceSymbol);
} catch (IndexOutOfBoundsException e) {
normalized.append(outputIndex.toString());
normalized.append(SYMBOL_REPLACE);
}
break tokenEnd;
default:
// should look at the token outside the loop - not here
// outputParam.append(SEPARATOR);
normalized.append(outputIndex.toString());
i--;
break tokenEnd;
}
}
break;
default:
normalized.append(ch);
break;
}
}
return normalized.toString();
}
NormalizedSql normalizedSql(String sql);
String combineOutputParams(String sql, List<String> outputParams);
}
@@ -22,33 +22,29 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.navercorp.pinpoint.common.util.OutputParameterParser;
import com.navercorp.pinpoint.common.util.ParsingResult;
import com.navercorp.pinpoint.common.util.SqlParser;
import java.util.List;
/**
* @author emeroad
*/
public class SqlParserTest {
public class DefaultSqlParserTest {
private final Logger logger = LoggerFactory.getLogger(this.getClass());
private SqlParser sqlParser = new SqlParser();
private SqlParser sqlParser = new DefaultSqlParser();
private OutputParameterParser outputParameterParser = new OutputParameterParser();
@Test
public void normalizedSql() {
ParsingResult parsingResult = sqlParser.normalizedSql("select * from table a = 1 and b=50 and c=? and d='11'");
String s = parsingResult.getSql();
NormalizedSql parsingResult = sqlParser.normalizedSql("select * from table a = 1 and b=50 and c=? and d='11'");
String s = parsingResult.getNormalizedSql();
logger.debug(s);
logger.debug(parsingResult.getOutput());
logger.debug(parsingResult.getParseParameter());
ParsingResult parsingResult2 = sqlParser.normalizedSql(" ");
String s2 = parsingResult2.getSql();
NormalizedSql parsingResult2 = sqlParser.normalizedSql(" ");
String s2 = parsingResult2.getNormalizedSql();
logger.debug(s2);
logger.debug("{}", (char) -1);
@@ -60,10 +56,10 @@ public class SqlParserTest {
logger.debug("{}", (int) Character.MIN_LOW_SURROGATE);
logger.debug("{}", (int) Character.MAX_HIGH_SURROGATE);
ParsingResult parsingResult3 = sqlParser.normalizedSql("''");
String s3 = parsingResult3.getSql();
NormalizedSql parsingResult3 = sqlParser.normalizedSql("''");
String s3 = parsingResult3.getNormalizedSql();
logger.debug("s3:{}", s3);
logger.debug("sb3:{}", parsingResult3.getOutput());
logger.debug("sb3:{}", parsingResult3.getParseParameter());
}
@Test
@@ -226,7 +222,7 @@ public class SqlParserTest {
assertEqual("'1234 456,7'", "'0$'", "1234 456,,7");
assertEqual("'1234''456,7'", "'0$'", "1234''456,,7");
ParsingResult parsingResult2 = this.sqlParser.normalizedSql("'1234''456,7'");
NormalizedSql parsingResult2 = this.sqlParser.normalizedSql("'1234''456,7'");
logger.debug("{}", parsingResult2);
// for string token
@@ -258,8 +254,8 @@ public class SqlParserTest {
private void assertCombine(String result, String sql, String outputParams) {
List<String> output = this.outputParameterParser.parseOutputParameter(outputParams);
ParsingResult parsingResult = this.sqlParser.normalizedSql(result);
Assert.assertEquals("sql", parsingResult.getSql(), sql);
NormalizedSql parsingResult = this.sqlParser.normalizedSql(result);
Assert.assertEquals("sql", parsingResult.getNormalizedSql(), sql);
String combine = this.sqlParser.combineOutputParams(sql, output);
Assert.assertEquals("combine", result, combine);
}
@@ -273,8 +269,8 @@ public class SqlParserTest {
private void assertEqual(String expected) {
ParsingResult parsingResult = sqlParser.normalizedSql(expected);
String normalizedSql = parsingResult.getSql();
NormalizedSql parsingResult = sqlParser.normalizedSql(expected);
String normalizedSql = parsingResult.getNormalizedSql();
try {
Assert.assertEquals(expected, normalizedSql);
} catch (AssertionError e) {
@@ -284,8 +280,8 @@ public class SqlParserTest {
}
private void assertEqual(String expected, String actual) {
ParsingResult parsingResult = sqlParser.normalizedSql(expected);
String normalizedSql = parsingResult.getSql();
NormalizedSql parsingResult = sqlParser.normalizedSql(expected);
String normalizedSql = parsingResult.getNormalizedSql();
try {
Assert.assertEquals(actual, normalizedSql);
} catch (AssertionError e) {
@@ -295,9 +291,9 @@ public class SqlParserTest {
}
private void assertEqual(String expected, String actual, String outputExpected) {
ParsingResult parsingResult = sqlParser.normalizedSql(expected);
String normalizedSql = parsingResult.getSql();
String output = parsingResult.getOutput();
NormalizedSql parsingResult = sqlParser.normalizedSql(expected);
String normalizedSql = parsingResult.getNormalizedSql();
String output = parsingResult.getParseParameter();
List<String> outputParams = outputParameterParser.parseOutputParameter(output);
String s = sqlParser.combineOutputParams(normalizedSql, outputParams);
logger.debug("combine:" + s);
@@ -308,12 +304,12 @@ public class SqlParserTest {
throw e;
}
Assert.assertEquals("outputParam check", outputExpected, parsingResult.getOutput());
Assert.assertEquals("outputParam check", outputExpected, parsingResult.getParseParameter());
}
private void assertEqualObject(String expected) {
ParsingResult parsingResult = sqlParser.normalizedSql(expected);
String normalizedSql = parsingResult.getSql();
NormalizedSql parsingResult = sqlParser.normalizedSql(expected);
String normalizedSql = parsingResult.getNormalizedSql();
try {
Assert.assertEquals("normalizedSql check", expected, normalizedSql);
Assert.assertSame(expected, normalizedSql);
@@ -26,7 +26,7 @@ import com.navercorp.pinpoint.bootstrap.plugin.annotation.Name;
import com.navercorp.pinpoint.bootstrap.plugin.annotation.TargetMethod;
import com.navercorp.pinpoint.bootstrap.plugin.annotation.Targets;
import com.navercorp.pinpoint.bootstrap.util.InterceptorUtils;
import com.navercorp.pinpoint.common.util.ParsingResult;
import com.navercorp.pinpoint.bootstrap.context.ParsingResult;
import com.navercorp.pinpoint.plugin.jdbc.common.JdbcDriverConstants;
import com.navercorp.pinpoint.plugin.jdbc.common.UnKnownDatabaseInfo;
@@ -31,7 +31,7 @@ import com.navercorp.pinpoint.bootstrap.logging.PLoggerFactory;
import com.navercorp.pinpoint.bootstrap.plugin.annotation.Name;
import com.navercorp.pinpoint.bootstrap.plugin.annotation.TargetMethod;
import com.navercorp.pinpoint.bootstrap.plugin.annotation.Targets;
import com.navercorp.pinpoint.common.util.ParsingResult;
import com.navercorp.pinpoint.bootstrap.context.ParsingResult;
import com.navercorp.pinpoint.plugin.jdbc.common.JdbcDriverConstants;
import com.navercorp.pinpoint.plugin.jdbc.common.UnKnownDatabaseInfo;
import com.navercorp.pinpoint.plugin.jdbc.common.bindvalue.BindValueUtils;
@@ -0,0 +1,28 @@
/*
* Copyright 2014 NAVER Corp.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.navercorp.pinpoint.profiler.context;
import com.navercorp.pinpoint.bootstrap.context.ParsingResult;
/**
* @author emeroad
*/
public interface CachingSqlNormalizer {
ParsingResult wrapSql(String sql);
boolean normalizedSql(ParsingResult sql);
}
@@ -0,0 +1,94 @@
/*
* Copyright 2014 NAVER Corp.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.navercorp.pinpoint.profiler.context;
import com.navercorp.pinpoint.bootstrap.context.ParsingResult;
import com.navercorp.pinpoint.common.util.*;
import com.navercorp.pinpoint.profiler.metadata.Result;
import com.navercorp.pinpoint.profiler.metadata.SimpleCache;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* @author emeroad
*/
public class DefaultCachingSqlNormalizer implements CachingSqlNormalizer {
private final Logger logger = LoggerFactory.getLogger(this.getClass());
private static final DefaultParsingResult EMPTY_OBJECT = new DefaultParsingResult("");
private final SimpleCache<String> sqlCache;
private final SqlParser sqlParser;
public DefaultCachingSqlNormalizer(int cacheSize) {
this.sqlCache = new SimpleCache<String>(cacheSize);
this.sqlParser = new DefaultSqlParser();
}
@Override
public ParsingResult wrapSql(String sql) {
if (sql == null) {
return EMPTY_OBJECT;
}
return new DefaultParsingResult(sql);
}
@Override
public boolean normalizedSql(ParsingResult parsingResult) {
if (parsingResult == null) {
return false;
}
if (parsingResult == EMPTY_OBJECT) {
return false;
}
if (parsingResult.getId() != ParsingResult.ID_NOT_EXIST) {
// already cached
return false;
}
if (!(parsingResult instanceof ParsingResultInternal)) {
if (logger.isWarnEnabled()) {
logger.warn("unsupported ParsingResult Type type {}");
}
throw new IllegalArgumentException("unsupported ParsingResult Type");
}
final ParsingResultInternal parsingResultInternal = (ParsingResultInternal) parsingResult;
final String originalSql = parsingResultInternal.getOriginalSql();
final NormalizedSql normalizedSql = this.sqlParser.normalizedSql(originalSql);
final Result cachingResult = this.sqlCache.put(normalizedSql.getNormalizedSql());
// set normalizedSql
// set sqlId
final boolean success = parsingResultInternal.setId(cachingResult.getId());
if (!success) {
if (logger.isWarnEnabled()) {
logger.warn("invalid state. setSqlId fail setId:{}, ParsingResultInternal:{}", cachingResult.getId(), parsingResultInternal);
}
}
parsingResultInternal.setSql(normalizedSql.getNormalizedSql());
parsingResultInternal.setOutput(normalizedSql.getParseParameter());
return cachingResult.isNewValue();
}
}
@@ -0,0 +1,93 @@
/*
* Copyright 2014 NAVER Corp.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.navercorp.pinpoint.profiler.context;
import com.navercorp.pinpoint.bootstrap.context.ParsingResult;
/**
* @author emeroad
*/
public class DefaultParsingResult implements ParsingResultInternal {
private String originalSql;
private String sql ;
private String output;
private int id = ParsingResult.ID_NOT_EXIST;
public DefaultParsingResult(String originalSql) {
this.originalSql = originalSql;
}
public String getOriginalSql() {
return originalSql;
}
@Override
public String getSql() {
return sql;
}
@Override
public void setSql(String sql) {
this.sql = sql;
}
@Override
public void setOutput(String output) {
this.output = output;
}
@Override
public int getId() {
return id;
}
public boolean setId(int id) {
// clear originalSql reference
this.originalSql = null;
if (this.id == ID_NOT_EXIST) {
this.id = id;
return true;
}
return false;
}
@Override
public String getOutput() {
if (this.output == null) {
return "";
}
return this.output;
}
@Override
public String toString() {
final StringBuilder sb = new StringBuilder("DefaultParsingResult{");
sb.append("sql='").append(sql).append('\'');
sb.append(", output=").append(output);
sb.append(", id=").append(id);
sb.append('}');
return sb.toString();
}
}
@@ -22,8 +22,7 @@ import com.navercorp.pinpoint.bootstrap.context.*;
import com.navercorp.pinpoint.bootstrap.interceptor.MethodDescriptor;
import com.navercorp.pinpoint.bootstrap.sampler.Sampler;
import com.navercorp.pinpoint.common.trace.ServiceType;
import com.navercorp.pinpoint.common.util.DefaultParsingResult;
import com.navercorp.pinpoint.common.util.ParsingResult;
import com.navercorp.pinpoint.bootstrap.context.ParsingResult;
import com.navercorp.pinpoint.common.util.SqlParser;
import com.navercorp.pinpoint.profiler.AgentInformation;
import com.navercorp.pinpoint.profiler.context.active.ActiveTraceFactory;
@@ -66,8 +65,7 @@ public class DefaultTraceContext implements TraceContext {
private EnhancedDataSender priorityDataSender;
private final SimpleCache<String> sqlCache;
private final SqlParser sqlParser = new SqlParser();
private final CachingSqlNormalizer cachingSqlNormalizer;
private final SimpleCache<String> apiCache = new SimpleCache<String>();
private final SimpleCache<String> stringCache = new SimpleCache<String>();
@@ -96,7 +94,8 @@ public class DefaultTraceContext implements TraceContext {
throw new NullPointerException("sampler must not be null");
}
this.agentInformation = agentInformation;
this.sqlCache = new SimpleCache<String>(sqlCacheSize);
this.cachingSqlNormalizer = new DefaultCachingSqlNormalizer(sqlCacheSize);
this.traceFactory = createTraceFactory(storageFactory, sampler, traceActiveThread);
@@ -264,9 +263,8 @@ public class DefaultTraceContext implements TraceContext {
@Override
public ParsingResult parseSql(final String sql) {
final DefaultParsingResult parsingResult = this.sqlParser.normalizedSql(sql);
return parsingResult;
// lazy sql normalization
return this.cachingSqlNormalizer.wrapSql(sql);
}
@Override
@@ -274,14 +272,9 @@ public class DefaultTraceContext implements TraceContext {
if (parsingResult == null) {
return false;
}
if (parsingResult.getId() != ParsingResult.ID_NOT_EXIST) {
// already cached
return false;
}
final String normalizedSql = parsingResult.getSql();
final Result cachingResult = this.sqlCache.put(normalizedSql);
if (cachingResult.isNewValue()) {
// lazy sql parsing
boolean isNewValue = this.cachingSqlNormalizer.normalizedSql(parsingResult);
if (isNewValue) {
if (isDebug) {
// TODO logging hit ratio could help debugging
logger.debug("NewSQLParsingResult:{}", parsingResult);
@@ -293,12 +286,12 @@ public class DefaultTraceContext implements TraceContext {
sqlMetaData.setAgentId(getAgentId());
sqlMetaData.setAgentStartTime(getAgentStartTime());
sqlMetaData.setSqlId(cachingResult.getId());
sqlMetaData.setSql(normalizedSql);
sqlMetaData.setSqlId(parsingResult.getId());
sqlMetaData.setSql(parsingResult.getSql());
this.priorityDataSender.request(sqlMetaData);
}
return parsingResult.setId(cachingResult.getId());
return isNewValue;
}
@Override
@@ -0,0 +1,35 @@
/*
* Copyright 2014 NAVER Corp.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.navercorp.pinpoint.profiler.context;
import com.navercorp.pinpoint.bootstrap.context.ParsingResult;
/**
* @author emeroad
*/
interface ParsingResultInternal extends ParsingResult {
String getOriginalSql();
boolean setId(int id);
void setSql(String sql);
void setOutput(String output);
}
@@ -22,7 +22,7 @@ import com.navercorp.pinpoint.bootstrap.context.SpanEventRecorder;
import com.navercorp.pinpoint.bootstrap.context.TraceContext;
import com.navercorp.pinpoint.common.trace.AnnotationKey;
import com.navercorp.pinpoint.common.trace.ServiceType;
import com.navercorp.pinpoint.common.util.ParsingResult;
import com.navercorp.pinpoint.bootstrap.context.ParsingResult;
import com.navercorp.pinpoint.thrift.dto.TIntStringStringValue;
/**
@@ -17,14 +17,13 @@
package com.navercorp.pinpoint.profiler.modifier.db.interceptor;
import com.navercorp.pinpoint.bootstrap.context.DatabaseInfo;
import com.navercorp.pinpoint.bootstrap.context.RecordableTrace;
import com.navercorp.pinpoint.bootstrap.context.SpanEventRecorder;
import com.navercorp.pinpoint.bootstrap.interceptor.*;
import com.navercorp.pinpoint.bootstrap.interceptor.tracevalue.DatabaseInfoTraceValue;
import com.navercorp.pinpoint.bootstrap.interceptor.tracevalue.DatabaseInfoTraceValueUtils;
import com.navercorp.pinpoint.bootstrap.interceptor.tracevalue.ParsingResultTraceValue;
import com.navercorp.pinpoint.bootstrap.util.InterceptorUtils;
import com.navercorp.pinpoint.common.util.ParsingResult;
import com.navercorp.pinpoint.bootstrap.context.ParsingResult;
/**
* @author emeroad
@@ -32,7 +32,7 @@ import com.navercorp.pinpoint.bootstrap.interceptor.tracevalue.DatabaseInfoTrace
import com.navercorp.pinpoint.bootstrap.interceptor.tracevalue.ParsingResultTraceValue;
import com.navercorp.pinpoint.bootstrap.logging.PLogger;
import com.navercorp.pinpoint.bootstrap.logging.PLoggerFactory;
import com.navercorp.pinpoint.common.util.ParsingResult;
import com.navercorp.pinpoint.bootstrap.context.ParsingResult;
/**
* @author emeroad
@@ -14,8 +14,9 @@
* limitations under the License.
*/
package com.navercorp.pinpoint.common.util;
package com.navercorp.pinpoint.profiler.context;
import com.navercorp.pinpoint.bootstrap.context.ParsingResult;
import org.junit.Assert;
import org.junit.Test;
@@ -23,7 +24,7 @@ public class DefaultParsingResultTest {
@Test
public void testId() throws Exception {
DefaultParsingResult result = new DefaultParsingResult();
DefaultParsingResult result = new DefaultParsingResult("");
Assert.assertEquals(ParsingResult.ID_NOT_EXIST, result.getId());
// update
@@ -32,5 +33,5 @@ public class DefaultParsingResultTest {
// already updated
Assert.assertFalse(result.setId(1));
}
}
}
@@ -26,7 +26,7 @@ import com.navercorp.pinpoint.bootstrap.context.TraceId;
import com.navercorp.pinpoint.bootstrap.context.TraceType;
import com.navercorp.pinpoint.bootstrap.interceptor.MethodDescriptor;
import com.navercorp.pinpoint.common.trace.ServiceType;
import com.navercorp.pinpoint.common.util.ParsingResult;
import com.navercorp.pinpoint.bootstrap.context.ParsingResult;
/**
* @author emeroad
@@ -22,6 +22,7 @@ import java.util.List;
import com.navercorp.pinpoint.common.bo.*;
import com.navercorp.pinpoint.common.trace.AnnotationKey;
import com.navercorp.pinpoint.common.util.DefaultSqlParser;
import com.navercorp.pinpoint.common.util.OutputParameterParser;
import com.navercorp.pinpoint.common.util.SqlParser;
import com.navercorp.pinpoint.web.calltree.span.CallTree;
@@ -62,7 +63,7 @@ public class SpanServiceImpl implements SpanService {
@Autowired
private StringMetaDataDao stringMetaDataDao;
private final SqlParser sqlParser = new SqlParser();
private final SqlParser sqlParser = new DefaultSqlParser();
private final OutputParameterParser outputParameterParser = new OutputParameterParser();
@Override