From 6db8204d1c90a7813cde8877ee8471077934c423 Mon Sep 17 00:00:00 2001 From: Jordan Moore Date: Mon, 28 Jan 2019 16:35:49 -0600 Subject: [PATCH] Commit code --- .gitignore | 29 ++ .mvn/wrapper/MavenWrapperDownloader.java | 110 +++++++ .mvn/wrapper/maven-wrapper.properties | 1 + README.md | 87 ++++++ mvnw | 286 ++++++++++++++++++ mvnw.cmd | 161 ++++++++++ pom.xml | 199 ++++++++++++ .../transforms/SchemaRegistryTransfer.java | 241 +++++++++++++++ 8 files changed, 1114 insertions(+) create mode 100644 .gitignore create mode 100755 .mvn/wrapper/MavenWrapperDownloader.java create mode 100755 .mvn/wrapper/maven-wrapper.properties create mode 100644 README.md create mode 100755 mvnw create mode 100755 mvnw.cmd create mode 100644 pom.xml create mode 100644 src/main/java/cricket/jmoore/kafka/connect/transforms/SchemaRegistryTransfer.java diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..31aed55 --- /dev/null +++ b/.gitignore @@ -0,0 +1,29 @@ +.idea/ +*.iml + +*.class + +*.log + +*.jar +*.zip +*.tar.gz +*.rar + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* + +### macOS ### +.DS_Store + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +### Maven ### +target/ +dependency-reduced-pom.xml +.mvn/timing.properties +.mvn/wrapper/maven-wrapper.jar diff --git a/.mvn/wrapper/MavenWrapperDownloader.java b/.mvn/wrapper/MavenWrapperDownloader.java new file mode 100755 index 0000000..fa4f7b4 --- /dev/null +++ b/.mvn/wrapper/MavenWrapperDownloader.java @@ -0,0 +1,110 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + +import java.net.*; +import java.io.*; +import java.nio.channels.*; +import java.util.Properties; + +public class MavenWrapperDownloader { + + /** + * Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided. + */ + private static final String DEFAULT_DOWNLOAD_URL = + "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.4.2/maven-wrapper-0.4.2.jar"; + + /** + * Path to the maven-wrapper.properties file, which might contain a downloadUrl property to + * use instead of the default one. + */ + private static final String MAVEN_WRAPPER_PROPERTIES_PATH = + ".mvn/wrapper/maven-wrapper.properties"; + + /** + * Path where the maven-wrapper.jar will be saved to. + */ + private static final String MAVEN_WRAPPER_JAR_PATH = + ".mvn/wrapper/maven-wrapper.jar"; + + /** + * Name of the property which should be used to override the default download url for the wrapper. + */ + private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl"; + + public static void main(String args[]) { + System.out.println("- Downloader started"); + File baseDirectory = new File(args[0]); + System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath()); + + // If the maven-wrapper.properties exists, read it and check if it contains a custom + // wrapperUrl parameter. + File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH); + String url = DEFAULT_DOWNLOAD_URL; + if(mavenWrapperPropertyFile.exists()) { + FileInputStream mavenWrapperPropertyFileInputStream = null; + try { + mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile); + Properties mavenWrapperProperties = new Properties(); + mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream); + url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url); + } catch (IOException e) { + System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'"); + } finally { + try { + if(mavenWrapperPropertyFileInputStream != null) { + mavenWrapperPropertyFileInputStream.close(); + } + } catch (IOException e) { + // Ignore ... + } + } + } + System.out.println("- Downloading from: : " + url); + + File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH); + if(!outputFile.getParentFile().exists()) { + if(!outputFile.getParentFile().mkdirs()) { + System.out.println( + "- ERROR creating output direcrory '" + outputFile.getParentFile().getAbsolutePath() + "'"); + } + } + System.out.println("- Downloading to: " + outputFile.getAbsolutePath()); + try { + downloadFileFromURL(url, outputFile); + System.out.println("Done"); + System.exit(0); + } catch (Throwable e) { + System.out.println("- Error downloading"); + e.printStackTrace(); + System.exit(1); + } + } + + private static void downloadFileFromURL(String urlString, File destination) throws Exception { + URL website = new URL(urlString); + ReadableByteChannel rbc; + rbc = Channels.newChannel(website.openStream()); + FileOutputStream fos = new FileOutputStream(destination); + fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); + fos.close(); + rbc.close(); + } + +} diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties new file mode 100755 index 0000000..00d32aa --- /dev/null +++ b/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1 @@ +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.5.4/apache-maven-3.5.4-bin.zip \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..6581df5 --- /dev/null +++ b/README.md @@ -0,0 +1,87 @@ +Schema Registry Transfer SMT +============================ + +A [Kafka Connect Single Message Transformation (SMT)][smt] that reads the serialized [wire format header][wire-format] of Confluent's `KafkaAvroSerializer`, performs a lookup against a source [Confluent Schema Registry][schema-registry] for the ID in the message, and registers that schema into a destination Registry for that topic/subject under a new ID. + +To be used where it is not feasible to make the destination Schema Registry as a follower to the source Registry, or when migrating topics to a new cluster. + +> _Requires that the Kafka Connect tasks can reach both Schema Registries._ + +Example Kafka Connectors where this could be applied. + +- [Comcast/MirrorTool-for-Kafka-Connect](https://github.com/Comcast/MirrorTool-for-Kafka-Connect) - Code was tested with this first, and verified that the topic-renaming logic of this connector worked fine with this SMT. +- [Salesforce/mirus](https://github.com/salesforce/mirus) +- [Confluent Replicator](https://docs.confluent.io/current/connect/kafka-connect-replicator/index.html) - While this already can copy the schema, we observed it is only possible via the `AvroConverter`, which must first parse the entire message into a Kafka Connect `Struct` object. Thus, the class here is considered a "shallow" copier — it only inspects [the first 5 bytes][wire-format] of the keys and values for the schema ids. +- [KIP-382 (MirrorMaker 2.0)](https://cwiki.apache.org/confluence/display/KAFKA/KIP-382%3A+MirrorMaker+2.0) - Still open at the time of writing. + + +## Installation + +1. Edit the Kafka Connect worker properties file on each worker to include a new directory. For example, `/opt/kafka-connect/plugins` + +```sh +plugin.path=/usr/share/java,/opt/kafka-connect/plugins +``` + +2. Build this project + +```sh +./mvnw clean package +``` + +3. Copy the JAR from `target` to all Kafka Connect workers under a directory set by `plugin.path` + +4. (Re)start Kafka Connect processes + +## Usage + +Standalone Kafka Connect configuration section + +```properties +# Requires that records are entirely byte-arrays. These can go in the worker or connector configuration. +key.converter=org.apache.kafka.connect.converters.ByteArrayConverter +value.converter=org.apache.kafka.connect.converters.ByteArrayConverter + +# Setup the SMT +transforms=AvroSchemaTransfer + +transforms.AvroSchemaTransfer.type=cricket.jmoore.kafka.connect.transforms.SchemaRegistryTransfer +transforms.AvroSchemaTransfer.src.schema.registry.url=http://schema-registry-1:8081 +transforms.AvroSchemaTransfer.dest.schema.registry.url=http://schema-registry-2:8081 +``` + +Distributed Kafka Connect configuration section + +```json +"config" : { + ... + + "__comment": "Requires that records are entirely byte-arrays. These can go in the worker or connector configuration.", + "key.converter": "org.apache.kafka.connect.converters.ByteArrayConverter", + "value.converter": "org.apache.kafka.connect.converters.ByteArrayConverter", + + "__comment": "Setup the SMT", + "transforms": "AvroSchemaTransfer", + + "transforms.AvroSchemaTransfer.type": "cricket.jmoore.kafka.connect.transforms.SchemaRegistryTransfer", + "transforms.AvroSchemaTransfer.src.schema.registry.url": "http://schema-registry-1:8081", + "transforms.AvroSchemaTransfer.dest.schema.registry.url": "http://schema-registry-2:8081" +} +``` + +## Advanced Configuration + +Configuration Parameter | Default | Description +----------------------- | ------- | ----------- +**include.message.keys** | true | Indicates whether Avro schemas from message keys in source records should be copied to the destination Registry. +**include.message.headers** | true | Indicates whether message headers from source records should be preserved after the transform. +**schema.capacity** | 100 | Capacity of schemas that can be cached in each `CachedSchemaRegistryClient` + + + + + + + [smt]: https://docs.confluent.io/current/connect/concepts.html#connect-transforms + [schema-registry]: https://docs.confluent.io/current/schema-registry/docs/index.html + [wire-format]: https://docs.confluent.io/current/schema-registry/docs/serializer-formatter.html#wire-format \ No newline at end of file diff --git a/mvnw b/mvnw new file mode 100755 index 0000000..5551fde --- /dev/null +++ b/mvnw @@ -0,0 +1,286 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Maven2 Start Up Batch script +# +# Required ENV vars: +# ------------------ +# JAVA_HOME - location of a JDK home dir +# +# Optional ENV vars +# ----------------- +# M2_HOME - location of maven2's installed home dir +# MAVEN_OPTS - parameters passed to the Java VM when running Maven +# e.g. to debug Maven itself, use +# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +# MAVEN_SKIP_RC - flag to disable loading of mavenrc files +# ---------------------------------------------------------------------------- + +if [ -z "$MAVEN_SKIP_RC" ] ; then + + if [ -f /etc/mavenrc ] ; then + . /etc/mavenrc + fi + + if [ -f "$HOME/.mavenrc" ] ; then + . "$HOME/.mavenrc" + fi + +fi + +# OS specific support. $var _must_ be set to either true or false. +cygwin=false; +darwin=false; +mingw=false +case "`uname`" in + CYGWIN*) cygwin=true ;; + MINGW*) mingw=true;; + Darwin*) darwin=true + # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home + # See https://developer.apple.com/library/mac/qa/qa1170/_index.html + if [ -z "$JAVA_HOME" ]; then + if [ -x "/usr/libexec/java_home" ]; then + export JAVA_HOME="`/usr/libexec/java_home`" + else + export JAVA_HOME="/Library/Java/Home" + fi + fi + ;; +esac + +if [ -z "$JAVA_HOME" ] ; then + if [ -r /etc/gentoo-release ] ; then + JAVA_HOME=`java-config --jre-home` + fi +fi + +if [ -z "$M2_HOME" ] ; then + ## resolve links - $0 may be a link to maven's home + PRG="$0" + + # need this for relative symlinks + while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG="`dirname "$PRG"`/$link" + fi + done + + saveddir=`pwd` + + M2_HOME=`dirname "$PRG"`/.. + + # make it fully qualified + M2_HOME=`cd "$M2_HOME" && pwd` + + cd "$saveddir" + # echo Using m2 at $M2_HOME +fi + +# For Cygwin, ensure paths are in UNIX format before anything is touched +if $cygwin ; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --unix "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --unix "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --unix "$CLASSPATH"` +fi + +# For Mingw, ensure paths are in UNIX format before anything is touched +if $mingw ; then + [ -n "$M2_HOME" ] && + M2_HOME="`(cd "$M2_HOME"; pwd)`" + [ -n "$JAVA_HOME" ] && + JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" + # TODO classpath? +fi + +if [ -z "$JAVA_HOME" ]; then + javaExecutable="`which javac`" + if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then + # readlink(1) is not available as standard on Solaris 10. + readLink=`which readlink` + if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then + if $darwin ; then + javaHome="`dirname \"$javaExecutable\"`" + javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" + else + javaExecutable="`readlink -f \"$javaExecutable\"`" + fi + javaHome="`dirname \"$javaExecutable\"`" + javaHome=`expr "$javaHome" : '\(.*\)/bin'` + JAVA_HOME="$javaHome" + export JAVA_HOME + fi + fi +fi + +if [ -z "$JAVACMD" ] ; then + if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD="`which java`" + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." >&2 + echo " We cannot execute $JAVACMD" >&2 + exit 1 +fi + +if [ -z "$JAVA_HOME" ] ; then + echo "Warning: JAVA_HOME environment variable is not set." +fi + +CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher + +# traverses directory structure from process work directory to filesystem root +# first directory with .mvn subdirectory is considered project base directory +find_maven_basedir() { + + if [ -z "$1" ] + then + echo "Path not specified to find_maven_basedir" + return 1 + fi + + basedir="$1" + wdir="$1" + while [ "$wdir" != '/' ] ; do + if [ -d "$wdir"/.mvn ] ; then + basedir=$wdir + break + fi + # workaround for JBEAP-8937 (on Solaris 10/Sparc) + if [ -d "${wdir}" ]; then + wdir=`cd "$wdir/.."; pwd` + fi + # end of workaround + done + echo "${basedir}" +} + +# concatenates all lines of a file +concat_lines() { + if [ -f "$1" ]; then + echo "$(tr -s '\n' ' ' < "$1")" + fi +} + +BASE_DIR=`find_maven_basedir "$(pwd)"` +if [ -z "$BASE_DIR" ]; then + exit 1; +fi + +########################################################################################## +# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +# This allows using the maven wrapper in projects that prohibit checking in binary data. +########################################################################################## +if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found .mvn/wrapper/maven-wrapper.jar" + fi +else + if [ "$MVNW_VERBOSE" = true ]; then + echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..." + fi + jarUrl="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.4.2/maven-wrapper-0.4.2.jar" + while IFS="=" read key value; do + case "$key" in (wrapperUrl) jarUrl="$value"; break ;; + esac + done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties" + if [ "$MVNW_VERBOSE" = true ]; then + echo "Downloading from: $jarUrl" + fi + wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" + + if command -v wget > /dev/null; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found wget ... using wget" + fi + wget "$jarUrl" -O "$wrapperJarPath" + elif command -v curl > /dev/null; then + if [ "$MVNW_VERBOSE" = true ]; then + echo "Found curl ... using curl" + fi + curl -o "$wrapperJarPath" "$jarUrl" + else + if [ "$MVNW_VERBOSE" = true ]; then + echo "Falling back to using Java to download" + fi + javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java" + if [ -e "$javaClass" ]; then + if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then + if [ "$MVNW_VERBOSE" = true ]; then + echo " - Compiling MavenWrapperDownloader.java ..." + fi + # Compiling the Java class + ("$JAVA_HOME/bin/javac" "$javaClass") + fi + if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then + # Running the downloader + if [ "$MVNW_VERBOSE" = true ]; then + echo " - Running MavenWrapperDownloader.java ..." + fi + ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR") + fi + fi + fi +fi +########################################################################################## +# End of extension +########################################################################################## + +export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} +if [ "$MVNW_VERBOSE" = true ]; then + echo $MAVEN_PROJECTBASEDIR +fi +MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" + +# For Cygwin, switch paths to Windows format before running java +if $cygwin; then + [ -n "$M2_HOME" ] && + M2_HOME=`cygpath --path --windows "$M2_HOME"` + [ -n "$JAVA_HOME" ] && + JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` + [ -n "$CLASSPATH" ] && + CLASSPATH=`cygpath --path --windows "$CLASSPATH"` + [ -n "$MAVEN_PROJECTBASEDIR" ] && + MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` +fi + +WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +exec "$JAVACMD" \ + $MAVEN_OPTS \ + -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ + "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ + ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" diff --git a/mvnw.cmd b/mvnw.cmd new file mode 100755 index 0000000..48363fa --- /dev/null +++ b/mvnw.cmd @@ -0,0 +1,161 @@ +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Maven2 Start Up Batch script +@REM +@REM Required ENV vars: +@REM JAVA_HOME - location of a JDK home dir +@REM +@REM Optional ENV vars +@REM M2_HOME - location of maven2's installed home dir +@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands +@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending +@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven +@REM e.g. to debug Maven itself, use +@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 +@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files +@REM ---------------------------------------------------------------------------- + +@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' +@echo off +@REM set title of command window +title %0 +@REM enable echoing my setting MAVEN_BATCH_ECHO to 'on' +@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% + +@REM set %HOME% to equivalent of $HOME +if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") + +@REM Execute a user defined script before this one +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre +@REM check for pre script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" +if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" +:skipRcPre + +@setlocal + +set ERROR_CODE=0 + +@REM To isolate internal variables from possible post scripts, we use another setlocal +@setlocal + +@REM ==== START VALIDATION ==== +if not "%JAVA_HOME%" == "" goto OkJHome + +echo. +echo Error: JAVA_HOME not found in your environment. >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +:OkJHome +if exist "%JAVA_HOME%\bin\java.exe" goto init + +echo. +echo Error: JAVA_HOME is set to an invalid directory. >&2 +echo JAVA_HOME = "%JAVA_HOME%" >&2 +echo Please set the JAVA_HOME variable in your environment to match the >&2 +echo location of your Java installation. >&2 +echo. +goto error + +@REM ==== END VALIDATION ==== + +:init + +@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". +@REM Fallback to current working directory if not found. + +set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% +IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir + +set EXEC_DIR=%CD% +set WDIR=%EXEC_DIR% +:findBaseDir +IF EXIST "%WDIR%"\.mvn goto baseDirFound +cd .. +IF "%WDIR%"=="%CD%" goto baseDirNotFound +set WDIR=%CD% +goto findBaseDir + +:baseDirFound +set MAVEN_PROJECTBASEDIR=%WDIR% +cd "%EXEC_DIR%" +goto endDetectBaseDir + +:baseDirNotFound +set MAVEN_PROJECTBASEDIR=%EXEC_DIR% +cd "%EXEC_DIR%" + +:endDetectBaseDir + +IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig + +@setlocal EnableExtensions EnableDelayedExpansion +for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a +@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% + +:endReadAdditionalConfig + +SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" +set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" +set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain + +set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.4.2/maven-wrapper-0.4.2.jar" +FOR /F "tokens=1,2 delims==" %%A IN (%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties) DO ( + IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B +) + +@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central +@REM This allows using the maven wrapper in projects that prohibit checking in binary data. +if exist %WRAPPER_JAR% ( + echo Found %WRAPPER_JAR% +) else ( + echo Couldn't find %WRAPPER_JAR%, downloading it ... + echo Downloading from: %DOWNLOAD_URL% + powershell -Command "(New-Object Net.WebClient).DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')" + echo Finished downloading %WRAPPER_JAR% +) +@REM End of extension + +%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* +if ERRORLEVEL 1 goto error +goto end + +:error +set ERROR_CODE=1 + +:end +@endlocal & set ERROR_CODE=%ERROR_CODE% + +if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost +@REM check for post script, once with legacy .bat ending and once with .cmd ending +if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" +if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" +:skipRcPost + +@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' +if "%MAVEN_BATCH_PAUSE%" == "on" pause + +if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% + +exit /B %ERROR_CODE% diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..a2290b9 --- /dev/null +++ b/pom.xml @@ -0,0 +1,199 @@ + + + 4.0.0 + + cricket.jmoore + schema-registry-transfer-smt + 0.1.0 + + + + confluent + https://packages.confluent.io/maven + + + + + UTF-8 + 1.8 + 1.8 + + 2.1.0 + 5.1.0 + -cp1 + 2.9.7 + 1.9.13 + + cricket.jmoore + + + + + + + org.slf4j + slf4j-api + 1.7.25 + provided + + + + org.apache.kafka + connect-transforms + ${kafka.version}${confluent.patch.version} + provided + + + + org.apache.zookeeper + zookeeper + 3.4.13 + provided + + + log4j + log4j + + + io.netty + netty + + + + + + com.101tec + zkclient + 0.10 + provided + + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + provided + + + + org.codehaus.jackson + jackson-core-asl + ${jackson.asl.version} + provided + + + + org.codehaus.jackson + jackson-mapper-asl + ${jackson.asl.version} + provided + + + + com.thoughtworks.paranamer + paranamer + 2.7 + provided + + + + org.tukaani + xz + 1.5 + provided + + + + org.xerial.snappy + snappy-java + 1.1.7.1 + provided + + + + org.apache.commons + commons-compress + 1.8.1 + provided + + + + + org.apache.avro + avro + 1.8.1 + + + + io.confluent + kafka-avro-serializer + ${confluent.version} + + + + io.confluent + kafka-schema-registry-client + ${confluent.version} + + + org.apache.kafka + kafka-clients + + + com.fasterxml.jackson.core + jackson-databind + + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.1 + + false + + + *:* + + META-INF/maven/** + META-INF/LICENSE* + META-INF/NOTICE* + META-INF/DEPENDENCIES + + + + + + io.confluent + ${shade.prefix}.confluent + + + org.apache.avro + ${shade.prefix}.avro + + + avro.shaded + ${shade.prefix}.avroshaded + + + + + + package + + shade + + + + + + + + \ No newline at end of file diff --git a/src/main/java/cricket/jmoore/kafka/connect/transforms/SchemaRegistryTransfer.java b/src/main/java/cricket/jmoore/kafka/connect/transforms/SchemaRegistryTransfer.java new file mode 100644 index 0000000..e3b594e --- /dev/null +++ b/src/main/java/cricket/jmoore/kafka/connect/transforms/SchemaRegistryTransfer.java @@ -0,0 +1,241 @@ +package cricket.jmoore.kafka.connect.transforms; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +import org.apache.kafka.common.cache.Cache; +import org.apache.kafka.common.cache.LRUCache; +import org.apache.kafka.common.cache.SynchronizedCache; +import org.apache.kafka.common.config.ConfigDef; +import org.apache.kafka.common.errors.SerializationException; +import org.apache.kafka.connect.connector.ConnectRecord; +import org.apache.kafka.connect.data.Schema; +import org.apache.kafka.connect.errors.ConnectException; +import org.apache.kafka.connect.transforms.Transformation; +import org.apache.kafka.connect.transforms.util.NonEmptyListValidator; +import org.apache.kafka.connect.transforms.util.SimpleConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient; +import io.confluent.kafka.schemaregistry.client.rest.exceptions.RestClientException; +import io.confluent.kafka.serializers.AbstractKafkaAvroSerDeConfig; +import io.confluent.kafka.serializers.subject.TopicNameStrategy; +import io.confluent.kafka.serializers.subject.strategy.SubjectNameStrategy; + +@SuppressWarnings("unused") +public class SchemaRegistryTransfer> implements Transformation { + public static final String OVERVIEW_DOC = "Inspect the Confluent KafkaAvroSerializer's wire-format header to copy schemas from one Schema Registry to another."; + private static final Logger log = LoggerFactory.getLogger(SchemaRegistryTransfer.class); + + private static final byte MAGIC_BYTE = 0x0; + + public static final ConfigDef CONFIG_DEF; + public static final String SCHEMA_CAPACITY_CONFIG_DOC = "The maximum amount of schemas to be stored for each Schema Registry client."; + public static final Integer SCHEMA_CAPACITY_CONFIG_DEFAULT = 100; + public static final String SRC_SCHEMA_REGISTRY_CONFIG_DOC = "A list of addresses for the Schema Registry to copy from. The consumer's Schema Registry."; + public static final String DEST_SCHEMA_REGISTRY_CONFIG_DOC = "A list of addresses for the Schema Registry to copy to. The producer's Schema Registry."; + public static final String INCLUDE_KEYS_CONFIG_DOC = "Whether or not to copy message key schemas between registries."; + public static final Boolean INCLUDE_KEYS_CONFIG_DEFAULT = true; + public static final String INCLUDE_HEADERS_CONFIG_DOC = "Whether or not to preserve the Kafka Connect Record headers."; + public static final Boolean INCLUDE_HEADERS_CONFIG_DEFAULT = true; + + private CachedSchemaRegistryClient sourceSchemaRegistryClient; + private CachedSchemaRegistryClient destSchemaRegistryClient; + private SubjectNameStrategy subjectNameStrategy; + private boolean includeKeys, includeHeaders; + + // caches from the source registry to the destination registry + private Cache schemaCache; + + public SchemaRegistryTransfer() { + } + + static { + CONFIG_DEF = (new ConfigDef()) + .define(ConfigName.SRC_SCHEMA_REGISTRY_URL, ConfigDef.Type.LIST, ConfigDef.NO_DEFAULT_VALUE, new NonEmptyListValidator(), ConfigDef.Importance.HIGH, SRC_SCHEMA_REGISTRY_CONFIG_DOC) + .define(ConfigName.DEST_SCHEMA_REGISTRY_URL, ConfigDef.Type.LIST, ConfigDef.NO_DEFAULT_VALUE, new NonEmptyListValidator(), ConfigDef.Importance.HIGH, DEST_SCHEMA_REGISTRY_CONFIG_DOC) + .define(ConfigName.SCHEMA_CAPACITY, ConfigDef.Type.INT, SCHEMA_CAPACITY_CONFIG_DEFAULT, ConfigDef.Importance.LOW, SCHEMA_CAPACITY_CONFIG_DOC) + .define(ConfigName.INCLUDE_KEYS, ConfigDef.Type.BOOLEAN, INCLUDE_KEYS_CONFIG_DEFAULT, ConfigDef.Importance.MEDIUM, INCLUDE_KEYS_CONFIG_DOC) + .define(ConfigName.INCLUDE_HEADERS, ConfigDef.Type.BOOLEAN, INCLUDE_HEADERS_CONFIG_DEFAULT, ConfigDef.Importance.MEDIUM, INCLUDE_HEADERS_CONFIG_DOC) + ; + // TODO: Other properties might be useful, e.g. the Subject Strategies + } + + @Override + public ConfigDef config() { + return CONFIG_DEF; + } + + @Override + public void configure(Map props) { + SimpleConfig config = new SimpleConfig(CONFIG_DEF, props); + + List sourceUrls = config.getList(ConfigName.SRC_SCHEMA_REGISTRY_URL); + List destUrls = config.getList(ConfigName.DEST_SCHEMA_REGISTRY_URL); + Integer schemaCapacity = config.getInt(ConfigName.SCHEMA_CAPACITY); + + this.schemaCache = new SynchronizedCache<>(new LRUCache<>(schemaCapacity)); + this.sourceSchemaRegistryClient = new CachedSchemaRegistryClient(sourceUrls, schemaCapacity); + this.destSchemaRegistryClient = new CachedSchemaRegistryClient(destUrls, schemaCapacity); + + this.includeKeys = config.getBoolean(ConfigName.INCLUDE_KEYS); + this.includeHeaders = config.getBoolean(ConfigName.INCLUDE_HEADERS); + + // TODO: Make the Strategy configurable, may be different for src and dest + // Strategy for the -key and -value subjects + this.subjectNameStrategy = new TopicNameStrategy(); + } + + @Override + public R apply(R r) { + final String topic = r.topic(); + + // Transcribe the key's schema id + Object key = r.key(); + Schema keySchema = r.keySchema(); + + Object updatedKey = key; + Optional destKeySchemaId; + if (includeKeys) { + if ((keySchema != null && keySchema.type() == Schema.BYTES_SCHEMA.type()) || + key instanceof byte[]) { + ByteBuffer b = ByteBuffer.wrap((byte[]) key); + destKeySchemaId = copySchema(b, topic, true); + b.putInt(1, destKeySchemaId.orElseThrow(() + -> new ConnectException("Transform failed. Unable to update record schema id. (isKey=true)"))); + updatedKey = b.array(); + } else { + throw new ConnectException("Transform failed. Record key does not have a byte[] schema."); + } + } + + // Transcribe the value's schema id + Object value = r.value(); + Schema valueSchema = r.valueSchema(); + + Object updatedValue; + Optional destValueSchemaId; + if ((valueSchema != null && valueSchema.type() == Schema.BYTES_SCHEMA.type()) || + value instanceof byte[]) { + ByteBuffer b = ByteBuffer.wrap((byte[]) value); + destValueSchemaId = copySchema(b, topic, false); + b.putInt(1, destValueSchemaId.orElseThrow(() + -> new ConnectException("Transform failed. Unable to update record schema id. (isKey=false)"))); + updatedValue = b.array(); + } else { + throw new ConnectException("Transform failed. Record value does not have a byte[] schema."); + } + + + return includeHeaders ? + r.newRecord(topic, r.kafkaPartition(), + keySchema, includeKeys ? updatedKey : r.key(), + valueSchema, updatedValue, + r.timestamp(), + r.headers()) + : + r.newRecord(topic, r.kafkaPartition(), + keySchema, includeKeys ? updatedKey : r.key(), + valueSchema, updatedValue, + r.timestamp()); + } + + protected Optional copySchema(ByteBuffer buffer, String topic, boolean isKey) { + SchemaAndId schemaAndDestId; + if (buffer.get() == MAGIC_BYTE) { + int sourceSchemaId = buffer.getInt(); + + schemaAndDestId = schemaCache.get(sourceSchemaId); + if (schemaAndDestId != null) { + log.trace("Schema id {} has been seen before. Not registering with destination registry again."); + } else { // cache miss + log.trace("Schema id {} has not been seen before", sourceSchemaId); + schemaAndDestId = new SchemaAndId(); + try { + log.trace("Looking up schema id {} in source registry", sourceSchemaId); + // Can't do getBySubjectAndId because that requires a Schema object for the strategy + schemaAndDestId.schema = sourceSchemaRegistryClient.getById(sourceSchemaId); + } catch (IOException | RestClientException e) { + log.error(String.format("Unable to fetch source schema for id %d.", sourceSchemaId), e); + throw new ConnectException(e); + } + + if (schemaAndDestId.schema == null) { + String msg = "Error getting schema from source registry. Not registering null schema with destination registry."; + log.error(msg); + throw new ConnectException(msg); + } + + try { + log.trace("Registering schema {} to destination registry", schemaAndDestId.schema); + // It could be possible that the destination naming strategy is different from the source + String subjectName = subjectNameStrategy.subjectName(topic, isKey, schemaAndDestId.schema); + schemaAndDestId.id = destSchemaRegistryClient.register(subjectName, schemaAndDestId.schema); + schemaCache.put(sourceSchemaId, schemaAndDestId); + } catch (IOException | RestClientException e) { + log.error(String.format("Unable to register source schema id %d to destination registry.", + sourceSchemaId), e); + return Optional.empty(); + } + } + } else { + throw new SerializationException("Unknown magic byte!"); + } + return Optional.ofNullable(schemaAndDestId.id); + } + + @Override + public void close() { + this.sourceSchemaRegistryClient = null; + this.destSchemaRegistryClient = null; + } + + interface ConfigName { + String SRC_SCHEMA_REGISTRY_URL = "src." + AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG; + String DEST_SCHEMA_REGISTRY_URL = "dest." + AbstractKafkaAvroSerDeConfig.SCHEMA_REGISTRY_URL_CONFIG; + String SCHEMA_CAPACITY = "schema.capacity"; + String INCLUDE_KEYS = "include.message.keys"; + String INCLUDE_HEADERS = "include.message.headers"; + } + + private static class SchemaAndId { + private Integer id; + private org.apache.avro.Schema schema; + + SchemaAndId() { + } + + SchemaAndId(int id, org.apache.avro.Schema schema) { + this.id = id; + this.schema = schema; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + SchemaAndId schemaAndId = (SchemaAndId) o; + return Objects.equals(id, schemaAndId.id) && + Objects.equals(schema, schemaAndId.schema); + } + + @Override + public int hashCode() { + return Objects.hash(id, schema); + } + + @Override + public String toString() { + return "SchemaAndId{" + + "id=" + id + + ", schema=" + schema + + '}'; + } + } + +}