Merge pull request #98 from mojo-zd/ssh-generator

ssh keypair dynamic

docker/shadow/Dockerfile_base

@@ -4,9 +4,6 @@ RUN apt-get update && apt-get install -y openssh-server dnsutils sshuttle iputil
 RUN mkdir /var/run/sshd
 RUN echo 'root:root' | chpasswd
 RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
-RUN mkdir -p /root/.ssh && echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC9JUB7MkMGGs0OEuG4Joqu920IVN2omYYsbJ8GoRhvenhr3WCXiAPYsbCJGWCnBTihIQJ4ybA/ncLi+t88pjjn/7yWdwiQT/gT7yEeyxeQdqpCuMQW149/\
-Fnr/GLziRRtY01XggYB4XBe2r0azND6RYw1PlCa57E5c6WqrtbgIgBR7e5fb1YKnc4aIF4/g5EOtpFr/VvAEf4KrRTERhP7oS4NTRjMXckXgfwC3o12ASUC57MTAjzinks2vUeZM+d1wcm1fb484oSg1gHE9\
-6QTIluoTEhHZ0tFTK2Vfm0hpwqQwEIlpfsMG0nNZfsHF+NF9ePM06m5+5NT0GySfImUX kt@public_key" > /root/.ssh/authorized_keys && chmod 400 /root/.ssh/authorized_keys
 # SSH login fix. Otherwise user is kicked off after login
 RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
 

docker/shadow/run.sh

@@ -1,4 +1,7 @@
 #!/bin/bash
+mkdir -p /root/.ssh
+cp /root/authorized/authorized_keys /root/.ssh
+
 /usr/sbin/sshd -D &
 
 if [[ "${1}" = "--debug" ]]; then

go.mod

@@ -21,6 +21,7 @@ require (
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966
 	github.com/thinkerou/favicon v0.1.0 // indirect
 	github.com/urfave/cli v0.0.0-20190203184040-693af58b4d51
+	golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392
 	istio.io/api v0.0.0-20200221025927-228308df3f1b
 	istio.io/client-go v0.0.0-20200221055756-736d3076b458
 	k8s.io/api v0.17.2

pkg/apiserver/server/server.go

@@ -2,6 +2,7 @@ package server
 
 import "github.com/alibaba/kt-connect/pkg/apiserver/common"
 
+// Init ...
 func Init(context common.Context) {
 	r := NewRouter(context)
 	r.Run(":8000")

pkg/apiserver/util/cluster.go

@@ -10,6 +10,7 @@ import (
 	"k8s.io/client-go/tools/clientcmd"
 )
 
+// GetKubernetesClient ...
 func GetKubernetesClient() (clientset kubernetes.Interface, config *restclient.Config, err error) {
 	config, err = GetKubeconfig()
 	if err != nil {
@@ -21,6 +22,7 @@ func GetKubernetesClient() (clientset kubernetes.Interface, config *restclient.C
 	return
 }
 
+// GetKubeconfig ...
 func GetKubeconfig() (config *restclient.Config, err error) {
 	kubeconfig := filepath.Join(homeDir(), ".kube", "config")
 	if _, err := os.Stat(kubeconfig); os.IsNotExist(err) {

pkg/kt/cluster/kubernetes.go

@@ -1,18 +1,19 @@
 package cluster
 
 import (
+	"fmt"
 	"time"
 
-	"k8s.io/apimachinery/pkg/util/intstr"
-
 	clusterWatcher "github.com/alibaba/kt-connect/pkg/apiserver/cluster"
 	"github.com/alibaba/kt-connect/pkg/kt/util"
+	"github.com/alibaba/kt-connect/pkg/kt/vars"
 	"github.com/rs/zerolog/log"
 	appV1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/selection"
+	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/kubernetes"
 )
 
@@ -38,10 +39,17 @@ func (k *Kubernetes) Deployment(name, namespace string) (deployment *appV1.Deplo
 }
 
 // CreateShadow create shadow
-func (k *Kubernetes) CreateShadow(name, namespace, image string, labels map[string]string) (podIP, podName string, err error) {
+func (k *Kubernetes) CreateShadow(name, namespace, image string, labels map[string]string) (podIP, podName, sshcm string, credential *util.SSHCredential, err error) {
 	return CreateShadow(k.Clientset, name, labels, namespace, image)
 }
 
+// CreateService create kubernetes service
+func (k *Kubernetes) CreateService(name, namespace string, port int, labels map[string]string) (*v1.Service, error) {
+	cli := k.Clientset.CoreV1().Services(namespace)
+	svc := generateService(name, namespace, labels, port)
+	return cli.Create(svc)
+}
+
 // ClusterCrids get cluster cirds
 func (k *Kubernetes) ClusterCrids(podCIDR string) (cidrs []string, err error) {
 	serviceList, err := k.Clientset.CoreV1().Services("").List(metaV1.ListOptions{})
@@ -108,16 +116,15 @@ func RemoveShadow(client *kubernetes.Clientset, namespace, name string) {
 	}
 }
 
-// CreateService create service in cluster
-func CreateService(name, namespace string,
-	labels map[string]string,
-	port int,
-	clientset *kubernetes.Clientset,
-) (err error) {
-	client := clientset.CoreV1().Services(namespace)
-	svc := generateService(name, namespace, labels, port)
-	_, err = client.Create(svc)
-	return err
+// RemoveSSHCM remove ssh public key of config map
+func RemoveSSHCM(client *kubernetes.Clientset, namespace, name string) {
+	cli := client.CoreV1().ConfigMaps(namespace)
+	deletePolicy := metaV1.DeletePropagationBackground
+	if err := cli.Delete(name, &metaV1.DeleteOptions{
+		PropagationPolicy: &deletePolicy,
+	}); err != nil {
+		log.Error().Err(err).Str("config map", name).Msg("delete config map failed")
+	}
 }
 
 // RemoveService create service in cluster
@@ -136,14 +143,39 @@ func CreateShadow(
 	labels map[string]string,
 	namespace,
 	image string,
-) (podIP, podName string, err error) {
+) (podIP, podName, sshcm string, credential *util.SSHCredential, err error) {
+
+	component, version := labels["kt-component"], labels["version"]
+	sshcm = fmt.Sprintf("kt-%s-public-key-%s", component, version)
+
+	generator, err := util.Generate(util.PrivateKeyPath(component, version))
+	if err != nil {
+		return
+	}
+
+	labels["kt"] = sshcm
+	cli := clientset.CoreV1().ConfigMaps(namespace)
+	_, err = cli.Create(&v1.ConfigMap{
+		ObjectMeta: metaV1.ObjectMeta{
+			Name:      sshcm,
+			Namespace: namespace,
+			Labels:    labels,
+		},
+		Data: map[string]string{
+			vars.SSHAuthKey: string(generator.PublicKey),
+		},
+	})
+	if err != nil {
+		return
+	}
 
 	localIPAddress := util.GetOutboundIP()
 	log.Info().Msgf("Client address %s", localIPAddress)
 	labels["remoteAddress"] = localIPAddress
 
+	labels["kt"] = name
 	client := clientset.AppsV1().Deployments(namespace)
-	deployment := generatorDeployment(namespace, name, labels, image)
+	deployment := generatorDeployment(namespace, name, labels, image, sshcm)
 	result, err := client.Create(deployment)
 	if err != nil {
 		return
@@ -157,6 +189,8 @@ func CreateShadow(
 	}
 	podIP = pod.Status.PodIP
 	podName = pod.GetObjectMeta().GetName()
+	credential = util.NewDefaultSSHCredential()
+	credential.PrivateKeyPath = generator.PrivateKeyPath
 	return
 }
 
@@ -255,7 +289,7 @@ func generateService(name, namespace string, labels map[string]string, port int)
 
 }
 
-func generatorDeployment(namespace, name string, labels map[string]string, image string) *appV1.Deployment {
+func generatorDeployment(namespace, name string, labels map[string]string, image, volume string) *appV1.Deployment {
 	return &appV1.Deployment{
 		ObjectMeta: metaV1.ObjectMeta{
 			Name:      name,
@@ -276,6 +310,30 @@ func generatorDeployment(namespace, name string, labels map[string]string, image
 							Name:            "standalone",
 							Image:           image,
 							ImagePullPolicy: "Always",
+							VolumeMounts: []v1.VolumeMount{
+								{
+									Name:      "ssh-public-key",
+									MountPath: fmt.Sprintf("/root/%s", vars.SSHAuthKey),
+								},
+							},
+						},
+					},
+					Volumes: []v1.Volume{
+						{
+							Name: "ssh-public-key",
+							VolumeSource: v1.VolumeSource{
+								ConfigMap: &v1.ConfigMapVolumeSource{
+									LocalObjectReference: v1.LocalObjectReference{
+										Name: volume,
+									},
+									Items: []v1.KeyToPath{
+										{
+											Key:  vars.SSHAuthKey,
+											Path: "authorized_keys",
+										},
+									},
+								},
+							},
 						},
 					},
 				},

pkg/kt/cluster/types.go

@@ -1,7 +1,9 @@
 package cluster
 
 import (
+	"github.com/alibaba/kt-connect/pkg/kt/util"
 	appV1 "k8s.io/api/apps/v1"
+	coreV1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/kubernetes"
 	v1 "k8s.io/client-go/listers/core/v1"
 )
@@ -24,7 +26,8 @@ type KubernetesInterface interface {
 	Scale(deployment *appV1.Deployment, replicas *int32) (err error)
 	ServiceHosts(namespace string) (hosts map[string]string)
 	ClusterCrids(podCIDR string) (cidrs []string, err error)
-	CreateShadow(name, namespace, image string, labels map[string]string) (podIP, podName string, err error)
+	CreateShadow(name, namespace, image string, labels map[string]string) (podIP, podName, sshcm string, credential *util.SSHCredential, err error)
+	CreateService(name, namespace string, port int, labels map[string]string) (*coreV1.Service, error)
 }
 
 // Kubernetes implements KubernetesInterface

pkg/kt/command/connect.go

@@ -91,7 +91,9 @@ func (action *Action) Connect(options *options.DaemonOptions) (err error) {
 		return
 	}
 
-	connectToCluster(&shadow, &kubernetes, options)
+	if err = connectToCluster(&shadow, &kubernetes, options); err != nil {
+		return
+	}
 
 	s := <-ch
 	log.Info().Msgf("Terminal Signal is %s", s)
@@ -107,8 +109,7 @@ func connectToCluster(shadow connect.ShadowInterface, kubernetes cluster.Kuberne
 	}
 
 	workload := fmt.Sprintf("kt-connect-daemon-%s", strings.ToLower(util.RandomString(5)))
-
-	endPointIP, podName, err := kubernetes.CreateShadow(
+	endPointIP, podName, sshcm, credential, err := kubernetes.CreateShadow(
 		workload, options.Namespace, options.Image, labels(workload, options),
 	)
 
@@ -118,23 +119,25 @@ func connectToCluster(shadow connect.ShadowInterface, kubernetes cluster.Kuberne
 
 	// record shadow name will clean up terminal
 	options.RuntimeOptions.Shadow = workload
+	options.RuntimeOptions.SSHCM = sshcm
 
 	cidrs, err := kubernetes.ClusterCrids(options.ConnectOptions.CIDR)
 	if err != nil {
 		return
 	}
 
-	return shadow.Outbound(podName, endPointIP, cidrs)
+	return shadow.Outbound(podName, endPointIP, credential, cidrs)
 }
 
 func labels(workload string, options *options.DaemonOptions) map[string]string {
 	labels := map[string]string{
-		"kt":           workload,
 		"kt-component": "connect",
 		"control-by":   "kt",
 	}
 	for k, v := range util.String2Map(options.Labels) {
 		labels[k] = v
 	}
+	splits := strings.Split(workload, "-")
+	labels["version"] = splits[len(splits)-1]
 	return labels
 }

pkg/kt/command/connect_test.go

@@ -59,11 +59,10 @@ func Test_shouldConnectToCluster(t *testing.T) {
 	ctl := gomock.NewController(t)
 	kubernetes := mock.NewMockKubernetesInterface(ctl)
 	shadow := mock.NewMockShadowInterface(ctl)
-
-	kubernetes.EXPECT().CreateShadow(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("172.168.0.2", "shadowName", nil).AnyTimes()
+	kubernetes.EXPECT().CreateShadow(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("172.168.0.2", "shadowName", "sshcm", nil, nil).AnyTimes()
 	kubernetes.EXPECT().ClusterCrids(gomock.Any()).Return([]string{"10.10.10.0/24"}, nil)
 
-	shadow.EXPECT().Outbound("shadowName", "172.168.0.2", []string{"10.10.10.0/24"}).Return(nil)
+	shadow.EXPECT().Outbound("shadowName", "172.168.0.2", gomock.Any(), []string{"10.10.10.0/24"}).Return(nil)
 
 	type args struct {
 		shadow     connect.ShadowInterface
@@ -91,8 +90,7 @@ func Test_shouldConnectClusterFailWhenFailCreateShadow(t *testing.T) {
 	ctl := gomock.NewController(t)
 	kubernetesInterface := mock.NewMockKubernetesInterface(ctl)
 	shadowInterface := mock.NewMockShadowInterface(ctl)
-
-	kubernetesInterface.EXPECT().CreateShadow(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("", "", errors.New("")).AnyTimes()
+	kubernetesInterface.EXPECT().CreateShadow(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("", "", "", nil, errors.New("")).AnyTimes()
 
 	type args struct {
 		shadow     connect.ShadowInterface
@@ -117,8 +115,7 @@ func Test_shouldConnectClusterFailWhenFailGetCrids(t *testing.T) {
 	ctl := gomock.NewController(t)
 	kubernetes := mock.NewMockKubernetesInterface(ctl)
 	shadow := mock.NewMockShadowInterface(ctl)
-
-	kubernetes.EXPECT().CreateShadow(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("172.168.0.2", "shadowName", nil).AnyTimes()
+	kubernetes.EXPECT().CreateShadow(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("172.168.0.2", "shadowName", "sshcm", nil, nil).AnyTimes()
 	kubernetes.EXPECT().ClusterCrids(gomock.Any()).Return([]string{}, errors.New("fail to get crid"))
 
 	type args struct {

pkg/kt/command/exchange.go

@@ -42,7 +42,6 @@ func newExchangeCommand(options *options.DaemonOptions, action ActionInterface)
 			if len(expose) == 0 {
 				return errors.New("-expose is required")
 			}
-
 			return action.Exchange(exchange, options)
 		},
 	}
@@ -65,12 +64,13 @@ func (action *Action) Exchange(exchange string, options *options.DaemonOptions)
 	}
 
 	// record context inorder to remove after command exit
-	options.RuntimeOptions.Origin = app.GetObjectMeta().GetName()
+	options.RuntimeOptions.Origin = app.GetName()
 	options.RuntimeOptions.Replicas = *app.Spec.Replicas
 
-	workload := app.GetObjectMeta().GetName() + "-kt-" + strings.ToLower(util.RandomString(5))
-	podIP, podName, err := kubernetes.CreateShadow(
-		workload, options.Namespace, options.Image, getExchangeLables(options.Labels, workload, app))
+	workload := app.GetName() + "-kt-" + strings.ToLower(util.RandomString(5))
+
+	podIP, podName, sshcm, credential, err := kubernetes.CreateShadow(
+		workload, options.Namespace, options.Image, getExchangeLabels(options.Labels, workload, app))
 	log.Info().Msgf("create exchange shadow %s in namespace %s", workload, options.Namespace)
 
 	if err != nil {
@@ -79,12 +79,17 @@ func (action *Action) Exchange(exchange string, options *options.DaemonOptions)
 
 	// record data
 	options.RuntimeOptions.Shadow = workload
+	options.RuntimeOptions.SSHCM = sshcm
 
 	down := int32(0)
-	kubernetes.Scale(app, &down)
+	if err = kubernetes.Scale(app, &down); err != nil {
+		return err
+	}
 
 	shadow := connect.Create(options)
-	shadow.Inbound(options.ExchangeOptions.Expose, podName, podIP)
+	if err = shadow.Inbound(options.ExchangeOptions.Expose, podName, podIP, credential); err != nil {
+		return err
+	}
 
 	s := <-ch
 	log.Info().Msgf("Terminal Signal is %s", s)
@@ -92,18 +97,22 @@ func (action *Action) Exchange(exchange string, options *options.DaemonOptions)
 	return nil
 }
 
-func getExchangeLables(customLabels string, workload string, origin *v1.Deployment) map[string]string {
+func getExchangeLabels(customLabels string, workload string, origin *v1.Deployment) map[string]string {
 	labels := map[string]string{
 		"kt":           workload,
 		"kt-component": "exchange",
 		"control-by":   "kt",
 	}
-	for k, v := range origin.Spec.Selector.MatchLabels {
-		labels[k] = v
+	if origin != nil {
+		for k, v := range origin.Spec.Selector.MatchLabels {
+			labels[k] = v
+		}
 	}
 	// extra labels must be applied after origin labels
 	for k, v := range util.String2Map(customLabels) {
 		labels[k] = v
 	}
+	splits := strings.Split(workload, "-")
+	labels["version"] = splits[len(splits)-1]
 	return labels
 }

pkg/kt/command/mesh.go

@@ -48,7 +48,6 @@ func newMeshCommand(options *options.DaemonOptions, action ActionInterface) cli.
 			if len(expose) == 0 {
 				return errors.New("-expose is required")
 			}
-
 			return action.Mesh(mesh, options)
 		},
 	}
@@ -74,17 +73,17 @@ func (action *Action) Mesh(mesh string, options *options.DaemonOptions) error {
 	workload := app.GetObjectMeta().GetName() + "-kt-" + meshVersion
 
 	labels := getMeshLabels(workload, meshVersion, app, options)
-
-	podIP, podName, err := kubernetes.CreateShadow(workload, options.Namespace, options.Image, labels)
+	podIP, podName, sshcm, credential, err := kubernetes.CreateShadow(workload, options.Namespace, options.Image, labels)
 	if err != nil {
 		return err
 	}
 
 	// record context data
 	options.RuntimeOptions.Shadow = workload
+	options.RuntimeOptions.SSHCM = sshcm
 
 	shadow := connect.Create(options)
-	err = shadow.Inbound(options.MeshOptions.Expose, podName, podIP)
+	err = shadow.Inbound(options.MeshOptions.Expose, podName, podIP, credential)
 
 	if err != nil {
 		return err
@@ -103,8 +102,10 @@ func getMeshLabels(workload string, meshVersion string, app *v1.Deployment, opti
 		"kt-component": ComponentMesh,
 		"control-by":   KubernetesTool,
 	}
-	for k, v := range app.Spec.Selector.MatchLabels {
-		labels[k] = v
+	if app != nil {
+		for k, v := range app.Spec.Selector.MatchLabels {
+			labels[k] = v
+		}
 	}
 	// extra labels must be applied after origin labels
 	for k, v := range util.String2Map(options.Labels) {

pkg/kt/command/run.go

@@ -3,6 +3,7 @@ package command
 import (
 	"errors"
 	"strconv"
+	"strings"
 
 	"github.com/alibaba/kt-connect/pkg/kt/cluster"
 	"github.com/alibaba/kt-connect/pkg/kt/connect"
@@ -46,8 +47,7 @@ func newRunCommand(options *options.DaemonOptions, action ActionInterface) cli.C
 // Run create a new service in cluster
 func (action *Action) Run(service string, options *options.DaemonOptions) error {
 	ch := SetUpCloseHandler(options)
-
-	clientset, err := cluster.GetKubernetesClient(options.KubeConfig)
+	kubernetes, err := cluster.Create(options.KubeConfig)
 	if err != nil {
 		return err
 	}
@@ -56,6 +56,7 @@ func (action *Action) Run(service string, options *options.DaemonOptions) error
 		"control-by":   "kt",
 		"kt-component": "run",
 		"kt":           service,
+		"version":      strings.ToLower(util.RandomString(5)),
 	}
 
 	// extra labels must be applied after origin labels
@@ -63,7 +64,7 @@ func (action *Action) Run(service string, options *options.DaemonOptions) error
 		labels[k] = v
 	}
 
-	podIP, podName, err := cluster.CreateShadow(clientset, service, labels, options.Namespace, options.Image)
+	podIP, podName, sshcm, credential, err := kubernetes.CreateShadow(service, options.Namespace, options.Image, labels)
 	if err != nil {
 		return err
 	}
@@ -71,17 +72,17 @@ func (action *Action) Run(service string, options *options.DaemonOptions) error
 
 	if options.RunOptions.Expose {
 		log.Info().Msgf("expose deployment %s to %s:%v", service, service, options.RunOptions.Port)
-		err := cluster.CreateService(service, options.Namespace, labels, options.RunOptions.Port, clientset)
-		if err != nil {
+		if _, err = kubernetes.CreateService(service, options.Namespace, options.RunOptions.Port, labels); err != nil {
 			return err
 		}
 		options.RuntimeOptions.Service = service
 	}
 
 	options.RuntimeOptions.Shadow = service
+	options.RuntimeOptions.SSHCM = sshcm
 
 	shadow := connect.Create(options)
-	err = shadow.Inbound(strconv.Itoa(options.RunOptions.Port), podName, podIP)
+	err = shadow.Inbound(strconv.Itoa(options.RunOptions.Port), podName, podIP, credential)
 	if err != nil {
 		return err
 	}

pkg/kt/command/util.go

@@ -3,6 +3,7 @@ package command
 import (
 	"os"
 	"os/signal"
+	"strings"
 	"syscall"
 
 	"github.com/alibaba/kt-connect/pkg/kt/cluster"
@@ -92,9 +93,18 @@ func CleanupWorkspace(options *options.DaemonOptions) {
 		cluster.RemoveShadow(client, options.Namespace, options.RuntimeOptions.Shadow)
 	}
 
+	if len(options.RuntimeOptions.SSHCM) > 0 {
+		log.Info().Msgf("- clean sshcm %s", options.RuntimeOptions.SSHCM)
+		cluster.RemoveSSHCM(client, options.Namespace, options.RuntimeOptions.SSHCM)
+	}
+
+	removePrivateKey(options)
 	if len(options.RuntimeOptions.Service) > 0 {
 		log.Info().Msgf("- cleanup service %s", options.RuntimeOptions.Service)
-		cluster.RemoveService(options.RuntimeOptions.Service, options.Namespace, client)
+		err = cluster.RemoveService(options.RuntimeOptions.Service, options.Namespace, client)
+		if err != nil {
+			log.Error().Err(err).Msg("delete service failed")
+		}
 	}
 }
 
@@ -107,3 +117,16 @@ func checkConnectRunning(pidFile string) {
 		log.Info().Msgf("'KT Connect' is runing, you can access local app from cluster and localhost")
 	}
 }
+
+// removePrivateKey remove the private key of ssh
+func removePrivateKey(options *options.DaemonOptions) {
+	if options.RuntimeOptions.SSHCM == "" {
+		return
+	}
+	splits := strings.Split(options.RuntimeOptions.SSHCM, "-")
+	component, version := splits[1], splits[len(splits)-1]
+	file := util.PrivateKeyPath(component, version)
+	if err := os.Remove(file); !os.IsNotExist(err) {
+		log.Error().Err(err).Msgf("can't delete %s", file)
+	}
+}

pkg/kt/connect/inbound.go

@@ -14,7 +14,7 @@ import (
 )
 
 // Inbound mapping local port from cluster
-func (s *Shadow) Inbound(exposePort, podName, remoteIP string) (err error) {
+func (s *Shadow) Inbound(exposePort, podName, remoteIP string, credential *util.SSHCredential) (err error) {
 	debug := s.Options.Debug
 	kubeConfig := s.Options.KubeConfig
 	namespace := s.Options.Namespace
@@ -44,6 +44,6 @@ func (s *Shadow) Inbound(exposePort, podName, remoteIP string) (err error) {
 		localPort = ports[1]
 		remotePort = ports[0]
 	}
-	cmd := ssh.ForwardRemoteRequestToLocal(localPort, "127.0.0.1", remotePort, localSSHPort)
+	cmd := ssh.ForwardRemoteRequestToLocal(localPort, credential.RemoteHost, remotePort, credential.PrivateKeyPath, localSSHPort)
 	return exec.BackgroundRun(cmd, "ssh remote port-forward", debug)
 }

pkg/kt/connect/outbound.go

@@ -14,12 +14,8 @@ import (
 )
 
 // Outbound start vpn connection
-func (s *Shadow) Outbound(name, podIP string, cidrs []string) (err error) {
+func (s *Shadow) Outbound(name, podIP string, credential *util.SSHCredential, cidrs []string) (err error) {
 	options := s.Options
-	err = util.PrepareSSHPrivateKey()
-	if err != nil {
-		return
-	}
 	err = exec.BackgroundRun(
 		kubectl.PortForward(
 			options.KubeConfig,
@@ -37,9 +33,9 @@ func (s *Shadow) Outbound(name, podIP string, cidrs []string) (err error) {
 		log.Info().Msgf("Start SOCKS5 Proxy: export http_proxy=socks5://127.0.0.1:%d", options.ConnectOptions.Socke5Proxy)
 		log.Info().Msgf("==============================================================")
 		_ = ioutil.WriteFile(".jvmrc", []byte(fmt.Sprintf("-DsocksProxyHost=127.0.0.1\n-DsocksProxyPort=%d", options.ConnectOptions.Socke5Proxy)), 0644)
-		err = exec.BackgroundRun(ssh.DynamicForwardLocalRequestToRemote("127.0.0.1", options.ConnectOptions.SSHPort, options.ConnectOptions.Socke5Proxy), "vpn(ssh)", options.Debug)
+		err = exec.BackgroundRun(ssh.DynamicForwardLocalRequestToRemote(credential.RemoteHost, credential.PrivateKeyPath, options.ConnectOptions.SSHPort, options.ConnectOptions.Socke5Proxy), "vpn(ssh)", options.Debug)
 	} else {
-		err = exec.BackgroundRun(sshuttle.SSHUttle("127.0.0.1", options.ConnectOptions.SSHPort, podIP, options.ConnectOptions.DisableDNS, cidrs, options.Debug), "vpn(sshuttle)", options.Debug)
+		err = exec.BackgroundRun(sshuttle.SSHUttle(credential.RemoteHost, credential.PrivateKeyPath, options.ConnectOptions.SSHPort, podIP, options.ConnectOptions.DisableDNS, cidrs, options.Debug), "vpn(sshuttle)", options.Debug)
 	}
 	if err != nil {
 		return

pkg/kt/connect/types.go

@@ -1,11 +1,14 @@
 package connect
 
-import "github.com/alibaba/kt-connect/pkg/kt/options"
+import (
+	"github.com/alibaba/kt-connect/pkg/kt/options"
+	"github.com/alibaba/kt-connect/pkg/kt/util"
+)
 
 // ShadowInterface shadow interface
 type ShadowInterface interface {
-	Inbound(exposePort, podName, remoteIP string) (err error)
-	Outbound(name, podIP string, cidrs []string) (err error)
+	Inbound(exposePort, podName, remoteIP string, credential *util.SSHCredential) (err error)
+	Outbound(name, podIP string, credential *util.SSHCredential, cidrs []string) (err error)
 }
 
 // Shadow shadow

pkg/kt/exec/run.go

@@ -29,7 +29,7 @@ func BackgroundRun(cmd *exec.Cmd, name string, debug bool) (err error) {
 
 func runCmd(cmd *exec.Cmd, name string, debug bool) (err error) {
 	log.Debug().Msgf("Child, os.Args = %+v", os.Args)
-	log.Debug().Msgf("Child, cmd.Args = %+v", cmd.Args)
+	log.Debug().Msgf("Child, name = %s, cmd.Args = %+v", name, cmd.Args)
 
 	var stdoutBuf, stderrBuf bytes.Buffer
 	stdoutIn, _ := cmd.StdoutPipe()

pkg/kt/exec/ssh/commands.go

@@ -3,8 +3,6 @@ package ssh
 import (
 	"fmt"
 	"os/exec"
-
-	"github.com/alibaba/kt-connect/pkg/kt/util"
 )
 
 // Version check sshuttle version
@@ -13,11 +11,11 @@ func Version() *exec.Cmd {
 }
 
 // ForwardRemoteRequestToLocal ssh remote port forward
-func ForwardRemoteRequestToLocal(localPort, remoteHost, remotePort string, remoteSSHPort int) *exec.Cmd {
+func ForwardRemoteRequestToLocal(localPort, remoteHost, remotePort, privateKeyPath string, remoteSSHPort int) *exec.Cmd {
 	return exec.Command("ssh",
 		"-oStrictHostKeyChecking=no",
 		"-oUserKnownHostsFile=/dev/null",
-		"-i", util.PrivateKeyPath(),
+		"-i", privateKeyPath,
 		"-R", remotePort+":127.0.0.1:"+localPort,
 		fmt.Sprintf("root@%s", remoteHost), "-p"+fmt.Sprintf("%d", remoteSSHPort),
 		"sh", "loop.sh",
@@ -25,11 +23,11 @@ func ForwardRemoteRequestToLocal(localPort, remoteHost, remotePort string, remot
 }
 
 // DynamicForwardLocalRequestToRemote ssh remote port forward
-func DynamicForwardLocalRequestToRemote(remoteHost string, remoteSSHPort int, proxyPort int) *exec.Cmd {
+func DynamicForwardLocalRequestToRemote(remoteHost, privateKeyPath string, remoteSSHPort int, proxyPort int) *exec.Cmd {
 	return exec.Command("ssh",
 		"-oStrictHostKeyChecking=no",
 		"-oUserKnownHostsFile=/dev/null",
-		"-i", util.PrivateKeyPath(),
+		"-i", privateKeyPath,
 		"-D", fmt.Sprintf("%d", proxyPort),
 		fmt.Sprintf("root@%s", remoteHost), "-p"+fmt.Sprintf("%d", remoteSSHPort),
 		"sh", "loop.sh",

pkg/kt/exec/sshuttle/commands.go

@@ -2,7 +2,6 @@ package sshuttle
 
 import (
 	"fmt"
-	"github.com/alibaba/kt-connect/pkg/kt/util"
 	"os/exec"
 )
 
@@ -12,7 +11,7 @@ func Version() *exec.Cmd {
 }
 
 // SSHUttle ssh-baed vpn connect
-func SSHUttle(remoteHost string, remotePort int, DNSServer string, disableDNS bool, cidrs []string, debug bool) *exec.Cmd {
+func SSHUttle(remoteHost, privateKeyPath string, remotePort int, DNSServer string, disableDNS bool, cidrs []string, debug bool) *exec.Cmd {
 	args := []string{}
 	if !disableDNS {
 		args = append(args, "--dns", "--to-ns", DNSServer)
@@ -22,7 +21,7 @@ func SSHUttle(remoteHost string, remotePort int, DNSServer string, disableDNS bo
 		args = append(args, "-v")
 	}
 
-	subCommand := fmt.Sprintf("ssh -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -i %s", util.PrivateKeyPath())
+	subCommand := fmt.Sprintf("ssh -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -i %s", privateKeyPath)
 	args = append(args, "-e", subCommand, "-r", fmt.Sprintf("root@%s:%d", remoteHost, remotePort), "-x", remoteHost)
 	args = append(args, cidrs...)
 	return exec.Command("sshuttle", args...)

pkg/kt/options/options.go

@@ -35,6 +35,8 @@ type runtimeOptions struct {
 	AppHome  string
 	// Shadow deployment name
 	Shadow string
+	// ssh public key name of config map. format is kt-xxx(component)-public-key-xxx(version)
+	SSHCM string
 	// The origin app name
 	Origin string
 	// The origin repicas

pkg/kt/util/ssh.go

@@ -0,0 +1,129 @@
+package util
+
+import (
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/alibaba/kt-connect/pkg/kt/vars"
+	"github.com/rs/zerolog/log"
+	"golang.org/x/crypto/ssh"
+)
+
+// SSHCredential ssh info
+type SSHCredential struct {
+	RemoteHost     string
+	Port           string
+	PrivateKeyPath string
+}
+
+// SSHGenerator ssh key pair generator
+type SSHGenerator struct {
+	PrivateKey, PublicKey []byte
+	PrivateKeyPath        string
+}
+
+// NewDefaultSSHCredential ...
+func NewDefaultSSHCredential() *SSHCredential {
+	return &SSHCredential{
+		Port:       "2222",
+		RemoteHost: "127.0.0.1",
+	}
+}
+
+// Generate generate SSHGenerator
+func Generate(privateKeyPath string) (*SSHGenerator, error) {
+	privateKey, err := generatePrivateKey(vars.SSHBitSize)
+	if err != nil {
+		return nil, err
+	}
+
+	publicKeyBytes, err := generatePublicKey(&privateKey.PublicKey)
+	if err != nil {
+		return nil, err
+	}
+	privateKeyBytes := encodePrivateKeyToPEM(privateKey)
+
+	ssh := &SSHGenerator{
+		PrivateKey:     privateKeyBytes,
+		PrivateKeyPath: privateKeyPath,
+		PublicKey:      publicKeyBytes,
+	}
+	err = writePrivateKey(ssh.PrivateKeyPath, ssh.PrivateKey)
+	return ssh, err
+}
+
+// PrivateKeyPath ...
+func PrivateKeyPath(component, identifier string) string {
+	return fmt.Sprintf("%s/ktctl/%s/"+vars.SSHPrivateKeyName, HomeDir(), component, identifier)
+}
+
+// generatePrivateKey creates a RSA Private Key of specified byte size
+func generatePrivateKey(bitSize int) (*rsa.PrivateKey, error) {
+	// Private Key generation
+	privateKey, err := rsa.GenerateKey(rand.Reader, bitSize)
+	if err != nil {
+		return nil, err
+	}
+
+	// Validate Private Key
+	err = privateKey.Validate()
+	if err != nil {
+		return nil, err
+	}
+
+	log.Info().Msg("private Key generated")
+	return privateKey, nil
+}
+
+// encodePrivateKeyToPEM encodes Private Key from RSA to PEM format
+func encodePrivateKeyToPEM(privateKey *rsa.PrivateKey) []byte {
+	// Get ASN.1 DER format
+	privDER := x509.MarshalPKCS1PrivateKey(privateKey)
+
+	// pem.Block
+	privBlock := pem.Block{
+		Type:    "RSA PRIVATE KEY",
+		Headers: nil,
+		Bytes:   privDER,
+	}
+
+	// Private key in PEM format
+	privatePEM := pem.EncodeToMemory(&privBlock)
+
+	return privatePEM
+}
+
+// generatePublicKey take a rsa.PublicKey and return bytes suitable for writing to .pub file
+// returns in the format "ssh-rsa ..."
+func generatePublicKey(privatekey *rsa.PublicKey) ([]byte, error) {
+	publicRsaKey, err := ssh.NewPublicKey(privatekey)
+	if err != nil {
+		return nil, err
+	}
+
+	pubKeyBytes := ssh.MarshalAuthorizedKey(publicRsaKey)
+	log.Info().Msg("public key generated")
+	return pubKeyBytes, nil
+}
+
+// writePrivateKey write ssh private key to privateKeyPath
+func writePrivateKey(privateKeyPath string, data []byte) error {
+	dir := filepath.Dir(privateKeyPath)
+	if _, err := os.Stat(dir); os.IsNotExist(err) {
+		if err = os.MkdirAll(dir, 0700); err != nil {
+			log.Error().Err(err).Str("dir", dir).Msg("can't create dir")
+			return err
+		}
+	}
+	if err := ioutil.WriteFile(privateKeyPath, data, 0700); err != nil {
+		log.Error().Err(err).Str("file", privateKeyPath).Msg("write ssh private key failed")
+		return err
+	}
+	return nil
+}

pkg/kt/util/strings.go

@@ -3,12 +3,14 @@ package util
 import (
 	"math/rand"
 	"strings"
+	"time"
 )
 
 var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
 
 // RandomString Generate RandomString
 func RandomString(n int) string {
+	rand.Seed(time.Now().Unix())
 	b := make([]rune, n)
 	for i := range b {
 		b[i] = letterRunes[rand.Intn(len(letterRunes))]
@@ -27,4 +29,4 @@ func String2Map(str string) map[string]string {
 		}
 	}
 	return res
-}
+}

pkg/kt/util/system.go

@@ -3,59 +3,13 @@ package util
 import (
 	"fmt"
 	"io/ioutil"
-	"math/rand"
 	"os"
 	"runtime"
-	"time"
 
 	"github.com/lextoumbourou/goodhosts"
 	"github.com/rs/zerolog/log"
 )
 
-var (
-	pk []byte
-)
-
-func init() {
-	rand.Seed(time.Now().UnixNano())
-	pk = []byte("-----BEGIN RSA PRIVATE KEY-----\n" +
-		"MIIEpAIBAAKCAQEAvSVAezJDBhrNDhLhuCaKrvdtCFTdqJmGLGyfBqEYb3p4a91g\n" +
-		"l4gD2LGwiRlgpwU4oSECeMmwP53C4vrfPKY45/+8lncIkE/4E+8hHssXkHaqQrjE\n" +
-		"FtePfxZ6/xi84kUbWNNV4IGAeFwXtq9GszQ+kWMNT5QmuexOXOlqq7W4CIAUe3uX\n" +
-		"29WCp3OGiBeP4ORDraRa/1bwBH+Cq0UxEYT+6EuDU0YzF3JF4H8At6NdgElAuezE\n" +
-		"wI84p5LNr1HmTPndcHJtX2+POKEoNYBxPekEyJbqExIR2dLRUytlX5tIacKkMBCJ\n" +
-		"aX7DBtJzWX7BxfjRfXjzNOpufuTU9BsknyJlFwIDAQABAoIBAQCS606s4xvAsCy7\n" +
-		"U9tUyUtMIRDmOdV7UtUvyKe15Igwf3bugiS3T4V9Wnh/5eB3m8yjDBr5a+ClaYup\n" +
-		"96hTWeI2AyWf0pIqVpOiGEsnuiVxp1sVPKPEAmiKFRIw+CwvrfJSCsZX/v+lfhNF\n" +
-		"adyG8nvvPntmZvO102IDNaQQALUUk+J69yvtb7ekfvZCSmanXx/R7y2+u8Hd8wtf\n" +
-		"fpyVcM1g7YZwhxto2uKyUE1T/myOT5+wULfYTMNynsLB6dXJJP+a89I6cRdFCIib\n" +
-		"kqoZ5FuaTXXucrgWGDne6DcvNbiZi1f8LRb6RFnlvv6D42xyjelyoGY7BktKsFwJ\n" +
-		"NwLR1lBhAoGBANyGB7ti190DZoDGQvIpSHd+JeZHoK3g49VNxGFU+SAhXa8gQn6K\n" +
-		"Xi5qNRD2XLTEnT36U20/bkcDv0oSTZikJhU0OqxgouVO3YZ2cZhXcoPZmf+vhgzI\n" +
-		"ufv0T8/HlQyr7Sp9VqfqlC7u1P83VbNro/D3V+wtNKog4g++DvKtsh8JAoGBANuS\n" +
-		"9XDaAnq5K0rjShNqCMRyHx6+kFPaLpL4kt1f4yra8w/m3pes63sO1vz/4wOhoalL\n" +
-		"imAEqTKTblinPhjCxbe4e/WqnAQM05XROdiGer2RhBIMCo2/YE3WLCWAyVCDtd3B\n" +
-		"Te9rPynSsAmtgDRuftusY7TAIuwZuG4K71Gw8UsfAoGAYmRm5MPYXqNaw9AyJIwo\n" +
-		"6i/dxx5kYdB6tzxoh6j7MsvQWggBwyYHmZwHq1bQzFMBeZrMSG1JzeOtIOaDurxa\n" +
-		"xZE1MJ45cCi9DHaifn9d99hKLtvo6qFQ4ksCpUl+hlXbjt63oFo43avwWyMcWN6J\n" +
-		"GkWx9A3DdrkPREjfsIWxeMkCgYEAtDhv6duWk2IujX32y+6JGaxNrK9eyORYu9r4\n" +
-		"uGi+jOs++ztUUgvlD5EDlo70poNgrBLLlbndohxuQqeqiSo8nGn4nJAXFB/u/pXH\n" +
-		"M9hVIAky7JkjhGqiweBbRcDp+4LPoB7MOAm/wzUhth/JDb/vsaBSCgZ143HM9c1V\n" +
-		"1qgztKMCgYAUhQRJB6ofGqiGsPN2KZw+0IoPNS3Tk0NTjzVh2o927B8zb0T0bO5e\n" +
-		"qe0OO7FFGcON6uSOkGu2p9KHUEm6OFaQLjdysjrGI7GVRYW7D/SSLidRREv2A70R\n" +
-		"f0/Mi8v9nD4ztroXQDeeL8O4rFTnfRdqs+MZ/MYoq9C5iE1IHJm7KQ==\n" +
-		"-----END RSA PRIVATE KEY-----")
-}
-
-// PrepareSSHPrivateKey generator ssh private key
-func PrepareSSHPrivateKey() (err error) {
-	err = ioutil.WriteFile(PrivateKeyPath(), pk, 400)
-	if err != nil {
-		log.Error().Msgf("Fails create temp ssh private key")
-	}
-	return
-}
-
 // IsDaemonRunning check daemon is running or not
 func IsDaemonRunning(pidFile string) bool {
 	if _, err := os.Stat(pidFile); os.IsNotExist(err) {
@@ -75,13 +29,6 @@ func HomeDir() string {
 	return "/root"
 }
 
-// PrivateKeyPath Get ssh private key path
-func PrivateKeyPath() string {
-	userHome := HomeDir()
-	privateKey := fmt.Sprintf("%s/.kt_id_rsa", userHome)
-	return privateKey
-}
-
 // CreateDirIfNotExist create dir
 func CreateDirIfNotExist(dir string) {
 	if _, err := os.Stat(dir); os.IsNotExist(err) {

pkg/kt/vars/vars.go

@@ -0,0 +1,10 @@
+package vars
+
+var (
+	// SSHPrivateKeyName ssh private key name
+	SSHPrivateKeyName = "kt_%s_id_rsa"
+	// SSHBitSize ssh bit size
+	SSHBitSize = 2048
+	// SSHAuthKey auth key name
+	SSHAuthKey = "authorized"
+)

pkg/mockd/mock/action_mock.go

@@ -5,10 +5,9 @@
 package mock
 
 import (
-	reflect "reflect"
-
 	options "github.com/alibaba/kt-connect/pkg/kt/options"
 	gomock "github.com/golang/mock/gomock"
+	reflect "reflect"
 )
 
 // MockActionInterface is a mock of ActionInterface interface

pkg/mockd/mock/connect_mock.go

@@ -5,9 +5,9 @@
 package mock
 
 import (
-	reflect "reflect"
-
+	util "github.com/alibaba/kt-connect/pkg/kt/util"
 	gomock "github.com/golang/mock/gomock"
+	reflect "reflect"
 )
 
 // MockShadowInterface is a mock of ShadowInterface interface
@@ -34,29 +34,29 @@ func (m *MockShadowInterface) EXPECT() *MockShadowInterfaceMockRecorder {
 }
 
 // Inbound mocks base method
-func (m *MockShadowInterface) Inbound(exposePort, podName, remoteIP string) error {
+func (m *MockShadowInterface) Inbound(exposePort, podName, remoteIP string, credential *util.SSHCredential) error {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "Inbound", exposePort, podName, remoteIP)
+	ret := m.ctrl.Call(m, "Inbound", exposePort, podName, remoteIP, credential)
 	ret0, _ := ret[0].(error)
 	return ret0
 }
 
 // Inbound indicates an expected call of Inbound
-func (mr *MockShadowInterfaceMockRecorder) Inbound(exposePort, podName, remoteIP interface{}) *gomock.Call {
+func (mr *MockShadowInterfaceMockRecorder) Inbound(exposePort, podName, remoteIP, credential interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Inbound", reflect.TypeOf((*MockShadowInterface)(nil).Inbound), exposePort, podName, remoteIP)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Inbound", reflect.TypeOf((*MockShadowInterface)(nil).Inbound), exposePort, podName, remoteIP, credential)
 }
 
 // Outbound mocks base method
-func (m *MockShadowInterface) Outbound(name, podIP string, cidrs []string) error {
+func (m *MockShadowInterface) Outbound(name, podIP string, credential *util.SSHCredential, cidrs []string) error {
 	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "Outbound", name, podIP, cidrs)
+	ret := m.ctrl.Call(m, "Outbound", name, podIP, credential, cidrs)
 	ret0, _ := ret[0].(error)
 	return ret0
 }
 
 // Outbound indicates an expected call of Outbound
-func (mr *MockShadowInterfaceMockRecorder) Outbound(name, podIP, cidrs interface{}) *gomock.Call {
+func (mr *MockShadowInterfaceMockRecorder) Outbound(name, podIP, credential, cidrs interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Outbound", reflect.TypeOf((*MockShadowInterface)(nil).Outbound), name, podIP, cidrs)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Outbound", reflect.TypeOf((*MockShadowInterface)(nil).Outbound), name, podIP, credential, cidrs)
 }

pkg/mockd/mock/kubernetes_mock.go

@@ -5,10 +5,11 @@
 package mock
 
 import (
-	reflect "reflect"
-
+	util "github.com/alibaba/kt-connect/pkg/kt/util"
 	gomock "github.com/golang/mock/gomock"
 	v1 "k8s.io/api/apps/v1"
+	v10 "k8s.io/api/core/v1"
+	reflect "reflect"
 )
 
 // MockKubernetesInterface is a mock of KubernetesInterface interface
@@ -93,13 +94,15 @@ func (mr *MockKubernetesInterfaceMockRecorder) ClusterCrids(podCIDR interface{})
 }
 
 // CreateShadow mocks base method
-func (m *MockKubernetesInterface) CreateShadow(name, namespace, image string, labels map[string]string) (string, string, error) {
+func (m *MockKubernetesInterface) CreateShadow(name, namespace, image string, labels map[string]string) (string, string, string, *util.SSHCredential, error) {
 	m.ctrl.T.Helper()
 	ret := m.ctrl.Call(m, "CreateShadow", name, namespace, image, labels)
 	ret0, _ := ret[0].(string)
 	ret1, _ := ret[1].(string)
-	ret2, _ := ret[2].(error)
-	return ret0, ret1, ret2
+	ret2, _ := ret[2].(string)
+	ret3, _ := ret[3].(*util.SSHCredential)
+	ret4, _ := ret[4].(error)
+	return ret0, ret1, ret2, ret3, ret4
 }
 
 // CreateShadow indicates an expected call of CreateShadow
@@ -107,3 +110,18 @@ func (mr *MockKubernetesInterfaceMockRecorder) CreateShadow(name, namespace, ima
 	mr.mock.ctrl.T.Helper()
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateShadow", reflect.TypeOf((*MockKubernetesInterface)(nil).CreateShadow), name, namespace, image, labels)
 }
+
+// CreateService mocks base method
+func (m *MockKubernetesInterface) CreateService(name, namespace string, port int, labels map[string]string) (*v10.Service, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "CreateService", name, namespace, port, labels)
+	ret0, _ := ret[0].(*v10.Service)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// CreateService indicates an expected call of CreateService
+func (mr *MockKubernetesInterfaceMockRecorder) CreateService(name, namespace, port, labels interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateService", reflect.TypeOf((*MockKubernetesInterface)(nil).CreateService), name, namespace, port, labels)
+}