diff --git a/docker/shadow/Dockerfile_base b/docker/shadow/Dockerfile_base index 54c5347..d737e3c 100644 --- a/docker/shadow/Dockerfile_base +++ b/docker/shadow/Dockerfile_base @@ -4,9 +4,6 @@ RUN apt-get update && apt-get install -y openssh-server dnsutils sshuttle iputil RUN mkdir /var/run/sshd RUN echo 'root:root' | chpasswd RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config -RUN mkdir -p /root/.ssh && echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC9JUB7MkMGGs0OEuG4Joqu920IVN2omYYsbJ8GoRhvenhr3WCXiAPYsbCJGWCnBTihIQJ4ybA/ncLi+t88pjjn/7yWdwiQT/gT7yEeyxeQdqpCuMQW149/\ -Fnr/GLziRRtY01XggYB4XBe2r0azND6RYw1PlCa57E5c6WqrtbgIgBR7e5fb1YKnc4aIF4/g5EOtpFr/VvAEf4KrRTERhP7oS4NTRjMXckXgfwC3o12ASUC57MTAjzinks2vUeZM+d1wcm1fb484oSg1gHE9\ -6QTIluoTEhHZ0tFTK2Vfm0hpwqQwEIlpfsMG0nNZfsHF+NF9ePM06m5+5NT0GySfImUX kt@public_key" > /root/.ssh/authorized_keys && chmod 400 /root/.ssh/authorized_keys # SSH login fix. Otherwise user is kicked off after login RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd diff --git a/docker/shadow/run.sh b/docker/shadow/run.sh index 1e07f97..ee8c8de 100644 --- a/docker/shadow/run.sh +++ b/docker/shadow/run.sh @@ -1,4 +1,7 @@ #!/bin/bash +mkdir -p /root/.ssh +cp /root/authorized/authorized_keys /root/.ssh + /usr/sbin/sshd -D & if [[ "${1}" = "--debug" ]]; then diff --git a/go.mod b/go.mod index 007ea1f..55044d1 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 github.com/thinkerou/favicon v0.1.0 // indirect github.com/urfave/cli v0.0.0-20190203184040-693af58b4d51 + golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392 istio.io/api v0.0.0-20200221025927-228308df3f1b istio.io/client-go v0.0.0-20200221055756-736d3076b458 k8s.io/api v0.17.2 diff --git a/pkg/apiserver/server/server.go b/pkg/apiserver/server/server.go index 9cc8f17..cf43b75 100644 --- a/pkg/apiserver/server/server.go +++ b/pkg/apiserver/server/server.go @@ -2,6 +2,7 @@ package server import "github.com/alibaba/kt-connect/pkg/apiserver/common" +// Init ... func Init(context common.Context) { r := NewRouter(context) r.Run(":8000") diff --git a/pkg/apiserver/util/cluster.go b/pkg/apiserver/util/cluster.go index aba21a3..26eae1d 100644 --- a/pkg/apiserver/util/cluster.go +++ b/pkg/apiserver/util/cluster.go @@ -10,6 +10,7 @@ import ( "k8s.io/client-go/tools/clientcmd" ) +// GetKubernetesClient ... func GetKubernetesClient() (clientset kubernetes.Interface, config *restclient.Config, err error) { config, err = GetKubeconfig() if err != nil { @@ -21,6 +22,7 @@ func GetKubernetesClient() (clientset kubernetes.Interface, config *restclient.C return } +// GetKubeconfig ... func GetKubeconfig() (config *restclient.Config, err error) { kubeconfig := filepath.Join(homeDir(), ".kube", "config") if _, err := os.Stat(kubeconfig); os.IsNotExist(err) { diff --git a/pkg/kt/cluster/kubernetes.go b/pkg/kt/cluster/kubernetes.go index 53bdd6e..829bbfa 100644 --- a/pkg/kt/cluster/kubernetes.go +++ b/pkg/kt/cluster/kubernetes.go @@ -1,18 +1,19 @@ package cluster import ( + "fmt" "time" - "k8s.io/apimachinery/pkg/util/intstr" - clusterWatcher "github.com/alibaba/kt-connect/pkg/apiserver/cluster" "github.com/alibaba/kt-connect/pkg/kt/util" + "github.com/alibaba/kt-connect/pkg/kt/vars" "github.com/rs/zerolog/log" appV1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/kubernetes" ) @@ -38,10 +39,17 @@ func (k *Kubernetes) Deployment(name, namespace string) (deployment *appV1.Deplo } // CreateShadow create shadow -func (k *Kubernetes) CreateShadow(name, namespace, image string, labels map[string]string) (podIP, podName string, err error) { +func (k *Kubernetes) CreateShadow(name, namespace, image string, labels map[string]string) (podIP, podName, sshcm string, credential *util.SSHCredential, err error) { return CreateShadow(k.Clientset, name, labels, namespace, image) } +// CreateService create kubernetes service +func (k *Kubernetes) CreateService(name, namespace string, port int, labels map[string]string) (*v1.Service, error) { + cli := k.Clientset.CoreV1().Services(namespace) + svc := generateService(name, namespace, labels, port) + return cli.Create(svc) +} + // ClusterCrids get cluster cirds func (k *Kubernetes) ClusterCrids(podCIDR string) (cidrs []string, err error) { serviceList, err := k.Clientset.CoreV1().Services("").List(metaV1.ListOptions{}) @@ -108,16 +116,15 @@ func RemoveShadow(client *kubernetes.Clientset, namespace, name string) { } } -// CreateService create service in cluster -func CreateService(name, namespace string, - labels map[string]string, - port int, - clientset *kubernetes.Clientset, -) (err error) { - client := clientset.CoreV1().Services(namespace) - svc := generateService(name, namespace, labels, port) - _, err = client.Create(svc) - return err +// RemoveSSHCM remove ssh public key of config map +func RemoveSSHCM(client *kubernetes.Clientset, namespace, name string) { + cli := client.CoreV1().ConfigMaps(namespace) + deletePolicy := metaV1.DeletePropagationBackground + if err := cli.Delete(name, &metaV1.DeleteOptions{ + PropagationPolicy: &deletePolicy, + }); err != nil { + log.Error().Err(err).Str("config map", name).Msg("delete config map failed") + } } // RemoveService create service in cluster @@ -136,14 +143,39 @@ func CreateShadow( labels map[string]string, namespace, image string, -) (podIP, podName string, err error) { +) (podIP, podName, sshcm string, credential *util.SSHCredential, err error) { + + component, version := labels["kt-component"], labels["version"] + sshcm = fmt.Sprintf("kt-%s-public-key-%s", component, version) + + generator, err := util.Generate(util.PrivateKeyPath(component, version)) + if err != nil { + return + } + + labels["kt"] = sshcm + cli := clientset.CoreV1().ConfigMaps(namespace) + _, err = cli.Create(&v1.ConfigMap{ + ObjectMeta: metaV1.ObjectMeta{ + Name: sshcm, + Namespace: namespace, + Labels: labels, + }, + Data: map[string]string{ + vars.SSHAuthKey: string(generator.PublicKey), + }, + }) + if err != nil { + return + } localIPAddress := util.GetOutboundIP() log.Info().Msgf("Client address %s", localIPAddress) labels["remoteAddress"] = localIPAddress + labels["kt"] = name client := clientset.AppsV1().Deployments(namespace) - deployment := generatorDeployment(namespace, name, labels, image) + deployment := generatorDeployment(namespace, name, labels, image, sshcm) result, err := client.Create(deployment) if err != nil { return @@ -157,6 +189,8 @@ func CreateShadow( } podIP = pod.Status.PodIP podName = pod.GetObjectMeta().GetName() + credential = util.NewDefaultSSHCredential() + credential.PrivateKeyPath = generator.PrivateKeyPath return } @@ -255,7 +289,7 @@ func generateService(name, namespace string, labels map[string]string, port int) } -func generatorDeployment(namespace, name string, labels map[string]string, image string) *appV1.Deployment { +func generatorDeployment(namespace, name string, labels map[string]string, image, volume string) *appV1.Deployment { return &appV1.Deployment{ ObjectMeta: metaV1.ObjectMeta{ Name: name, @@ -276,6 +310,30 @@ func generatorDeployment(namespace, name string, labels map[string]string, image Name: "standalone", Image: image, ImagePullPolicy: "Always", + VolumeMounts: []v1.VolumeMount{ + { + Name: "ssh-public-key", + MountPath: fmt.Sprintf("/root/%s", vars.SSHAuthKey), + }, + }, + }, + }, + Volumes: []v1.Volume{ + { + Name: "ssh-public-key", + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: volume, + }, + Items: []v1.KeyToPath{ + { + Key: vars.SSHAuthKey, + Path: "authorized_keys", + }, + }, + }, + }, }, }, }, diff --git a/pkg/kt/cluster/types.go b/pkg/kt/cluster/types.go index d0f6e5a..33d09be 100644 --- a/pkg/kt/cluster/types.go +++ b/pkg/kt/cluster/types.go @@ -1,7 +1,9 @@ package cluster import ( + "github.com/alibaba/kt-connect/pkg/kt/util" appV1 "k8s.io/api/apps/v1" + coreV1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" v1 "k8s.io/client-go/listers/core/v1" ) @@ -24,7 +26,8 @@ type KubernetesInterface interface { Scale(deployment *appV1.Deployment, replicas *int32) (err error) ServiceHosts(namespace string) (hosts map[string]string) ClusterCrids(podCIDR string) (cidrs []string, err error) - CreateShadow(name, namespace, image string, labels map[string]string) (podIP, podName string, err error) + CreateShadow(name, namespace, image string, labels map[string]string) (podIP, podName, sshcm string, credential *util.SSHCredential, err error) + CreateService(name, namespace string, port int, labels map[string]string) (*coreV1.Service, error) } // Kubernetes implements KubernetesInterface diff --git a/pkg/kt/command/connect.go b/pkg/kt/command/connect.go index 3de65f6..ac55b5f 100644 --- a/pkg/kt/command/connect.go +++ b/pkg/kt/command/connect.go @@ -91,7 +91,9 @@ func (action *Action) Connect(options *options.DaemonOptions) (err error) { return } - connectToCluster(&shadow, &kubernetes, options) + if err = connectToCluster(&shadow, &kubernetes, options); err != nil { + return + } s := <-ch log.Info().Msgf("Terminal Signal is %s", s) @@ -107,8 +109,7 @@ func connectToCluster(shadow connect.ShadowInterface, kubernetes cluster.Kuberne } workload := fmt.Sprintf("kt-connect-daemon-%s", strings.ToLower(util.RandomString(5))) - - endPointIP, podName, err := kubernetes.CreateShadow( + endPointIP, podName, sshcm, credential, err := kubernetes.CreateShadow( workload, options.Namespace, options.Image, labels(workload, options), ) @@ -118,23 +119,25 @@ func connectToCluster(shadow connect.ShadowInterface, kubernetes cluster.Kuberne // record shadow name will clean up terminal options.RuntimeOptions.Shadow = workload + options.RuntimeOptions.SSHCM = sshcm cidrs, err := kubernetes.ClusterCrids(options.ConnectOptions.CIDR) if err != nil { return } - return shadow.Outbound(podName, endPointIP, cidrs) + return shadow.Outbound(podName, endPointIP, credential, cidrs) } func labels(workload string, options *options.DaemonOptions) map[string]string { labels := map[string]string{ - "kt": workload, "kt-component": "connect", "control-by": "kt", } for k, v := range util.String2Map(options.Labels) { labels[k] = v } + splits := strings.Split(workload, "-") + labels["version"] = splits[len(splits)-1] return labels } diff --git a/pkg/kt/command/connect_test.go b/pkg/kt/command/connect_test.go index c06522f..8389dd1 100644 --- a/pkg/kt/command/connect_test.go +++ b/pkg/kt/command/connect_test.go @@ -59,11 +59,10 @@ func Test_shouldConnectToCluster(t *testing.T) { ctl := gomock.NewController(t) kubernetes := mock.NewMockKubernetesInterface(ctl) shadow := mock.NewMockShadowInterface(ctl) - - kubernetes.EXPECT().CreateShadow(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("172.168.0.2", "shadowName", nil).AnyTimes() + kubernetes.EXPECT().CreateShadow(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("172.168.0.2", "shadowName", "sshcm", nil, nil).AnyTimes() kubernetes.EXPECT().ClusterCrids(gomock.Any()).Return([]string{"10.10.10.0/24"}, nil) - shadow.EXPECT().Outbound("shadowName", "172.168.0.2", []string{"10.10.10.0/24"}).Return(nil) + shadow.EXPECT().Outbound("shadowName", "172.168.0.2", gomock.Any(), []string{"10.10.10.0/24"}).Return(nil) type args struct { shadow connect.ShadowInterface @@ -91,8 +90,7 @@ func Test_shouldConnectClusterFailWhenFailCreateShadow(t *testing.T) { ctl := gomock.NewController(t) kubernetesInterface := mock.NewMockKubernetesInterface(ctl) shadowInterface := mock.NewMockShadowInterface(ctl) - - kubernetesInterface.EXPECT().CreateShadow(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("", "", errors.New("")).AnyTimes() + kubernetesInterface.EXPECT().CreateShadow(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("", "", "", nil, errors.New("")).AnyTimes() type args struct { shadow connect.ShadowInterface @@ -117,8 +115,7 @@ func Test_shouldConnectClusterFailWhenFailGetCrids(t *testing.T) { ctl := gomock.NewController(t) kubernetes := mock.NewMockKubernetesInterface(ctl) shadow := mock.NewMockShadowInterface(ctl) - - kubernetes.EXPECT().CreateShadow(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("172.168.0.2", "shadowName", nil).AnyTimes() + kubernetes.EXPECT().CreateShadow(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).Return("172.168.0.2", "shadowName", "sshcm", nil, nil).AnyTimes() kubernetes.EXPECT().ClusterCrids(gomock.Any()).Return([]string{}, errors.New("fail to get crid")) type args struct { diff --git a/pkg/kt/command/exchange.go b/pkg/kt/command/exchange.go index 5cff936..7d46b37 100644 --- a/pkg/kt/command/exchange.go +++ b/pkg/kt/command/exchange.go @@ -42,7 +42,6 @@ func newExchangeCommand(options *options.DaemonOptions, action ActionInterface) if len(expose) == 0 { return errors.New("-expose is required") } - return action.Exchange(exchange, options) }, } @@ -65,12 +64,13 @@ func (action *Action) Exchange(exchange string, options *options.DaemonOptions) } // record context inorder to remove after command exit - options.RuntimeOptions.Origin = app.GetObjectMeta().GetName() + options.RuntimeOptions.Origin = app.GetName() options.RuntimeOptions.Replicas = *app.Spec.Replicas - workload := app.GetObjectMeta().GetName() + "-kt-" + strings.ToLower(util.RandomString(5)) - podIP, podName, err := kubernetes.CreateShadow( - workload, options.Namespace, options.Image, getExchangeLables(options.Labels, workload, app)) + workload := app.GetName() + "-kt-" + strings.ToLower(util.RandomString(5)) + + podIP, podName, sshcm, credential, err := kubernetes.CreateShadow( + workload, options.Namespace, options.Image, getExchangeLabels(options.Labels, workload, app)) log.Info().Msgf("create exchange shadow %s in namespace %s", workload, options.Namespace) if err != nil { @@ -79,12 +79,17 @@ func (action *Action) Exchange(exchange string, options *options.DaemonOptions) // record data options.RuntimeOptions.Shadow = workload + options.RuntimeOptions.SSHCM = sshcm down := int32(0) - kubernetes.Scale(app, &down) + if err = kubernetes.Scale(app, &down); err != nil { + return err + } shadow := connect.Create(options) - shadow.Inbound(options.ExchangeOptions.Expose, podName, podIP) + if err = shadow.Inbound(options.ExchangeOptions.Expose, podName, podIP, credential); err != nil { + return err + } s := <-ch log.Info().Msgf("Terminal Signal is %s", s) @@ -92,18 +97,22 @@ func (action *Action) Exchange(exchange string, options *options.DaemonOptions) return nil } -func getExchangeLables(customLabels string, workload string, origin *v1.Deployment) map[string]string { +func getExchangeLabels(customLabels string, workload string, origin *v1.Deployment) map[string]string { labels := map[string]string{ "kt": workload, "kt-component": "exchange", "control-by": "kt", } - for k, v := range origin.Spec.Selector.MatchLabels { - labels[k] = v + if origin != nil { + for k, v := range origin.Spec.Selector.MatchLabels { + labels[k] = v + } } // extra labels must be applied after origin labels for k, v := range util.String2Map(customLabels) { labels[k] = v } + splits := strings.Split(workload, "-") + labels["version"] = splits[len(splits)-1] return labels } diff --git a/pkg/kt/command/mesh.go b/pkg/kt/command/mesh.go index f708532..dc7efb8 100644 --- a/pkg/kt/command/mesh.go +++ b/pkg/kt/command/mesh.go @@ -48,7 +48,6 @@ func newMeshCommand(options *options.DaemonOptions, action ActionInterface) cli. if len(expose) == 0 { return errors.New("-expose is required") } - return action.Mesh(mesh, options) }, } @@ -74,17 +73,17 @@ func (action *Action) Mesh(mesh string, options *options.DaemonOptions) error { workload := app.GetObjectMeta().GetName() + "-kt-" + meshVersion labels := getMeshLabels(workload, meshVersion, app, options) - - podIP, podName, err := kubernetes.CreateShadow(workload, options.Namespace, options.Image, labels) + podIP, podName, sshcm, credential, err := kubernetes.CreateShadow(workload, options.Namespace, options.Image, labels) if err != nil { return err } // record context data options.RuntimeOptions.Shadow = workload + options.RuntimeOptions.SSHCM = sshcm shadow := connect.Create(options) - err = shadow.Inbound(options.MeshOptions.Expose, podName, podIP) + err = shadow.Inbound(options.MeshOptions.Expose, podName, podIP, credential) if err != nil { return err @@ -103,8 +102,10 @@ func getMeshLabels(workload string, meshVersion string, app *v1.Deployment, opti "kt-component": ComponentMesh, "control-by": KubernetesTool, } - for k, v := range app.Spec.Selector.MatchLabels { - labels[k] = v + if app != nil { + for k, v := range app.Spec.Selector.MatchLabels { + labels[k] = v + } } // extra labels must be applied after origin labels for k, v := range util.String2Map(options.Labels) { diff --git a/pkg/kt/command/run.go b/pkg/kt/command/run.go index f34f4f1..48dd899 100644 --- a/pkg/kt/command/run.go +++ b/pkg/kt/command/run.go @@ -3,6 +3,7 @@ package command import ( "errors" "strconv" + "strings" "github.com/alibaba/kt-connect/pkg/kt/cluster" "github.com/alibaba/kt-connect/pkg/kt/connect" @@ -46,8 +47,7 @@ func newRunCommand(options *options.DaemonOptions, action ActionInterface) cli.C // Run create a new service in cluster func (action *Action) Run(service string, options *options.DaemonOptions) error { ch := SetUpCloseHandler(options) - - clientset, err := cluster.GetKubernetesClient(options.KubeConfig) + kubernetes, err := cluster.Create(options.KubeConfig) if err != nil { return err } @@ -56,6 +56,7 @@ func (action *Action) Run(service string, options *options.DaemonOptions) error "control-by": "kt", "kt-component": "run", "kt": service, + "version": strings.ToLower(util.RandomString(5)), } // extra labels must be applied after origin labels @@ -63,7 +64,7 @@ func (action *Action) Run(service string, options *options.DaemonOptions) error labels[k] = v } - podIP, podName, err := cluster.CreateShadow(clientset, service, labels, options.Namespace, options.Image) + podIP, podName, sshcm, credential, err := kubernetes.CreateShadow(service, options.Namespace, options.Image, labels) if err != nil { return err } @@ -71,17 +72,17 @@ func (action *Action) Run(service string, options *options.DaemonOptions) error if options.RunOptions.Expose { log.Info().Msgf("expose deployment %s to %s:%v", service, service, options.RunOptions.Port) - err := cluster.CreateService(service, options.Namespace, labels, options.RunOptions.Port, clientset) - if err != nil { + if _, err = kubernetes.CreateService(service, options.Namespace, options.RunOptions.Port, labels); err != nil { return err } options.RuntimeOptions.Service = service } options.RuntimeOptions.Shadow = service + options.RuntimeOptions.SSHCM = sshcm shadow := connect.Create(options) - err = shadow.Inbound(strconv.Itoa(options.RunOptions.Port), podName, podIP) + err = shadow.Inbound(strconv.Itoa(options.RunOptions.Port), podName, podIP, credential) if err != nil { return err } diff --git a/pkg/kt/command/util.go b/pkg/kt/command/util.go index 35aa896..16e0600 100644 --- a/pkg/kt/command/util.go +++ b/pkg/kt/command/util.go @@ -3,6 +3,7 @@ package command import ( "os" "os/signal" + "strings" "syscall" "github.com/alibaba/kt-connect/pkg/kt/cluster" @@ -92,9 +93,18 @@ func CleanupWorkspace(options *options.DaemonOptions) { cluster.RemoveShadow(client, options.Namespace, options.RuntimeOptions.Shadow) } + if len(options.RuntimeOptions.SSHCM) > 0 { + log.Info().Msgf("- clean sshcm %s", options.RuntimeOptions.SSHCM) + cluster.RemoveSSHCM(client, options.Namespace, options.RuntimeOptions.SSHCM) + } + + removePrivateKey(options) if len(options.RuntimeOptions.Service) > 0 { log.Info().Msgf("- cleanup service %s", options.RuntimeOptions.Service) - cluster.RemoveService(options.RuntimeOptions.Service, options.Namespace, client) + err = cluster.RemoveService(options.RuntimeOptions.Service, options.Namespace, client) + if err != nil { + log.Error().Err(err).Msg("delete service failed") + } } } @@ -107,3 +117,16 @@ func checkConnectRunning(pidFile string) { log.Info().Msgf("'KT Connect' is runing, you can access local app from cluster and localhost") } } + +// removePrivateKey remove the private key of ssh +func removePrivateKey(options *options.DaemonOptions) { + if options.RuntimeOptions.SSHCM == "" { + return + } + splits := strings.Split(options.RuntimeOptions.SSHCM, "-") + component, version := splits[1], splits[len(splits)-1] + file := util.PrivateKeyPath(component, version) + if err := os.Remove(file); !os.IsNotExist(err) { + log.Error().Err(err).Msgf("can't delete %s", file) + } +} diff --git a/pkg/kt/connect/inbound.go b/pkg/kt/connect/inbound.go index c79d043..daa2033 100644 --- a/pkg/kt/connect/inbound.go +++ b/pkg/kt/connect/inbound.go @@ -14,7 +14,7 @@ import ( ) // Inbound mapping local port from cluster -func (s *Shadow) Inbound(exposePort, podName, remoteIP string) (err error) { +func (s *Shadow) Inbound(exposePort, podName, remoteIP string, credential *util.SSHCredential) (err error) { debug := s.Options.Debug kubeConfig := s.Options.KubeConfig namespace := s.Options.Namespace @@ -44,6 +44,6 @@ func (s *Shadow) Inbound(exposePort, podName, remoteIP string) (err error) { localPort = ports[1] remotePort = ports[0] } - cmd := ssh.ForwardRemoteRequestToLocal(localPort, "127.0.0.1", remotePort, localSSHPort) + cmd := ssh.ForwardRemoteRequestToLocal(localPort, credential.RemoteHost, remotePort, credential.PrivateKeyPath, localSSHPort) return exec.BackgroundRun(cmd, "ssh remote port-forward", debug) } diff --git a/pkg/kt/connect/outbound.go b/pkg/kt/connect/outbound.go index 516165d..f8d0a94 100644 --- a/pkg/kt/connect/outbound.go +++ b/pkg/kt/connect/outbound.go @@ -14,12 +14,8 @@ import ( ) // Outbound start vpn connection -func (s *Shadow) Outbound(name, podIP string, cidrs []string) (err error) { +func (s *Shadow) Outbound(name, podIP string, credential *util.SSHCredential, cidrs []string) (err error) { options := s.Options - err = util.PrepareSSHPrivateKey() - if err != nil { - return - } err = exec.BackgroundRun( kubectl.PortForward( options.KubeConfig, @@ -37,9 +33,9 @@ func (s *Shadow) Outbound(name, podIP string, cidrs []string) (err error) { log.Info().Msgf("Start SOCKS5 Proxy: export http_proxy=socks5://127.0.0.1:%d", options.ConnectOptions.Socke5Proxy) log.Info().Msgf("==============================================================") _ = ioutil.WriteFile(".jvmrc", []byte(fmt.Sprintf("-DsocksProxyHost=127.0.0.1\n-DsocksProxyPort=%d", options.ConnectOptions.Socke5Proxy)), 0644) - err = exec.BackgroundRun(ssh.DynamicForwardLocalRequestToRemote("127.0.0.1", options.ConnectOptions.SSHPort, options.ConnectOptions.Socke5Proxy), "vpn(ssh)", options.Debug) + err = exec.BackgroundRun(ssh.DynamicForwardLocalRequestToRemote(credential.RemoteHost, credential.PrivateKeyPath, options.ConnectOptions.SSHPort, options.ConnectOptions.Socke5Proxy), "vpn(ssh)", options.Debug) } else { - err = exec.BackgroundRun(sshuttle.SSHUttle("127.0.0.1", options.ConnectOptions.SSHPort, podIP, options.ConnectOptions.DisableDNS, cidrs, options.Debug), "vpn(sshuttle)", options.Debug) + err = exec.BackgroundRun(sshuttle.SSHUttle(credential.RemoteHost, credential.PrivateKeyPath, options.ConnectOptions.SSHPort, podIP, options.ConnectOptions.DisableDNS, cidrs, options.Debug), "vpn(sshuttle)", options.Debug) } if err != nil { return diff --git a/pkg/kt/connect/types.go b/pkg/kt/connect/types.go index a36c1bd..a09dedc 100644 --- a/pkg/kt/connect/types.go +++ b/pkg/kt/connect/types.go @@ -1,11 +1,14 @@ package connect -import "github.com/alibaba/kt-connect/pkg/kt/options" +import ( + "github.com/alibaba/kt-connect/pkg/kt/options" + "github.com/alibaba/kt-connect/pkg/kt/util" +) // ShadowInterface shadow interface type ShadowInterface interface { - Inbound(exposePort, podName, remoteIP string) (err error) - Outbound(name, podIP string, cidrs []string) (err error) + Inbound(exposePort, podName, remoteIP string, credential *util.SSHCredential) (err error) + Outbound(name, podIP string, credential *util.SSHCredential, cidrs []string) (err error) } // Shadow shadow diff --git a/pkg/kt/exec/run.go b/pkg/kt/exec/run.go index b0e102a..dfd2f87 100644 --- a/pkg/kt/exec/run.go +++ b/pkg/kt/exec/run.go @@ -29,7 +29,7 @@ func BackgroundRun(cmd *exec.Cmd, name string, debug bool) (err error) { func runCmd(cmd *exec.Cmd, name string, debug bool) (err error) { log.Debug().Msgf("Child, os.Args = %+v", os.Args) - log.Debug().Msgf("Child, cmd.Args = %+v", cmd.Args) + log.Debug().Msgf("Child, name = %s, cmd.Args = %+v", name, cmd.Args) var stdoutBuf, stderrBuf bytes.Buffer stdoutIn, _ := cmd.StdoutPipe() diff --git a/pkg/kt/exec/ssh/commands.go b/pkg/kt/exec/ssh/commands.go index d9b586d..99c91b3 100644 --- a/pkg/kt/exec/ssh/commands.go +++ b/pkg/kt/exec/ssh/commands.go @@ -3,8 +3,6 @@ package ssh import ( "fmt" "os/exec" - - "github.com/alibaba/kt-connect/pkg/kt/util" ) // Version check sshuttle version @@ -13,11 +11,11 @@ func Version() *exec.Cmd { } // ForwardRemoteRequestToLocal ssh remote port forward -func ForwardRemoteRequestToLocal(localPort, remoteHost, remotePort string, remoteSSHPort int) *exec.Cmd { +func ForwardRemoteRequestToLocal(localPort, remoteHost, remotePort, privateKeyPath string, remoteSSHPort int) *exec.Cmd { return exec.Command("ssh", "-oStrictHostKeyChecking=no", "-oUserKnownHostsFile=/dev/null", - "-i", util.PrivateKeyPath(), + "-i", privateKeyPath, "-R", remotePort+":127.0.0.1:"+localPort, fmt.Sprintf("root@%s", remoteHost), "-p"+fmt.Sprintf("%d", remoteSSHPort), "sh", "loop.sh", @@ -25,11 +23,11 @@ func ForwardRemoteRequestToLocal(localPort, remoteHost, remotePort string, remot } // DynamicForwardLocalRequestToRemote ssh remote port forward -func DynamicForwardLocalRequestToRemote(remoteHost string, remoteSSHPort int, proxyPort int) *exec.Cmd { +func DynamicForwardLocalRequestToRemote(remoteHost, privateKeyPath string, remoteSSHPort int, proxyPort int) *exec.Cmd { return exec.Command("ssh", "-oStrictHostKeyChecking=no", "-oUserKnownHostsFile=/dev/null", - "-i", util.PrivateKeyPath(), + "-i", privateKeyPath, "-D", fmt.Sprintf("%d", proxyPort), fmt.Sprintf("root@%s", remoteHost), "-p"+fmt.Sprintf("%d", remoteSSHPort), "sh", "loop.sh", diff --git a/pkg/kt/exec/sshuttle/commands.go b/pkg/kt/exec/sshuttle/commands.go index d1f7942..414a8a4 100644 --- a/pkg/kt/exec/sshuttle/commands.go +++ b/pkg/kt/exec/sshuttle/commands.go @@ -2,7 +2,6 @@ package sshuttle import ( "fmt" - "github.com/alibaba/kt-connect/pkg/kt/util" "os/exec" ) @@ -12,7 +11,7 @@ func Version() *exec.Cmd { } // SSHUttle ssh-baed vpn connect -func SSHUttle(remoteHost string, remotePort int, DNSServer string, disableDNS bool, cidrs []string, debug bool) *exec.Cmd { +func SSHUttle(remoteHost, privateKeyPath string, remotePort int, DNSServer string, disableDNS bool, cidrs []string, debug bool) *exec.Cmd { args := []string{} if !disableDNS { args = append(args, "--dns", "--to-ns", DNSServer) @@ -22,7 +21,7 @@ func SSHUttle(remoteHost string, remotePort int, DNSServer string, disableDNS bo args = append(args, "-v") } - subCommand := fmt.Sprintf("ssh -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -i %s", util.PrivateKeyPath()) + subCommand := fmt.Sprintf("ssh -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -i %s", privateKeyPath) args = append(args, "-e", subCommand, "-r", fmt.Sprintf("root@%s:%d", remoteHost, remotePort), "-x", remoteHost) args = append(args, cidrs...) return exec.Command("sshuttle", args...) diff --git a/pkg/kt/options/options.go b/pkg/kt/options/options.go index 16a0726..0a9eb01 100644 --- a/pkg/kt/options/options.go +++ b/pkg/kt/options/options.go @@ -35,6 +35,8 @@ type runtimeOptions struct { AppHome string // Shadow deployment name Shadow string + // ssh public key name of config map. format is kt-xxx(component)-public-key-xxx(version) + SSHCM string // The origin app name Origin string // The origin repicas diff --git a/pkg/kt/util/ssh.go b/pkg/kt/util/ssh.go new file mode 100644 index 0000000..b448256 --- /dev/null +++ b/pkg/kt/util/ssh.go @@ -0,0 +1,129 @@ +package util + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "fmt" + "io/ioutil" + "os" + "path/filepath" + + "github.com/alibaba/kt-connect/pkg/kt/vars" + "github.com/rs/zerolog/log" + "golang.org/x/crypto/ssh" +) + +// SSHCredential ssh info +type SSHCredential struct { + RemoteHost string + Port string + PrivateKeyPath string +} + +// SSHGenerator ssh key pair generator +type SSHGenerator struct { + PrivateKey, PublicKey []byte + PrivateKeyPath string +} + +// NewDefaultSSHCredential ... +func NewDefaultSSHCredential() *SSHCredential { + return &SSHCredential{ + Port: "2222", + RemoteHost: "127.0.0.1", + } +} + +// Generate generate SSHGenerator +func Generate(privateKeyPath string) (*SSHGenerator, error) { + privateKey, err := generatePrivateKey(vars.SSHBitSize) + if err != nil { + return nil, err + } + + publicKeyBytes, err := generatePublicKey(&privateKey.PublicKey) + if err != nil { + return nil, err + } + privateKeyBytes := encodePrivateKeyToPEM(privateKey) + + ssh := &SSHGenerator{ + PrivateKey: privateKeyBytes, + PrivateKeyPath: privateKeyPath, + PublicKey: publicKeyBytes, + } + err = writePrivateKey(ssh.PrivateKeyPath, ssh.PrivateKey) + return ssh, err +} + +// PrivateKeyPath ... +func PrivateKeyPath(component, identifier string) string { + return fmt.Sprintf("%s/ktctl/%s/"+vars.SSHPrivateKeyName, HomeDir(), component, identifier) +} + +// generatePrivateKey creates a RSA Private Key of specified byte size +func generatePrivateKey(bitSize int) (*rsa.PrivateKey, error) { + // Private Key generation + privateKey, err := rsa.GenerateKey(rand.Reader, bitSize) + if err != nil { + return nil, err + } + + // Validate Private Key + err = privateKey.Validate() + if err != nil { + return nil, err + } + + log.Info().Msg("private Key generated") + return privateKey, nil +} + +// encodePrivateKeyToPEM encodes Private Key from RSA to PEM format +func encodePrivateKeyToPEM(privateKey *rsa.PrivateKey) []byte { + // Get ASN.1 DER format + privDER := x509.MarshalPKCS1PrivateKey(privateKey) + + // pem.Block + privBlock := pem.Block{ + Type: "RSA PRIVATE KEY", + Headers: nil, + Bytes: privDER, + } + + // Private key in PEM format + privatePEM := pem.EncodeToMemory(&privBlock) + + return privatePEM +} + +// generatePublicKey take a rsa.PublicKey and return bytes suitable for writing to .pub file +// returns in the format "ssh-rsa ..." +func generatePublicKey(privatekey *rsa.PublicKey) ([]byte, error) { + publicRsaKey, err := ssh.NewPublicKey(privatekey) + if err != nil { + return nil, err + } + + pubKeyBytes := ssh.MarshalAuthorizedKey(publicRsaKey) + log.Info().Msg("public key generated") + return pubKeyBytes, nil +} + +// writePrivateKey write ssh private key to privateKeyPath +func writePrivateKey(privateKeyPath string, data []byte) error { + dir := filepath.Dir(privateKeyPath) + if _, err := os.Stat(dir); os.IsNotExist(err) { + if err = os.MkdirAll(dir, 0700); err != nil { + log.Error().Err(err).Str("dir", dir).Msg("can't create dir") + return err + } + } + if err := ioutil.WriteFile(privateKeyPath, data, 0700); err != nil { + log.Error().Err(err).Str("file", privateKeyPath).Msg("write ssh private key failed") + return err + } + return nil +} diff --git a/pkg/kt/util/strings.go b/pkg/kt/util/strings.go index 4248c5f..c105016 100644 --- a/pkg/kt/util/strings.go +++ b/pkg/kt/util/strings.go @@ -3,12 +3,14 @@ package util import ( "math/rand" "strings" + "time" ) var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") // RandomString Generate RandomString func RandomString(n int) string { + rand.Seed(time.Now().Unix()) b := make([]rune, n) for i := range b { b[i] = letterRunes[rand.Intn(len(letterRunes))] @@ -27,4 +29,4 @@ func String2Map(str string) map[string]string { } } return res -} \ No newline at end of file +} diff --git a/pkg/kt/util/system.go b/pkg/kt/util/system.go index d61f6f7..455aa01 100644 --- a/pkg/kt/util/system.go +++ b/pkg/kt/util/system.go @@ -3,59 +3,13 @@ package util import ( "fmt" "io/ioutil" - "math/rand" "os" "runtime" - "time" "github.com/lextoumbourou/goodhosts" "github.com/rs/zerolog/log" ) -var ( - pk []byte -) - -func init() { - rand.Seed(time.Now().UnixNano()) - pk = []byte("-----BEGIN RSA PRIVATE KEY-----\n" + - "MIIEpAIBAAKCAQEAvSVAezJDBhrNDhLhuCaKrvdtCFTdqJmGLGyfBqEYb3p4a91g\n" + - "l4gD2LGwiRlgpwU4oSECeMmwP53C4vrfPKY45/+8lncIkE/4E+8hHssXkHaqQrjE\n" + - "FtePfxZ6/xi84kUbWNNV4IGAeFwXtq9GszQ+kWMNT5QmuexOXOlqq7W4CIAUe3uX\n" + - "29WCp3OGiBeP4ORDraRa/1bwBH+Cq0UxEYT+6EuDU0YzF3JF4H8At6NdgElAuezE\n" + - "wI84p5LNr1HmTPndcHJtX2+POKEoNYBxPekEyJbqExIR2dLRUytlX5tIacKkMBCJ\n" + - "aX7DBtJzWX7BxfjRfXjzNOpufuTU9BsknyJlFwIDAQABAoIBAQCS606s4xvAsCy7\n" + - "U9tUyUtMIRDmOdV7UtUvyKe15Igwf3bugiS3T4V9Wnh/5eB3m8yjDBr5a+ClaYup\n" + - "96hTWeI2AyWf0pIqVpOiGEsnuiVxp1sVPKPEAmiKFRIw+CwvrfJSCsZX/v+lfhNF\n" + - "adyG8nvvPntmZvO102IDNaQQALUUk+J69yvtb7ekfvZCSmanXx/R7y2+u8Hd8wtf\n" + - "fpyVcM1g7YZwhxto2uKyUE1T/myOT5+wULfYTMNynsLB6dXJJP+a89I6cRdFCIib\n" + - "kqoZ5FuaTXXucrgWGDne6DcvNbiZi1f8LRb6RFnlvv6D42xyjelyoGY7BktKsFwJ\n" + - "NwLR1lBhAoGBANyGB7ti190DZoDGQvIpSHd+JeZHoK3g49VNxGFU+SAhXa8gQn6K\n" + - "Xi5qNRD2XLTEnT36U20/bkcDv0oSTZikJhU0OqxgouVO3YZ2cZhXcoPZmf+vhgzI\n" + - "ufv0T8/HlQyr7Sp9VqfqlC7u1P83VbNro/D3V+wtNKog4g++DvKtsh8JAoGBANuS\n" + - "9XDaAnq5K0rjShNqCMRyHx6+kFPaLpL4kt1f4yra8w/m3pes63sO1vz/4wOhoalL\n" + - "imAEqTKTblinPhjCxbe4e/WqnAQM05XROdiGer2RhBIMCo2/YE3WLCWAyVCDtd3B\n" + - "Te9rPynSsAmtgDRuftusY7TAIuwZuG4K71Gw8UsfAoGAYmRm5MPYXqNaw9AyJIwo\n" + - "6i/dxx5kYdB6tzxoh6j7MsvQWggBwyYHmZwHq1bQzFMBeZrMSG1JzeOtIOaDurxa\n" + - "xZE1MJ45cCi9DHaifn9d99hKLtvo6qFQ4ksCpUl+hlXbjt63oFo43avwWyMcWN6J\n" + - "GkWx9A3DdrkPREjfsIWxeMkCgYEAtDhv6duWk2IujX32y+6JGaxNrK9eyORYu9r4\n" + - "uGi+jOs++ztUUgvlD5EDlo70poNgrBLLlbndohxuQqeqiSo8nGn4nJAXFB/u/pXH\n" + - "M9hVIAky7JkjhGqiweBbRcDp+4LPoB7MOAm/wzUhth/JDb/vsaBSCgZ143HM9c1V\n" + - "1qgztKMCgYAUhQRJB6ofGqiGsPN2KZw+0IoPNS3Tk0NTjzVh2o927B8zb0T0bO5e\n" + - "qe0OO7FFGcON6uSOkGu2p9KHUEm6OFaQLjdysjrGI7GVRYW7D/SSLidRREv2A70R\n" + - "f0/Mi8v9nD4ztroXQDeeL8O4rFTnfRdqs+MZ/MYoq9C5iE1IHJm7KQ==\n" + - "-----END RSA PRIVATE KEY-----") -} - -// PrepareSSHPrivateKey generator ssh private key -func PrepareSSHPrivateKey() (err error) { - err = ioutil.WriteFile(PrivateKeyPath(), pk, 400) - if err != nil { - log.Error().Msgf("Fails create temp ssh private key") - } - return -} - // IsDaemonRunning check daemon is running or not func IsDaemonRunning(pidFile string) bool { if _, err := os.Stat(pidFile); os.IsNotExist(err) { @@ -75,13 +29,6 @@ func HomeDir() string { return "/root" } -// PrivateKeyPath Get ssh private key path -func PrivateKeyPath() string { - userHome := HomeDir() - privateKey := fmt.Sprintf("%s/.kt_id_rsa", userHome) - return privateKey -} - // CreateDirIfNotExist create dir func CreateDirIfNotExist(dir string) { if _, err := os.Stat(dir); os.IsNotExist(err) { diff --git a/pkg/kt/vars/vars.go b/pkg/kt/vars/vars.go new file mode 100644 index 0000000..30d2cd1 --- /dev/null +++ b/pkg/kt/vars/vars.go @@ -0,0 +1,10 @@ +package vars + +var ( + // SSHPrivateKeyName ssh private key name + SSHPrivateKeyName = "kt_%s_id_rsa" + // SSHBitSize ssh bit size + SSHBitSize = 2048 + // SSHAuthKey auth key name + SSHAuthKey = "authorized" +) diff --git a/pkg/mockd/mock/action_mock.go b/pkg/mockd/mock/action_mock.go index c83f990..01f3deb 100644 --- a/pkg/mockd/mock/action_mock.go +++ b/pkg/mockd/mock/action_mock.go @@ -5,10 +5,9 @@ package mock import ( - reflect "reflect" - options "github.com/alibaba/kt-connect/pkg/kt/options" gomock "github.com/golang/mock/gomock" + reflect "reflect" ) // MockActionInterface is a mock of ActionInterface interface diff --git a/pkg/mockd/mock/connect_mock.go b/pkg/mockd/mock/connect_mock.go index 766f6ce..e5a8dab 100644 --- a/pkg/mockd/mock/connect_mock.go +++ b/pkg/mockd/mock/connect_mock.go @@ -5,9 +5,9 @@ package mock import ( - reflect "reflect" - + util "github.com/alibaba/kt-connect/pkg/kt/util" gomock "github.com/golang/mock/gomock" + reflect "reflect" ) // MockShadowInterface is a mock of ShadowInterface interface @@ -34,29 +34,29 @@ func (m *MockShadowInterface) EXPECT() *MockShadowInterfaceMockRecorder { } // Inbound mocks base method -func (m *MockShadowInterface) Inbound(exposePort, podName, remoteIP string) error { +func (m *MockShadowInterface) Inbound(exposePort, podName, remoteIP string, credential *util.SSHCredential) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Inbound", exposePort, podName, remoteIP) + ret := m.ctrl.Call(m, "Inbound", exposePort, podName, remoteIP, credential) ret0, _ := ret[0].(error) return ret0 } // Inbound indicates an expected call of Inbound -func (mr *MockShadowInterfaceMockRecorder) Inbound(exposePort, podName, remoteIP interface{}) *gomock.Call { +func (mr *MockShadowInterfaceMockRecorder) Inbound(exposePort, podName, remoteIP, credential interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Inbound", reflect.TypeOf((*MockShadowInterface)(nil).Inbound), exposePort, podName, remoteIP) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Inbound", reflect.TypeOf((*MockShadowInterface)(nil).Inbound), exposePort, podName, remoteIP, credential) } // Outbound mocks base method -func (m *MockShadowInterface) Outbound(name, podIP string, cidrs []string) error { +func (m *MockShadowInterface) Outbound(name, podIP string, credential *util.SSHCredential, cidrs []string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Outbound", name, podIP, cidrs) + ret := m.ctrl.Call(m, "Outbound", name, podIP, credential, cidrs) ret0, _ := ret[0].(error) return ret0 } // Outbound indicates an expected call of Outbound -func (mr *MockShadowInterfaceMockRecorder) Outbound(name, podIP, cidrs interface{}) *gomock.Call { +func (mr *MockShadowInterfaceMockRecorder) Outbound(name, podIP, credential, cidrs interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Outbound", reflect.TypeOf((*MockShadowInterface)(nil).Outbound), name, podIP, cidrs) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Outbound", reflect.TypeOf((*MockShadowInterface)(nil).Outbound), name, podIP, credential, cidrs) } diff --git a/pkg/mockd/mock/kubernetes_mock.go b/pkg/mockd/mock/kubernetes_mock.go index 4206dd0..eaa9e0c 100644 --- a/pkg/mockd/mock/kubernetes_mock.go +++ b/pkg/mockd/mock/kubernetes_mock.go @@ -5,10 +5,11 @@ package mock import ( - reflect "reflect" - + util "github.com/alibaba/kt-connect/pkg/kt/util" gomock "github.com/golang/mock/gomock" v1 "k8s.io/api/apps/v1" + v10 "k8s.io/api/core/v1" + reflect "reflect" ) // MockKubernetesInterface is a mock of KubernetesInterface interface @@ -93,13 +94,15 @@ func (mr *MockKubernetesInterfaceMockRecorder) ClusterCrids(podCIDR interface{}) } // CreateShadow mocks base method -func (m *MockKubernetesInterface) CreateShadow(name, namespace, image string, labels map[string]string) (string, string, error) { +func (m *MockKubernetesInterface) CreateShadow(name, namespace, image string, labels map[string]string) (string, string, string, *util.SSHCredential, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "CreateShadow", name, namespace, image, labels) ret0, _ := ret[0].(string) ret1, _ := ret[1].(string) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 + ret2, _ := ret[2].(string) + ret3, _ := ret[3].(*util.SSHCredential) + ret4, _ := ret[4].(error) + return ret0, ret1, ret2, ret3, ret4 } // CreateShadow indicates an expected call of CreateShadow @@ -107,3 +110,18 @@ func (mr *MockKubernetesInterfaceMockRecorder) CreateShadow(name, namespace, ima mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateShadow", reflect.TypeOf((*MockKubernetesInterface)(nil).CreateShadow), name, namespace, image, labels) } + +// CreateService mocks base method +func (m *MockKubernetesInterface) CreateService(name, namespace string, port int, labels map[string]string) (*v10.Service, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateService", name, namespace, port, labels) + ret0, _ := ret[0].(*v10.Service) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateService indicates an expected call of CreateService +func (mr *MockKubernetesInterfaceMockRecorder) CreateService(name, namespace, port, labels interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateService", reflect.TypeOf((*MockKubernetesInterface)(nil).CreateService), name, namespace, port, labels) +}