diff --git a/cmd/ctl/BUILD.bazel b/cmd/ctl/BUILD.bazel
index f8edae38b..395e75fd3 100644
--- a/cmd/ctl/BUILD.bazel
+++ b/cmd/ctl/BUILD.bazel
@@ -38,6 +38,7 @@ filegroup(
         "//cmd/ctl/pkg/create:all-srcs",
         "//cmd/ctl/pkg/deny:all-srcs",
         "//cmd/ctl/pkg/experimental:all-srcs",
+        "//cmd/ctl/pkg/factory:all-srcs",
         "//cmd/ctl/pkg/inspect:all-srcs",
         "//cmd/ctl/pkg/install:all-srcs",
         "//cmd/ctl/pkg/renew:all-srcs",
diff --git a/cmd/ctl/cmd/BUILD.bazel b/cmd/ctl/cmd/BUILD.bazel
index 5133b25c0..c376d025c 100644
--- a/cmd/ctl/cmd/BUILD.bazel
+++ b/cmd/ctl/cmd/BUILD.bazel
@@ -18,9 +18,7 @@ go_library(
         "//cmd/ctl/pkg/version:go_default_library",
         "@com_github_spf13_cobra//:go_default_library",
         "@io_k8s_cli_runtime//pkg/genericclioptions:go_default_library",
-        "@io_k8s_client_go//plugin/pkg/client/auth:go_default_library",
         "@io_k8s_klog_v2//:go_default_library",
-        "@io_k8s_kubectl//pkg/cmd/util:go_default_library",
     ],
 )
 
diff --git a/cmd/ctl/cmd/cmd.go b/cmd/ctl/cmd/cmd.go
index 3e7f6826c..c1b393d4c 100644
--- a/cmd/ctl/cmd/cmd.go
+++ b/cmd/ctl/cmd/cmd.go
@@ -25,10 +25,7 @@ import (
 
 	"github.com/spf13/cobra"
 	"k8s.io/cli-runtime/pkg/genericclioptions"
-	// Load all auth plugins
-	_ "k8s.io/client-go/plugin/pkg/client/auth"
 	"k8s.io/klog/v2"
-	"k8s.io/kubectl/pkg/cmd/util"
 
 	"github.com/jetstack/cert-manager/cmd/ctl/pkg/approve"
 	"github.com/jetstack/cert-manager/cmd/ctl/pkg/check"
@@ -51,12 +48,6 @@ kubectl cert-manager is a CLI tool manage and configure cert-manager resources f
 	}
 	cmds.SetUsageTemplate(usageTemplate)
 
-	kubeConfigFlags := genericclioptions.NewConfigFlags(true)
-	kubeConfigFlags.AddFlags(cmds.PersistentFlags())
-	matchVersionKubeConfigFlags := util.NewMatchVersionFlags(kubeConfigFlags)
-	matchVersionKubeConfigFlags.AddFlags(cmds.PersistentFlags())
-	factory := util.NewFactory(matchVersionKubeConfigFlags)
-
 	cmds.Flags().AddGoFlagSet(flag.CommandLine)
 	flag.CommandLine.Parse([]string{})
 	fakefs := flag.NewFlagSet("fake", flag.ExitOnError)
@@ -69,16 +60,16 @@ kubectl cert-manager is a CLI tool manage and configure cert-manager resources f
 	ioStreams := genericclioptions.IOStreams{In: in, Out: out, ErrOut: err}
 	cmds.AddCommand(version.NewCmdVersion(ctx, ioStreams, factory))
 	cmds.AddCommand(convert.NewCmdConvert(ctx, ioStreams))
-	cmds.AddCommand(create.NewCmdCreate(ctx, ioStreams, factory))
-	cmds.AddCommand(renew.NewCmdRenew(ctx, ioStreams, factory))
-	cmds.AddCommand(status.NewCmdStatus(ctx, ioStreams, factory))
-	cmds.AddCommand(inspect.NewCmdInspect(ctx, ioStreams, factory))
-	cmds.AddCommand(approve.NewCmdApprove(ctx, ioStreams, factory))
-	cmds.AddCommand(deny.NewCmdDeny(ctx, ioStreams, factory))
-	cmds.AddCommand(check.NewCmdCheck(ctx, ioStreams, factory))
+	cmds.AddCommand(create.NewCmdCreate(ctx, ioStreams))
+	cmds.AddCommand(renew.NewCmdRenew(ctx, ioStreams))
+	cmds.AddCommand(status.NewCmdStatus(ctx, ioStreams))
+	cmds.AddCommand(inspect.NewCmdInspect(ctx, ioStreams))
+	cmds.AddCommand(approve.NewCmdApprove(ctx, ioStreams))
+	cmds.AddCommand(deny.NewCmdDeny(ctx, ioStreams))
+	cmds.AddCommand(check.NewCmdCheck(ctx, ioStreams))
 
 	// Experimental features
-	cmds.AddCommand(experimental.NewCmdExperimental(ctx, ioStreams, factory))
+	cmds.AddCommand(experimental.NewCmdExperimental(ctx, ioStreams))
 
 	return cmds
 }