set correct exit codes when exiting

Signed-off-by: Inteon <42113979+inteon@users.noreply.github.com>

BUILD.bazel

@@ -60,6 +60,7 @@ filegroup(
         "//cmd/cainjector:all-srcs",
         "//cmd/controller:all-srcs",
         "//cmd/ctl:all-srcs",
+        "//cmd/util:all-srcs",
         "//cmd/webhook:all-srcs",
         "//deploy:all-srcs",
         "//devel:all-srcs",

cmd/acmesolver/BUILD.bazel

@@ -8,7 +8,7 @@ go_library(
     visibility = ["//visibility:private"],
     deps = [
         "//cmd/acmesolver/app:go_default_library",
-        "//pkg/util/cmd:go_default_library",
+        "//cmd/util:go_default_library",
     ],
 )
 

cmd/acmesolver/app/BUILD.bazel

@@ -6,9 +6,9 @@ go_library(
     importpath = "github.com/jetstack/cert-manager/cmd/acmesolver/app",
     visibility = ["//visibility:public"],
     deps = [
+        "//cmd/util:go_default_library",
         "//pkg/issuer/acme/http/solver:go_default_library",
         "//pkg/logs:go_default_library",
-        "//pkg/util:go_default_library",
         "@com_github_spf13_cobra//:go_default_library",
     ],
 )

cmd/acmesolver/app/app.go

@@ -21,9 +21,9 @@ import (
 
 	"github.com/spf13/cobra"
 
+	"github.com/jetstack/cert-manager/cmd/util"
 	"github.com/jetstack/cert-manager/pkg/issuer/acme/http/solver"
 	logf "github.com/jetstack/cert-manager/pkg/logs"
-	"github.com/jetstack/cert-manager/pkg/util"
 )
 
 func NewACMESolverCommand(stopCh <-chan struct{}) *cobra.Command {

cmd/acmesolver/main.go

@@ -21,7 +21,7 @@ import (
 	"os"
 
 	"github.com/jetstack/cert-manager/cmd/acmesolver/app"
-	utilcmd "github.com/jetstack/cert-manager/pkg/util/cmd"
+	"github.com/jetstack/cert-manager/cmd/util"
 )
 
 // acmesolver solves ACME http-01 challenges. This is intended to run as a pod
@@ -29,10 +29,13 @@ import (
 // cert-manager.
 
 func main() {
-	stopCh := utilcmd.SetupSignalHandler()
+	stopCh, exit := util.SetupExitHandler(util.GracefulShutdown)
+	defer exit() // This function might call os.Exit, so defer last
+
 	cmd := app.NewACMESolverCommand(stopCh)
 
 	if err := cmd.Execute(); err != nil {
 		fmt.Fprintf(os.Stderr, "%s\n", err)
+		util.SetExitCode(err)
 	}
 }

cmd/cainjector/BUILD.bazel

@@ -8,9 +8,8 @@ go_library(
     visibility = ["//visibility:private"],
     deps = [
         "//cmd/cainjector/app:go_default_library",
+        "//cmd/util:go_default_library",
         "//pkg/logs:go_default_library",
-        "//pkg/util:go_default_library",
-        "//pkg/util/cmd:go_default_library",
         "@io_k8s_sigs_controller_runtime//:go_default_library",
     ],
 )

cmd/cainjector/main.go

@@ -25,19 +25,20 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 
 	"github.com/jetstack/cert-manager/cmd/cainjector/app"
+	"github.com/jetstack/cert-manager/cmd/util"
 	logf "github.com/jetstack/cert-manager/pkg/logs"
-	"github.com/jetstack/cert-manager/pkg/util"
-	utilcmd "github.com/jetstack/cert-manager/pkg/util/cmd"
 )
 
 func main() {
+	// Set up signal handlers and a cancellable context which gets cancelled on
+	// when either SIGINT or SIGTERM are received.
+	stopCh, exit := util.SetupExitHandler(util.GracefulShutdown)
+	defer exit() // This function might call os.Exit, so defer last
+
 	logf.InitLogs(flag.CommandLine)
 	defer logf.FlushLogs()
 	ctrl.SetLogger(logf.Log)
 
-	// Set up signal handlers and a cancellable context which gets cancelled on
-	// when either SIGINT or SIGTERM are received.
-	stopCh := utilcmd.SetupSignalHandler()
 	ctx := util.ContextWithStopCh(context.Background(), stopCh)
 
 	cmd := app.NewCommandStartInjectorController(ctx, os.Stdout, os.Stderr)
@@ -46,6 +47,6 @@ func main() {
 	flag.CommandLine.Parse([]string{})
 	if err := cmd.Execute(); err != nil {
 		cmd.PrintErrln(err)
-		os.Exit(1)
+		util.SetExitCode(err)
 	}
 }

cmd/controller/BUILD.bazel

@@ -8,8 +8,8 @@ go_library(
     visibility = ["//visibility:private"],
     deps = [
         "//cmd/controller/app:go_default_library",
+        "//cmd/util:go_default_library",
         "//pkg/logs:go_default_library",
-        "//pkg/util/cmd:go_default_library",
     ],
 )
 

cmd/controller/app/BUILD.bazel

@@ -10,6 +10,7 @@ go_library(
     visibility = ["//visibility:public"],
     deps = [
         "//cmd/controller/app/options:go_default_library",
+        "//cmd/util:go_default_library",
         "//pkg/acme/accounts:go_default_library",
         "//pkg/client/clientset/versioned:go_default_library",
         "//pkg/client/clientset/versioned/scheme:go_default_library",

cmd/controller/app/controller.go

@@ -42,6 +42,7 @@ import (
 	gwinformers "sigs.k8s.io/gateway-api/pkg/client/informers/externalversions"
 
 	"github.com/jetstack/cert-manager/cmd/controller/app/options"
+	cmdutil "github.com/jetstack/cert-manager/cmd/util"
 	"github.com/jetstack/cert-manager/pkg/acme/accounts"
 	clientset "github.com/jetstack/cert-manager/pkg/client/clientset/versioned"
 	intscheme "github.com/jetstack/cert-manager/pkg/client/clientset/versioned/scheme"
@@ -63,7 +64,7 @@ const controllerAgentName = "cert-manager"
 const resyncPeriod = 10 * time.Hour
 
 func Run(opts *options.ControllerOptions, stopCh <-chan struct{}) {
-	rootCtx := util.ContextWithStopCh(context.Background(), stopCh)
+	rootCtx := cmdutil.ContextWithStopCh(context.Background(), stopCh)
 	rootCtx = logf.NewContext(rootCtx, nil, "controller")
 	log := logf.FromContext(rootCtx)
 

cmd/controller/main.go

@@ -18,24 +18,25 @@ package main
 
 import (
 	"flag"
-	"os"
 
 	"github.com/jetstack/cert-manager/cmd/controller/app"
+	"github.com/jetstack/cert-manager/cmd/util"
 	logf "github.com/jetstack/cert-manager/pkg/logs"
-	utilcmd "github.com/jetstack/cert-manager/pkg/util/cmd"
 )
 
 func main() {
+	stopCh, exit := util.SetupExitHandler(util.GracefulShutdown)
+	defer exit() // This function might call os.Exit, so defer last
+
 	logf.InitLogs(flag.CommandLine)
 	defer logf.FlushLogs()
 
-	stopCh := utilcmd.SetupSignalHandler()
 	cmd := app.NewCommandStartCertManagerController(stopCh)
 	cmd.Flags().AddGoFlagSet(flag.CommandLine)
 
 	flag.CommandLine.Parse([]string{})
 	if err := cmd.Execute(); err != nil {
 		logf.Log.Error(err, "error executing command")
-		os.Exit(1)
+		util.SetExitCode(err)
 	}
 }

cmd/ctl/BUILD.bazel

@@ -8,8 +8,7 @@ go_library(
     visibility = ["//visibility:private"],
     deps = [
         "//cmd/ctl/cmd:go_default_library",
-        "//pkg/util:go_default_library",
-        "//pkg/util/cmd:go_default_library",
+        "//cmd/util:go_default_library",
     ],
 )
 

cmd/ctl/cmd/cmd.go

@@ -28,7 +28,7 @@ import (
 	// Load all auth plugins
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 	"k8s.io/klog/v2"
-	cmdutil "k8s.io/kubectl/pkg/cmd/util"
+	"k8s.io/kubectl/pkg/cmd/util"
 
 	"github.com/jetstack/cert-manager/cmd/ctl/pkg/approve"
 	"github.com/jetstack/cert-manager/cmd/ctl/pkg/check"
@@ -53,9 +53,9 @@ kubectl cert-manager is a CLI tool manage and configure cert-manager resources f
 
 	kubeConfigFlags := genericclioptions.NewConfigFlags(true)
 	kubeConfigFlags.AddFlags(cmds.PersistentFlags())
-	matchVersionKubeConfigFlags := cmdutil.NewMatchVersionFlags(kubeConfigFlags)
+	matchVersionKubeConfigFlags := util.NewMatchVersionFlags(kubeConfigFlags)
 	matchVersionKubeConfigFlags.AddFlags(cmds.PersistentFlags())
-	factory := cmdutil.NewFactory(matchVersionKubeConfigFlags)
+	factory := util.NewFactory(matchVersionKubeConfigFlags)
 
 	cmds.Flags().AddGoFlagSet(flag.CommandLine)
 	flag.CommandLine.Parse([]string{})

cmd/ctl/main.go

@@ -22,17 +22,18 @@ import (
 	"os"
 
 	ctlcmd "github.com/jetstack/cert-manager/cmd/ctl/cmd"
-	"github.com/jetstack/cert-manager/pkg/util"
-	utilcmd "github.com/jetstack/cert-manager/pkg/util/cmd"
+	"github.com/jetstack/cert-manager/cmd/util"
 )
 
 func main() {
-	stopCh := utilcmd.SetupSignalHandler()
+	stopCh, exit := util.SetupExitHandler(util.AlwaysErrCode)
+	defer exit() // This function might call os.Exit, so defer last
+
 	ctx := util.ContextWithStopCh(context.Background(), stopCh)
 	cmd := ctlcmd.NewCertManagerCtlCommand(ctx, os.Stdin, os.Stdout, os.Stderr)
 
 	if err := cmd.Execute(); err != nil {
 		fmt.Fprintf(os.Stderr, "%s\n", err)
-		os.Exit(1)
+		util.SetExitCode(err)
 	}
 }

cmd/ctl/pkg/check/api/BUILD.bazel

@@ -6,6 +6,7 @@ go_library(
     importpath = "github.com/jetstack/cert-manager/cmd/ctl/pkg/check/api",
     visibility = ["//visibility:public"],
     deps = [
+        "//cmd/util:go_default_library",
         "//pkg/util/cmapichecker:go_default_library",
         "@com_github_spf13_cobra//:go_default_library",
         "@io_k8s_apimachinery//pkg/util/wait:go_default_library",

cmd/ctl/pkg/check/api/api.go

@@ -21,7 +21,7 @@ import (
 	"errors"
 	"fmt"
 	"log"
-	"os"
+	"runtime"
 	"time"
 
 	"github.com/spf13/cobra"
@@ -32,6 +32,7 @@ import (
 	"k8s.io/kubectl/pkg/util/i18n"
 	"k8s.io/kubectl/pkg/util/templates"
 
+	cmcmdutil "github.com/jetstack/cert-manager/cmd/util"
 	"github.com/jetstack/cert-manager/pkg/util/cmapichecker"
 )
 
@@ -150,7 +151,9 @@ func (o *Options) Run(ctx context.Context) {
 			log.Printf("Timed out after %s", o.Wait)
 		}
 
-		os.Exit(1)
+		cmcmdutil.SetExitCode(pollContext.Err())
+
+		runtime.Goexit() // Do soft exit (handle all defers, that should set correct exit code)
 	}
 
 	log.Printf("The cert-manager API is ready")

cmd/util/BUILD.bazel

@@ -3,11 +3,13 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library")
 go_library(
     name = "go_default_library",
     srcs = [
+        "context.go",
+        "exit.go",
         "signal.go",
         "signal_posix.go",
         "signal_windows.go",
     ],
-    importpath = "github.com/jetstack/cert-manager/pkg/util/cmd",
+    importpath = "github.com/jetstack/cert-manager/cmd/util",
     visibility = ["//visibility:public"],
 )
 

cmd/util/context.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2020 The cert-manager Authors.
+Copyright 2021 The cert-manager Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

cmd/util/exit.go

@@ -0,0 +1,29 @@
+/*
+Copyright 2021 The cert-manager Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"context"
+	"errors"
+)
+
+// SetExitCode sets the exit code to 1 if the error is not a context.Canceled error.
+func SetExitCode(err error) {
+	if (err != nil) && !errors.Is(err, context.Canceled) {
+		errorExitCodeChannel <- 1 // Indicate that there was an error
+	}
+}

cmd/util/signal.go

@@ -0,0 +1,78 @@
+/*
+Copyright 2021 The cert-manager Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package util
+
+import (
+	"os"
+	"os/signal"
+	"syscall"
+)
+
+var onlyOneSignalHandler = make(chan struct{})
+var errorExitCodeChannel = make(chan int, 1)
+
+// ExitBehavior controls how the program should be terminated
+// in response to a shutdown signal.
+type ExitBehavior int
+
+const (
+	// AlwaysErrCode indicates the exit code of the program should always be nonzero
+	// and should correspond to the numeric value of the signal that was received.
+	AlwaysErrCode ExitBehavior = iota
+
+	// GracefulShutdown treats a shutdown signal as a request to exit gracefully, terminating
+	// goroutines and returning an exit code of 0 if there are no errors during shutdown.
+	GracefulShutdown ExitBehavior = iota
+)
+
+// SetupExitHandler:
+// A stop channel is returned which is closed on receiving a shutdown signal (SIGTERM
+// or SIGINT). If a second signal is caught, the program is terminated directly with
+// exit code 130.
+// SetupExitHandler also returns an exit function, this exit function calls os.Exit(...)
+// if there is a exit code in the errorExitCodeChannel.
+// The errorExitCodeChannel receives exit codes when SetExitCode is called or when
+// a shutdown signal is received (only if exitBehavior is AlwaysErrCode).
+func SetupExitHandler(exitBehavior ExitBehavior) (<-chan struct{}, func()) {
+	close(onlyOneSignalHandler) // panics when called twice
+
+	stop := make(chan struct{})
+	c := make(chan os.Signal, 2)
+	signal.Notify(c, shutdownSignals...)
+	go func() {
+		// first signal. Close stop chan and pass exit code to exitCodeChannel.
+		exitCode := 128 + int((<-c).(syscall.Signal))
+		if exitBehavior == AlwaysErrCode {
+			errorExitCodeChannel <- exitCode
+		}
+		close(stop)
+		// second signal. Exit directly.
+		<-c
+		os.Exit(130)
+	}()
+
+	return stop, func() {
+		select {
+		case signal := <-errorExitCodeChannel:
+			os.Exit(signal)
+		default:
+			// Do not exit, there are no exit codes in the channel,
+			// so just continue and let the main function go out of
+			// scope instead.
+		}
+	}
+}

cmd/util/signal_posix.go

@@ -16,7 +16,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package cmd
+package util
 
 import (
 	"os"

cmd/util/signal_windows.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package cmd
+package util
 
 import (
 	"os"

cmd/webhook/BUILD.bazel

@@ -7,9 +7,9 @@ go_library(
     importpath = "github.com/jetstack/cert-manager/cmd/webhook",
     visibility = ["//visibility:private"],
     deps = [
+        "//cmd/util:go_default_library",
         "//cmd/webhook/app:go_default_library",
         "//pkg/logs:go_default_library",
-        "//pkg/util/cmd:go_default_library",
     ],
 )
 

cmd/webhook/app/BUILD.bazel

@@ -6,6 +6,7 @@ go_library(
     importpath = "github.com/jetstack/cert-manager/cmd/webhook/app",
     visibility = ["//visibility:public"],
     deps = [
+        "//cmd/util:go_default_library",
         "//cmd/webhook/app/options:go_default_library",
         "//pkg/logs:go_default_library",
         "//pkg/util:go_default_library",

cmd/webhook/app/webhook.go

@@ -25,6 +25,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/clientcmd"
 
+	cmdutil "github.com/jetstack/cert-manager/cmd/util"
 	"github.com/jetstack/cert-manager/cmd/webhook/app/options"
 	logf "github.com/jetstack/cert-manager/pkg/logs"
 	"github.com/jetstack/cert-manager/pkg/util"
@@ -102,7 +103,7 @@ func NewServerCommand(stopCh <-chan struct{}) *cobra.Command {
 		Use:   "webhook",
 		Short: fmt.Sprintf("Webhook component providing API validation, mutation and conversion functionality for cert-manager (%s) (%s)", util.AppVersion, util.AppGitCommit),
 		RunE: func(cmd *cobra.Command, args []string) error {
-			ctx := util.ContextWithStopCh(context.Background(), stopCh)
+			ctx := cmdutil.ContextWithStopCh(context.Background(), stopCh)
 			ctx = logf.NewContext(ctx, nil, "webhook")
 			log := logf.FromContext(ctx)
 

cmd/webhook/main.go

@@ -18,24 +18,25 @@ package main
 
 import (
 	"flag"
-	"os"
 
+	"github.com/jetstack/cert-manager/cmd/util"
 	"github.com/jetstack/cert-manager/cmd/webhook/app"
 	logf "github.com/jetstack/cert-manager/pkg/logs"
-	utilcmd "github.com/jetstack/cert-manager/pkg/util/cmd"
 )
 
 func main() {
+	stopCh, exit := util.SetupExitHandler(util.GracefulShutdown)
+	defer exit() // This function might call os.Exit, so defer last
+
 	logf.InitLogs(flag.CommandLine)
 	defer logf.FlushLogs()
 
-	stopCh := utilcmd.SetupSignalHandler()
 	cmd := app.NewServerCommand(stopCh)
 	cmd.Flags().AddGoFlagSet(flag.CommandLine)
 
 	flag.CommandLine.Parse([]string{})
 	if err := cmd.Execute(); err != nil {
 		logf.Log.Error(err, "error executing command")
-		os.Exit(1)
+		util.SetExitCode(err)
 	}
 }

pkg/acme/webhook/cmd/BUILD.bazel

@@ -6,10 +6,10 @@ go_library(
     importpath = "github.com/jetstack/cert-manager/pkg/acme/webhook/cmd",
     visibility = ["//visibility:public"],
     deps = [
+        "//cmd/util:go_default_library",
         "//pkg/acme/webhook:go_default_library",
         "//pkg/acme/webhook/cmd/server:go_default_library",
         "//pkg/logs:go_default_library",
-        "@io_k8s_apiserver//pkg/server:go_default_library",
         "@io_k8s_component_base//logs:go_default_library",
     ],
 )

pkg/acme/webhook/cmd/cmd.go

@@ -21,15 +21,18 @@ import (
 	"os"
 	"runtime"
 
-	genericapiserver "k8s.io/apiserver/pkg/server"
 	"k8s.io/component-base/logs"
 
+	"github.com/jetstack/cert-manager/cmd/util"
 	"github.com/jetstack/cert-manager/pkg/acme/webhook"
 	"github.com/jetstack/cert-manager/pkg/acme/webhook/cmd/server"
 	logf "github.com/jetstack/cert-manager/pkg/logs"
 )
 
 func RunWebhookServer(groupName string, hooks ...webhook.Solver) {
+	stopCh, exit := util.SetupExitHandler(util.GracefulShutdown)
+	defer exit() // This function might call os.Exit, so defer last
+
 	logs.InitLogs()
 	defer logs.FlushLogs()
 
@@ -37,12 +40,10 @@ func RunWebhookServer(groupName string, hooks ...webhook.Solver) {
 		runtime.GOMAXPROCS(runtime.NumCPU())
 	}
 
-	stopCh := genericapiserver.SetupSignalHandler()
-
 	cmd := server.NewCommandStartWebhookServer(os.Stdout, os.Stderr, stopCh, groupName, hooks...)
 	cmd.Flags().AddGoFlagSet(flag.CommandLine)
 	if err := cmd.Execute(); err != nil {
 		logf.Log.Error(err, "error executing command")
-		os.Exit(1)
+		util.SetExitCode(err)
 	}
 }

pkg/util/BUILD.bazel

@@ -3,7 +3,6 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 go_library(
     name = "go_default_library",
     srcs = [
-        "context.go",
         "useragent.go",
         "util.go",
         "version.go",
@@ -34,7 +33,6 @@ filegroup(
     srcs = [
         ":package-srcs",
         "//pkg/util/cmapichecker:all-srcs",
-        "//pkg/util/cmd:all-srcs",
         "//pkg/util/coverage:all-srcs",
         "//pkg/util/errors:all-srcs",
         "//pkg/util/feature:all-srcs",

pkg/util/cmd/signal.go

@@ -1,43 +0,0 @@
-/*
-Copyright 2020 The cert-manager Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package cmd
-
-import (
-	"os"
-	"os/signal"
-)
-
-var onlyOneSignalHandler = make(chan struct{})
-
-// SetupSignalHandler registered for SIGTERM and SIGINT. A stop channel is returned
-// which is closed on one of these signals. If a second signal is caught, the program
-// is terminated with exit code 1.
-func SetupSignalHandler() <-chan struct{} {
-	close(onlyOneSignalHandler) // panics when called twice
-
-	stop := make(chan struct{})
-	c := make(chan os.Signal, 2)
-	signal.Notify(c, shutdownSignals...)
-	go func() {
-		<-c
-		close(stop)
-		<-c
-		os.Exit(1) // second signal. Exit directly.
-	}()
-
-	return stop
-}