diff --git a/apply.sh b/apply.sh index dccffeb..26663a4 100755 --- a/apply.sh +++ b/apply.sh @@ -8,9 +8,6 @@ helm repo add nfs-subdir-external-provisioner https://kubernetes-sigs.github.io/ helm install nfs-subdir-external-provisioner nfs-subdir-external-provisioner/nfs-subdir-external-provisioner \ --set nfs.server=192.168.1.4 --set nfs.path=/k8s -# namespaces -kubectl create namespace longhorn - # taints kubectl taint nodes server-1 node=special:NoSchedule diff --git a/nginx-values.yaml b/archive/nginx-values.yaml similarity index 76% rename from nginx-values.yaml rename to archive/nginx-values.yaml index 6dceea4..c1879e6 100644 --- a/nginx-values.yaml +++ b/archive/nginx-values.yaml @@ -1,4 +1,4 @@ -kind: Deployment +kind: DaemonSet hostNetwork: true metrics: enabled: true @@ -12,8 +12,9 @@ stats: podAnnotations: fluentbit.io/parser: k8s-nginx-ingress -# nodeSelector: -# gateway: "true" +nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: - key: node operator: Equal @@ -23,19 +24,12 @@ tolerations: operator: Equal value: "" effect: NoSchedule -#replicaCount: 1 service: type: NodePort ports: http: 80 https: 443 - # nodePorts: - # http: 80 - # https: 443 - -# config: -# "use-forwarded-headers": "true" defaultBackend: podAnnotations: diff --git a/cluster.yaml b/cluster.yaml index 42e9c71..c89e004 100644 --- a/cluster.yaml +++ b/cluster.yaml @@ -12,10 +12,10 @@ etcd: certFile: "/etc/etcd/etcd.pem" keyFile: "/etc/etcd/etcd-key.pem" networking: - serviceSubnet: "10.96.0.0/12" + serviceSubnet: "10.96.0.0/16" podSubnet: "10.244.0.0/16" dnsDomain: "cluster.local" -controlPlaneEndpoint: "192.168.1.2:6443" +controlPlaneEndpoint: "192.168.1.2:6443" # updated to this vip certificatesDir: "/etc/kubernetes/pki" imageRepository: "k8s.gcr.io" clusterName: "home-cluster" diff --git a/ha-config/haproxy b/ha-config/haproxy new file mode 100644 index 0000000..88c2e08 --- /dev/null +++ b/ha-config/haproxy @@ -0,0 +1,40 @@ +# HAProxy Configure /etc/haproxy/haproxy.cfg +global + log 127.0.0.1 local2 + chroot /var/lib/haproxy + pidfile /var/run/haproxy.pid + maxconn 4000 + user haproxy + group haproxy + daemon + # turn on stats unix socket + stats socket /var/lib/haproxy/stats +#--------------------------------------------------------------------- +# common defaults that all the 'listen' and 'backend' sections will +# use if not designated in their block +#--------------------------------------------------------------------- +defaults + log global + option httplog + option dontlognull + timeout connect 5000 + timeout client 5000 + timeout server 5000 +#--------------------------------------------------------------------- +# main frontend which proxys to the backends +#--------------------------------------------------------------------- +frontend kube-apiserver + bind *:6443 + mode tcp + option tcplog + default_backend kube-apiserver +#--------------------------------------------------------------------- +# round robin balancing between the various backends +#--------------------------------------------------------------------- +backend kube-apiserver + mode tcp + option tcplog + balance roundrobin + default-server inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 250 maxqueue 256 weight 100 + server server-3 192.168.1.2:6443 check + server ssd-3 192.168.1.22:6443 check diff --git a/kube-vip/daemonset.yaml b/kube-vip/daemonset.yaml new file mode 100644 index 0000000..dc436c3 --- /dev/null +++ b/kube-vip/daemonset.yaml @@ -0,0 +1,82 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app.kubernetes.io/name: kube-vip-ds + app.kubernetes.io/version: v0.4.4 + name: kube-vip-ds + namespace: kube-system +spec: + selector: + matchLabels: + app.kubernetes.io/name: kube-vip-ds + template: + metadata: + labels: + app.kubernetes.io/name: kube-vip-ds + app.kubernetes.io/version: v0.4.4 + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_interface + value: ens160 + - name: vip_cidr + value: "32" + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: "false" + - name: svc_enable + value: "true" + - name: vip_leaderelection + value: "true" + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" + - name : lb_enable + value: "true" + - name: address + value: 192.168.1.10 + image: ghcr.io/kube-vip/kube-vip:v0.4.4 + imagePullPolicy: Always + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + hostNetwork: true + serviceAccountName: kube-vip + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + updateStrategy: {} +status: + currentNumberScheduled: 0 + desiredNumberScheduled: 0 + numberMisscheduled: 0 + numberReady: 0 diff --git a/kube-vip/rbac.yaml b/kube-vip/rbac.yaml new file mode 100644 index 0000000..f35d79a --- /dev/null +++ b/kube-vip/rbac.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-vip + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + name: system:kube-vip-role +rules: + - apiGroups: [""] + resources: ["services", "services/status", "nodes"] + verbs: ["list","get","watch", "update"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["list", "get", "watch", "update", "create"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: system:kube-vip-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-vip-role +subjects: +- kind: ServiceAccount + name: kube-vip + namespace: kube-system diff --git a/recovery.sh b/recovery.sh index 31bd4c2..1264be8 100644 --- a/recovery.sh +++ b/recovery.sh @@ -24,6 +24,9 @@ sudo docker run --rm \ 'k8s.gcr.io/etcd:3.2.24' \ /bin/sh -c "etcdctl snapshot restore '/backup/snapshot-2022-04-26-05_00_11_UTC.db'; echo 'done'; ls /default.etcd/; mv /default.etcd/member /var/lib/etcd" +# install nfs-common + +sudo apt install nfs-common ## update docker cgroup @@ -32,12 +35,28 @@ vim /lib/systemd/system/docker.service # update ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock --exec-opt native.cgroupdriver=systemd -systemctl daemon-reload -systemctl restart docker +sudo systemctl daemon-reload +sudo systemctl restart docker # check cgroup docker system info --format '{{.CgroupDriver}}' +# if it fails at + +[ERROR CRI]: container runtime is not running: output: time="2021-05-28T14:11:57+08:00" level=fatal msg="getting status of runtime failed: rpc error: code = Unimplemented desc = unknown service runtime.v1alpha2.RuntimeService" + +# then + +vim /etc/containerd/config.toml + +# comment out + +disabled_plugins = ["cri"] + +# restart containerd + +systemctl restart containerd + # re init cluster sudo kubeadm init --ignore-preflight-errors=DirAvailable--var-lib-etcd --control-plane-endpoint 192.168.1.2 \ --pod-network-cidr=10.244.0.0/16 --v 5 @@ -76,7 +95,7 @@ kubeadm join 192.168.1.2:6443 --token xxx --discovery-token-ca-cert-hash sha256: sudo swapoff -a sudo rm /swap.img -vim /etc/fstab # and remove swap +sudo vim /etc/fstab # and remove swap # check sudo swapon --show @@ -98,4 +117,4 @@ sudo kubeadm upgrade diff --config cluster.yaml --v=5 sudo kubeadm upgrade plan --config cluster.yaml --v=5 # apply -sudo kubeadm upgrade apply --config cluster.yaml --ignore-preflight-errors all --force --v=5 +sudo kubeadm upgrade apply --config cluster.yaml --ignore-preflight-errors all --v=5