CRI-O기반의 k8s 설치

사전사항

OS환경설정

$> swapoff -a

$> cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
br_netfilter
EOF

$> cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF

$> sudo sysctl --system

crio / kubernetees 패키지 리포지터리 구성

$> cat /etc/yum.repos.d/libcontainers.repo
[devel_kubic_libcontainers_stable]
name=Stable Releases of Upstream github.com/containers packages (CentOS_8)
type=rpm-md
baseurl=https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/CentOS_8/
gpgcheck=1
gpgkey=https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/CentOS_8/repodata/repomd.xml.key
enabled=1

$> cat /etc/yum.repos.d/cri-o-1.23.repo
[devel_kubic_libcontainers_stable_cri-o_1.23]
name=devel:kubic:libcontainers:stable:cri-o:1.23 (CentOS_8)
type=rpm-md
baseurl=https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/1.23/CentOS_8/
gpgcheck=1
gpgkey=https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/1.23/CentOS_8/repodata/repomd.xml.key
enabled=1

$> cat /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-$basearch
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg

패키지 설치

$> yum install kubelet kubeadm kubectl libcgroup cri-o cri-tools -y
$> systemctl enable crio --now
$> systemctl enable kubelet

클러스터 생성 (control palin 1번에서만 수행)

kubeadm 클러스터 생성

$> kubeadm init --control-plane-endpoint 172.21.107.238:6443 --pod-network-cidr 10.250.0.0/16 --ignore-preflight-errors=all --upload-certs

## 결과값중에 control / worker 노드별 join 명령이 다르기 때문에 별도로 복사해놓어야 함

##Control node용

$> kubeadm join 172.21.107.238:6443 --token abcd \
       --discovery-token-ca-cert-hash sha256:yyy \
       --control-plane --certificate-key zzz

## Worker Node용

$> kubeadm join 172.21.107.238:6443 --token abcd \
       --discovery-token-ca-cert-hash sha256:yyyy \
 --ignore-preflight-errors=all

인증서 정보 복사

$> mkdir -p $HOME/.kube
$> /bin/cp /etc/kubernetes/admin.conf $HOME/.kube/config
$> chown $(id -u):$(id -g) $HOME/.kube/config
$> export KUBECONFIG=/etc/kubernetes/admin.conf

CNI 설치(Calico)

$> curl https://projectcalico.docs.tigera.io/manifests/calico.yaml -O
$> kubectl apply -f calico.yaml

클러스터 연동

타 Control plain 연동 (Control Plain 한대씩 순차 작업 수행)

$> kubeadm join 172.21.107.238:6443 --token abcd \
       --discovery-token-ca-cert-hash sha256:yyy \
       --control-plane --certificate-key zzz

노드 연동 확인 (Control plain에서 수행)

$> kubectl get no
NAME                         STATUS   ROLES                  AGE    VERSION
k8stesttx-k8s-master-dev01   Ready    control-plane,master   3h6m   v1.23.5
k8stesttx-k8s-master-dev02   Ready    control-plane,master   3h6m   v1.23.5
k8stesttx-k8s-master-dev03   Ready    control-plane,master   3h6m   v1.23.5

Worker Node 연동

$> kubeadm join 172.21.107.238:6443 --token abcd \
       --discovery-token-ca-cert-hash sha256:yyyy \
 --ignore-preflight-errors=all

노드 연동 확인 (Control plain에서 수행)

$> kubectl get no
NAME                         STATUS   ROLES                  AGE    VERSION
...
k8stesttx-k8s-worker-dev01   Ready    <none>                 40m    v1.23.5
k8stesttx-k8s-worker-dev02   Ready    <none>                 40m    v1.23.5
k8stesttx-k8s-worker-dev03   Ready    <none>                 40m    v1.23.5

k8s 인증서 10년으로 연장

인증서 연장 스크립트

#!/usr/bin/env bash

set -o errexit
set -o pipefail
# set -o xtrace

# set output color
NC='\033[0m'
RED='\033[31m'
GREEN='\033[32m'
YELLOW='\033[33m'
BLUE='\033[34m'

log::err() {
 printf "[$(date +'%Y-%m-%dT%H:%M:%S.%2N%z')][${RED}ERROR${NC}] %b\n" "$@"
}

log::info() {
 printf "[$(date +'%Y-%m-%dT%H:%M:%S.%2N%z')][INFO] %b\n" "$@"
}

log::warning() {
 printf "[$(date +'%Y-%m-%dT%H:%M:%S.%2N%z')][${YELLOW}WARNING${NC}] \033[0m%b\n" "$@"
}

check_file() {
 if [[ ! -r ${1} ]]; then
    log::err "can not find ${1}"
   exit 1
 fi
}

# get x509v3 subject alternative name from the old certificate
cert::get_subject_alt_name() {
 local cert=${1}.crt
 local alt_name

  check_file "${cert}"
 alt_name=$(openssl x509 -text -noout -in "${cert}" | grep -A1 'Alternative' | tail -n1 | sed 's/[[:space:]]*Address//g')
 printf "%s\n" "${alt_name}"
}

# get subject from the old certificate
cert::get_subj() {
 local cert=${1}.crt
 local subj

  check_file "${cert}"
 subj=$(openssl x509 -text -noout -in "${cert}" | grep "Subject:" | sed 's/Subject:/\//g;s/\,/\//;s/[[:space:]]//g')
 printf "%s\n" "${subj}"
}

cert::backup_file() {
 local file=${1}
 if [[ ! -e ${file}.old-$(date +%Y%m%d) ]]; then
    cp -rp "${file}" "${file}.old-$(date +%Y%m%d)"
    log::info "backup ${file} to ${file}.old-$(date +%Y%m%d)"
 else
    log::warning "does not backup, ${file}.old-$(date +%Y%m%d) already exists"
 fi
}

# check certificate expiration
cert::check_cert_expiration() {
 local cert=${1}.crt
 local cert_expires

 cert_expires=$(openssl x509 -text -noout -in "${cert}" | awk -F ": " '/Not After/{print$2}')
 printf "%s\n" "${cert_expires}"
}

# check kubeconfig expiration
cert::check_kubeconfig_expiration() {
 local config=${1}.conf
 local cert
 local cert_expires

 cert=$(grep "client-certificate-data" "${config}" | awk '{print$2}' | base64 -d)
 cert_expires=$(openssl x509 -text -noout -in <(printf "%s" "${cert}") | awk -F ": " '/Not After/{print$2}')
 printf "%s\n" "${cert_expires}"
}

# check etcd certificates expiration
cert::check_etcd_certs_expiration() {
 local cert
 local certs

 certs=(
   "${ETCD_CERT_CA}"
   "${ETCD_CERT_SERVER}"
   "${ETCD_CERT_PEER}"
   "${ETCD_CERT_HEALTHCHECK_CLIENT}"
   "${ETCD_CERT_APISERVER_ETCD_CLIENT}"
 )

 for cert in "${certs[@]}"; do
   if [[ ! -r ${cert} ]]; then
     printf "%-50s%-30s\n" "${cert}.crt" "$(cert::check_cert_expiration "${cert}")"
   fi
 done
}

# check master certificates expiration
cert::check_master_certs_expiration() {
 local certs
 local kubeconfs
 local cert
 local conf

 certs=(
   "${CERT_CA}"
   "${CERT_APISERVER}"
   "${CERT_APISERVER_KUBELET_CLIENT}"
   "${FRONT_PROXY_CA}"
   "${FRONT_PROXY_CLIENT}"
 )

 kubeconfs=(
   "${CONF_CONTROLLER_MANAGER}"
   "${CONF_SCHEDULER}"
   "${CONF_ADMIN}"
 )

 printf "%-50s%-30s\n" "CERTIFICATE" "EXPIRES"

 for conf in "${kubeconfs[@]}"; do
   if [[ ! -r ${conf} ]]; then
     printf "%-50s%-30s\n" "${conf}.config" "$(cert::check_kubeconfig_expiration "${conf}")"
   fi
 done

 for cert in "${certs[@]}"; do
   if [[ ! -r ${cert} ]]; then
     printf "%-50s%-30s\n" "${cert}.crt" "$(cert::check_cert_expiration "${cert}")"
   fi
 done
}

# check all certificates expiration
cert::check_all_expiration() {
  cert::check_master_certs_expiration
  cert::check_etcd_certs_expiration
}

# generate certificate whit client, server or peer
# Args:
#   $1 (the name of certificate)
#   $2 (the type of certificate, must be one of client, server, peer)
#   $3 (the subject of certificates)
#   $4 (the validity of certificates) (days)
#   $5 (the name of ca)
#   $6 (the x509v3 subject alternative name of certificate when the type of certificate is server or peer)
cert::gen_cert() {
 local cert_name=${1}
 local cert_type=${2}
 local subj=${3}
 local cert_days=${4}
 local ca_name=${5}
 local alt_name=${6}
 local ca_cert=${ca_name}.crt
 local ca_key=${ca_name}.key
 local cert=${cert_name}.crt
 local key=${cert_name}.key
 local csr=${cert_name}.csr
 local common_csr_conf='distinguished_name = dn\n[dn]\n[v3_ext]\nkeyUsage = critical, digitalSignature, keyEncipherment\n'

 for file in "${ca_cert}" "${ca_key}" "${cert}" "${key}"; do
    check_file "${file}"
 done

 case "${cert_type}" in
  client)
   csr_conf=$(printf "%bextendedKeyUsage = clientAuth\n" "${common_csr_conf}")
    ;;
  server)
   csr_conf=$(printf "%bextendedKeyUsage = serverAuth\nsubjectAltName = %b\n" "${common_csr_conf}" "${alt_name}")
    ;;
  peer)
   csr_conf=$(printf "%bextendedKeyUsage = serverAuth, clientAuth\nsubjectAltName = %b\n" "${common_csr_conf}" "${alt_name}")
    ;;
  *)
    log::err "unknow, unsupported certs type: ${YELLOW}${cert_type}${NC}, supported type: client, server, peer"
   exit 1
    ;;
 esac

 # gen csr
 openssl req -new -key "${key}" -subj "${subj}" -reqexts v3_ext \
   -config <(printf "%b" "${csr_conf}") \
   -out "${csr}" >/dev/null 2>&1
 # gen cert
 openssl x509 -in "${csr}" -req -CA "${ca_cert}" -CAkey "${ca_key}" -CAcreateserial -extensions v3_ext \
   -extfile <(printf "%b" "${csr_conf}") \
   -days "${cert_days}" -out "${cert}" >/dev/null 2>&1

  rm -f "${csr}"
}

cert::update_kubeconf() {
 local cert_name=${1}
 local kubeconf_file=${cert_name}.conf
 local cert=${cert_name}.crt
 local key=${cert_name}.key
 local subj
 local cert_base64

  check_file "${kubeconf_file}"
 # get the key from the old kubeconf
 grep "client-key-data" "${kubeconf_file}" | awk '{print$2}' | base64 -d >"${key}"
 # get the old certificate from the old kubeconf
 grep "client-certificate-data" "${kubeconf_file}" | awk '{print$2}' | base64 -d >"${cert}"
 # get subject from the old certificate
 subj=$(cert::get_subj "${cert_name}")
  cert::gen_cert "${cert_name}" "client" "${subj}" "${CERT_DAYS}" "${CERT_CA}"
 # get certificate base64 code
 cert_base64=$(base64 -w 0 "${cert}")

 # set certificate base64 code to kubeconf
 sed -i 's/client-certificate-data:.*/client-certificate-data: '"${cert_base64}"'/g' "${kubeconf_file}"

  rm -f "${cert}"
  rm -f "${key}"
}

cert::update_etcd_cert() {
 local subj
 local subject_alt_name
 local cert

 # generate etcd server,peer certificate
 # /etc/kubernetes/pki/etcd/server
 # /etc/kubernetes/pki/etcd/peer
 for cert in ${ETCD_CERT_SERVER} ${ETCD_CERT_PEER}; do
   subj=$(cert::get_subj "${cert}")
   subject_alt_name=$(cert::get_subject_alt_name "${cert}")
    cert::gen_cert "${cert}" "peer" "${subj}" "${CERT_DAYS}" "${ETCD_CERT_CA}" "${subject_alt_name}"
    log::info "${GREEN}updated ${BLUE}${cert}.conf${NC}"
 done

 # generate etcd healthcheck-client,apiserver-etcd-client certificate
 # /etc/kubernetes/pki/etcd/healthcheck-client
 # /etc/kubernetes/pki/apiserver-etcd-client
 for cert in ${ETCD_CERT_HEALTHCHECK_CLIENT} ${ETCD_CERT_APISERVER_ETCD_CLIENT}; do
   subj=$(cert::get_subj "${cert}")
    cert::gen_cert "${cert}" "client" "${subj}" "${CERT_DAYS}" "${ETCD_CERT_CA}"
    log::info "${GREEN}updated ${BLUE}${cert}.conf${NC}"
 done

 # restart etcd
 docker ps | awk '/k8s_etcd/{print$1}' | xargs -r -I '{}' docker restart {} >/dev/null 2>&1 || true
  log::info "restarted etcd"
}

cert::update_master_cert() {
 local subj
 local subject_alt_name
 local conf

 # generate apiserver server certificate
 # /etc/kubernetes/pki/apiserver
 subj=$(cert::get_subj "${CERT_APISERVER}")
 subject_alt_name=$(cert::get_subject_alt_name "${CERT_APISERVER}")
  cert::gen_cert "${CERT_APISERVER}" "server" "${subj}" "${CERT_DAYS}" "${CERT_CA}" "${subject_alt_name}"
  log::info "${GREEN}updated ${BLUE}${CERT_APISERVER}.crt${NC}"

 # generate apiserver-kubelet-client certificate
 # /etc/kubernetes/pki/apiserver-kubelet-client
 subj=$(cert::get_subj "${CERT_APISERVER_KUBELET_CLIENT}")
  cert::gen_cert "${CERT_APISERVER_KUBELET_CLIENT}" "client" "${subj}" "${CERT_DAYS}" "${CERT_CA}"
  log::info "${GREEN}updated ${BLUE}${CERT_APISERVER_KUBELET_CLIENT}.crt${NC}"

 # generate kubeconf for controller-manager,scheduler and kubelet
 # /etc/kubernetes/controller-manager,scheduler,admin,kubelet.conf
 for conf in ${CONF_CONTROLLER_MANAGER} ${CONF_SCHEDULER} ${CONF_ADMIN} ${CONF_KUBELET}; do
   if [[ ${conf##*/} == "kubelet" ]]; then
     # https://github.com/kubernetes/kubeadm/issues/1753
     set +e
      grep kubelet-client-current.pem /etc/kubernetes/kubelet.conf >/dev/null 2>&1
     kubelet_cert_auto_update=$?
     set -e
     if [[ "$kubelet_cert_auto_update" == "0" ]]; then
        log::info "does not need to update kubelet.conf"
       continue
     fi
   fi

   # update kubeconf
   cert::update_kubeconf "${conf}"
    log::info "${GREEN}updated ${BLUE}${conf}.conf${NC}"

   # copy admin.conf to ${HOME}/.kube/config
   if [[ ${conf##*/} == "admin" ]]; then
      mkdir -p "${HOME}/.kube"
     local config=${HOME}/.kube/config
     local config_backup
     config_backup=${HOME}/.kube/config.old-$(date +%Y%m%d)
     if [[ -f ${config} ]] && [[ ! -f ${config_backup} ]]; then
        cp -fp "${config}" "${config_backup}"
        log::info "backup ${config} to ${config_backup}"
     fi
      cp -fp "${conf}.conf" "${HOME}/.kube/config"
      log::info "copy the admin.conf to ${HOME}/.kube/config"
   fi
 done

 # generate front-proxy-client certificate
 # /etc/kubernetes/pki/front-proxy-client
 subj=$(cert::get_subj "${FRONT_PROXY_CLIENT}")
  cert::gen_cert "${FRONT_PROXY_CLIENT}" "client" "${subj}" "${CERT_DAYS}" "${FRONT_PROXY_CA}"
  log::info "${GREEN}updated ${BLUE}${FRONT_PROXY_CLIENT}.crt${NC}"

 # restart apiserver, controller-manager, scheduler and kubelet
 for item in "apiserver" "controller-manager" "scheduler"; do
    docker ps | awk '/k8s_kube-'${item}'/{print$1}' | xargs -r -I '{}' docker restart {} >/dev/null 2>&1 || true
    log::info "restarted ${item}"
 done
  systemctl restart kubelet || true
  log::info "restarted kubelet"
}

main() {
 local node_type=$1

 CERT_DAYS=3650

 KUBE_PATH=/etc/kubernetes
 PKI_PATH=${KUBE_PATH}/pki

 # master certificates path
 # apiserver
 CERT_CA=${PKI_PATH}/ca
 CERT_APISERVER=${PKI_PATH}/apiserver
 CERT_APISERVER_KUBELET_CLIENT=${PKI_PATH}/apiserver-kubelet-client
 CONF_CONTROLLER_MANAGER=${KUBE_PATH}/controller-manager
 CONF_SCHEDULER=${KUBE_PATH}/scheduler
 CONF_ADMIN=${KUBE_PATH}/admin
 CONF_KUBELET=${KUBE_PATH}/kubelet
 # front-proxy
 FRONT_PROXY_CA=${PKI_PATH}/front-proxy-ca
 FRONT_PROXY_CLIENT=${PKI_PATH}/front-proxy-client

 # etcd certificates path
 ETCD_CERT_CA=${PKI_PATH}/etcd/ca
 ETCD_CERT_SERVER=${PKI_PATH}/etcd/server
 ETCD_CERT_PEER=${PKI_PATH}/etcd/peer
 ETCD_CERT_HEALTHCHECK_CLIENT=${PKI_PATH}/etcd/healthcheck-client
 ETCD_CERT_APISERVER_ETCD_CLIENT=${PKI_PATH}/apiserver-etcd-client

 case ${node_type} in
 # etcd)
 # # update etcd certificates
 #   cert::update_etcd_cert
 # ;;
 master)
   # check certificates expiration
   cert::check_master_certs_expiration
   # backup $KUBE_PATH to $KUBE_PATH.old-$(date +%Y%m%d)
   cert::backup_file "${KUBE_PATH}"
   # update master certificates and kubeconf
   log::info "${GREEN}updating...${NC}"
    cert::update_master_cert
    log::info "${GREEN}done!!!${NC}"
   # check certificates expiration after certificates updated
   cert::check_master_certs_expiration
    ;;
  all)
   # check certificates expiration
   cert::check_all_expiration
   # backup $KUBE_PATH to $KUBE_PATH.old-$(date +%Y%m%d)
   cert::backup_file "${KUBE_PATH}"
   # update etcd certificates
   log::info "${GREEN}updating...${NC}"
    cert::update_etcd_cert
   # update master certificates and kubeconf
   cert::update_master_cert
    log::info "${GREEN}done!!!${NC}"
   # check certificates expiration after certificates updated
   cert::check_all_expiration
    ;;
  check)
   # check certificates expiration
   cert::check_all_expiration
    ;;
  *)
    log::err "unknown, unsupported cert type: ${node_type}, supported type: \"all\", \"master\""
   printf "Documentation: https://github.com/yuyicai/update-kube-cert
  example:
    '\033[32m./update-kubeadm-cert.sh all\033[0m' update all etcd certificates, master certificates and kubeconf
      /etc/kubernetes
      ├── admin.conf
      ├── controller-manager.conf
      ├── scheduler.conf
      ├── kubelet.conf
      └── pki
          ├── apiserver.crt
          ├── apiserver-etcd-client.crt
          ├── apiserver-kubelet-client.crt
          ├── front-proxy-client.crt
          └── etcd
              ├── healthcheck-client.crt
              ├── peer.crt
              └── server.crt

    '\033[32m./update-kubeadm-cert.sh master\033[0m' update only master certificates and kubeconf
      /etc/kubernetes
      ├── admin.conf
      ├── controller-manager.conf
      ├── scheduler.conf
      ├── kubelet.conf
      └── pki
          ├── apiserver.crt
          ├── apiserver-kubelet-client.crt
          └── front-proxy-client.crt
"
   exit 1
    ;;
 esac
}

main "$@"

업데이트 전 인증서 정보 확인

$> kubeadm certs check-expiration
[check-expiration] Reading configuration from the cluster...
[check-expiration] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'

CERTIFICATE                EXPIRES                  RESIDUAL TIME   CERTIFICATE AUTHORITY   EXTERNALLY MANAGED
admin.conf                 Apr 17, 2023 06:09 UTC   364d            ca                      no
apiserver                  Apr 17, 2023 06:09 UTC   364d            ca                      no
apiserver-etcd-client      Apr 17, 2023 06:09 UTC   364d            etcd-ca                 no
apiserver-kubelet-client   Apr 17, 2023 06:09 UTC   364d            ca                      no
controller-manager.conf    Apr 17, 2023 06:09 UTC   364d            ca                      no
etcd-healthcheck-client    Apr 17, 2023 06:09 UTC   364d            etcd-ca                 no
etcd-peer                  Apr 17, 2023 06:09 UTC   364d            etcd-ca                 no
etcd-server                Apr 17, 2023 06:09 UTC   364d            etcd-ca                 no
front-proxy-client         Apr 17, 2023 06:09 UTC   364d            front-proxy-ca          no
scheduler.conf             Apr 17, 2023 06:09 UTC   364d            ca                      no

CERTIFICATE AUTHORITY   EXPIRES                  RESIDUAL TIME   EXTERNALLY MANAGED
ca                      Apr 17, 2032 04:17 UTC   9y              no
etcd-ca                 Apr 17, 2032 04:17 UTC   9y              no
front-proxy-ca          Apr 17, 2032 04:17 UTC   9y              no

인증서 업데이트 (Control Plain 1대씩 순차 작업 수행, 서버단위로 30초 가량 대기 필요)
```
$> chmod +x cert_update.sh
$> ./cert_update.sh
...
```

인증서 갱신정보 확인

$> kubeadm certs check-expiration
[check-expiration] Reading configuration from the cluster...
[check-expiration] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'

CERTIFICATE                EXPIRES                  RESIDUAL TIME   CERTIFICATE AUTHORITY   EXTERNALLY MANAGED
admin.conf                 Apr 17, 2032 06:09 UTC   9y              ca                      no
apiserver                  Apr 17, 2032 06:09 UTC   9y              ca                      no
apiserver-etcd-client      Apr 17, 2032 06:09 UTC   9y              etcd-ca                 no
apiserver-kubelet-client   Apr 17, 2032 06:09 UTC   9y              ca                      no
controller-manager.conf    Apr 17, 2032 06:09 UTC   9y              ca                      no
etcd-healthcheck-client    Apr 17, 2032 06:09 UTC   9y              etcd-ca                 no
etcd-peer                  Apr 17, 2032 06:09 UTC   9y              etcd-ca                 no
etcd-server                Apr 17, 2032 06:09 UTC   9y              etcd-ca                 no
front-proxy-client         Apr 17, 2032 06:09 UTC   9y              front-proxy-ca          no
scheduler.conf             Apr 17, 2032 06:09 UTC   9y              ca                      no

CERTIFICATE AUTHORITY   EXPIRES                  RESIDUAL TIME   EXTERNALLY MANAGED
ca                      Apr 17, 2032 04:17 UTC   9y              no
etcd-ca                 Apr 17, 2032 04:17 UTC   9y              no
front-proxy-ca          Apr 17, 2032 04:17 UTC   9y              no

Reference

CRI-O기반의 k8s 설치

docker기반의 nexus설치

onlyoffice를 컨테이너 기반에서 설치하기

pip로 docker-compose 설치 오류

QEMU tcp원격접속 허용하기

VMware GuestOS에서 마우스 5버튼 작동안될 경우 조치방법

rancher 패스워드 초기화 방법

CRI-O기반의 k8s 설치

사전사항

클러스터 생성 (control palin 1번에서만 수행)

클러스터 연동

k8s 인증서 10년으로 연장

No Comments