关于k8s-operator:k8soperator-系列-自动生成-webhook-证书并在过期时自动刷新

k8s webhook 要求必须应用 HTTPS，Operator webhook 的证书配置是个麻烦事。

家喻户晓 cert-manager 能够帮咱们生成证书，但证书生成后还须要批改 ValidatingWebhook 和 MutingWebhook。

上面要介绍的是另一种形式：github.com/open-policy-agent/cert-controller，最后是在 open-policy-agent/gatekeeper 中看到这种用法，感觉十分 Geek。

Operator 内置：

证书主动生成
过期刷新
webhook 配置中的 caBundle 主动注入

不依赖内部组件，真正做到一键部署 Operator，十分不便散发 Operator。

先启动 rotator：

setupFinished := make(chan struct{})
if !disableCertRotation {setupLog.Info("setting up cert rotation")
  err := rotator.AddRotator(mgr, &rotator.CertRotator{
    SecretKey: types.NamespacedName{Namespace: k8s.GetOperatorNamespace(),
      Name:      secretName,
    },
    CertDir:        certDir,
    CAName:         caName,
    CAOrganization: caOrganization,
    DNSName:        dnsName,
    IsReady:        setupFinished,
    Webhooks:       webhooks,
  })
  if err != nil {setupLog.Error(err, "unable to set up cert rotation")
    os.Exit(1)
  }
} else {close(setupFinished)
}

if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {setupLog.Error(err, "problem running manager")
  os.Exit(1)
}

setupFinished 之后再启动 Reconciler 和 Webhook server：

go func() {
  <-setupFinished

  err := (&controllers.EtcdReconciler{Client: mgr.GetClient(),
    Log:    zaplog.Sugar().Named("controllers").Named("Etcd"),
    Scheme: mgr.GetScheme(),}).SetupWithManager(mgr)
  if err != nil {setupLog.Error(err, "unable to create controller", "controller", "Etcd")
    os.Exit(1)
  }

  if certDir != "" {err = (&dbv1.Etcd{}).SetupWebhookWithManager(mgr)
    if err != nil {setupLog.Error(err, "unable to SetupWebhookWithManager")
      os.Exit(1)
    }
  }
}()

controllerManager 有个个性，如果曾经启动，则前面 Add 的 Runnable 间接启动：

// Add sets dependencies on i, and adds it to the list of Runnables to start.
func (cm *controllerManager) Add(r Runnable) error {
    // ...

    if shouldStart {
        // If already started, start the controller
        cm.startRunnable(r)
    }

    return nil
}

残缺例子：win5do/etcd-operator (github.com)

Webhook 应用的证书不要求肯定是 k8s 集群 CA 根证书颁发的证书，所以咱们能够应用自签证书。

生成自签 CA 根证书：

// CreateCACert creates the self-signed CA cert and private key that will
// be used to sign the server certificate
func (cr *CertRotator) CreateCACert(begin, end time.Time) (*KeyPairArtifacts, error) {
    // ...
    key, err := rsa.GenerateKey(rand.Reader, 2048)
    if err != nil {return nil, errors.Wrap(err, "generating key")
    }
    der, err := x509.CreateCertificate(rand.Reader, templ, templ, key.Public(), key)
    if err != nil {return nil, errors.Wrap(err, "creating certificate")
    }
    certPEM, keyPEM, err := pemEncode(der, key)
    if err != nil {return nil, errors.Wrap(err, "encoding PEM")
    }
    cert, err := x509.ParseCertificate(der)
    if err != nil {return nil, errors.Wrap(err, "parsing certificate")
    }

    return &KeyPairArtifacts{Cert: cert, Key: key, CertPEM: certPEM, KeyPEM: keyPEM}, nil
}

依据 service-name.namespace.svc 生成服务器 tls.key 和 tls.crt：

// CreateCertPEM takes the results of CreateCACert and uses it to create the
// PEM-encoded public certificate and private key, respectively
func (cr *CertRotator) CreateCertPEM(ca *KeyPairArtifacts, begin, end time.Time) ([]byte, []byte, error) {
    // ...
    key, err := rsa.GenerateKey(rand.Reader, 2048)
    if err != nil {return nil, nil, errors.Wrap(err, "generating key")
    }
    der, err := x509.CreateCertificate(rand.Reader, templ, ca.Cert, key.Public(), ca.Key)
    if err != nil {return nil, nil, errors.Wrap(err, "creating certificate")
    }
    certPEM, keyPEM, err := pemEncode(der, key)
    if err != nil {return nil, nil, errors.Wrap(err, "encoding PEM")
    }
    return certPEM, keyPEM, nil
}

写入到 secret 中，并期待 mount：

// ensureCertsMounted ensure the cert files exist.
func (cr *CertRotator) ensureCertsMounted() {checkFn := func() (bool, error) {
        certFile := cr.CertDir + "/" + certName
        _, err := os.Stat(certFile)
        if err == nil {return true, nil}
        return false, nil
    }
    if err := wait.ExponentialBackoff(wait.Backoff{
        Duration: 1 * time.Second,
        Factor:   2,
        Jitter:   1,
        Steps:    10,
    }, checkFn); err != nil {crLog.Error(err, "max retries for checking certs existence")
        close(cr.certsNotMounted)
        return
    }
    crLog.Info(fmt.Sprintf("certs are ready in %s", cr.CertDir))
    close(cr.certsMounted)
}

启动一个 Reconceile 监听 secret，如果有更新则将 caBundle 注入到 webhook config 中：

func injectCertToWebhook(wh *unstructured.Unstructured, certPem []byte) error {webhooks, found, err := unstructured.NestedSlice(wh.Object, "webhooks")
    if err != nil {return err}
    if !found {return errors.New("`webhooks` field not found in ValidatingWebhookConfiguration")
    }
    for i, h := range webhooks {hook, ok := h.(map[string]interface{})
        if !ok {return errors.Errorf("webhook %d is not well-formed", i)
        }
        if err := unstructured.SetNestedField(hook, base64.StdEncoding.EncodeToString(certPem), "clientConfig", "caBundle"); err != nil {return err}
        webhooks[i] = hook
    }
    if err := unstructured.SetNestedSlice(wh.Object, webhooks, "webhooks"); err != nil {return err}
    return nil
}

应用

原理