k8s webhook 要求必须应用 HTTPS，Operator webhook 的证书配置是个麻烦事。

家喻户晓 cert-manager 能够帮咱们生成证书，但证书生成后还须要批改 ValidatingWebhook 和 MutingWebhook。

上面要介绍的是另一种形式：github.com/open-policy-agent/cert-controller，最后是在 open-policy-agent/gatekeeper 中看到这种用法，感觉十分 Geek。

Operator 内置：

  • 证书主动生成
  • 过期刷新
  • webhook 配置中的 caBundle 主动注入

不依赖内部组件，真正做到一键部署 Operator，十分不便散发 Operator。

应用

先启动 rotator：

setupFinished := make(chan struct{})if !disableCertRotation {  setupLog.Info("setting up cert rotation")  err := rotator.AddRotator(mgr, &rotator.CertRotator{    SecretKey: types.NamespacedName{      Namespace: k8s.GetOperatorNamespace(),      Name:      secretName,    },    CertDir:        certDir,    CAName:         caName,    CAOrganization: caOrganization,    DNSName:        dnsName,    IsReady:        setupFinished,    Webhooks:       webhooks,  })  if err != nil {    setupLog.Error(err, "unable to set up cert rotation")    os.Exit(1)  }} else {  close(setupFinished)}if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {  setupLog.Error(err, "problem running manager")  os.Exit(1)}

setupFinished 之后再启动 Reconciler 和 Webhook server：

go func() {  <-setupFinished  err := (&controllers.EtcdReconciler{    Client: mgr.GetClient(),    Log:    zaplog.Sugar().Named("controllers").Named("Etcd"),    Scheme: mgr.GetScheme(),  }).SetupWithManager(mgr)  if err != nil {    setupLog.Error(err, "unable to create controller", "controller", "Etcd")    os.Exit(1)  }  if certDir != "" {    err = (&dbv1.Etcd{}).SetupWebhookWithManager(mgr)    if err != nil {      setupLog.Error(err, "unable to SetupWebhookWithManager")      os.Exit(1)    }  }}()

controllerManager 有个个性，如果曾经启动，则前面 Add 的 Runnable 间接启动：

// Add sets dependencies on i, and adds it to the list of Runnables to start.func (cm *controllerManager) Add(r Runnable) error {    // ...    if shouldStart {        // If already started, start the controller        cm.startRunnable(r)    }    return nil}

残缺例子：win5do/etcd-operator (github.com)

原理

Webhook 应用的证书不要求肯定是 k8s 集群 CA 根证书颁发的证书，所以咱们能够应用自签证书。

生成自签 CA 根证书：

// CreateCACert creates the self-signed CA cert and private key that will// be used to sign the server certificatefunc (cr *CertRotator) CreateCACert(begin, end time.Time) (*KeyPairArtifacts, error) {    // ...    key, err := rsa.GenerateKey(rand.Reader, 2048)    if err != nil {        return nil, errors.Wrap(err, "generating key")    }    der, err := x509.CreateCertificate(rand.Reader, templ, templ, key.Public(), key)    if err != nil {        return nil, errors.Wrap(err, "creating certificate")    }    certPEM, keyPEM, err := pemEncode(der, key)    if err != nil {        return nil, errors.Wrap(err, "encoding PEM")    }    cert, err := x509.ParseCertificate(der)    if err != nil {        return nil, errors.Wrap(err, "parsing certificate")    }    return &KeyPairArtifacts{Cert: cert, Key: key, CertPEM: certPEM, KeyPEM: keyPEM}, nil}

依据 service-name.namespace.svc 生成服务器 tls.key 和 tls.crt：

// CreateCertPEM takes the results of CreateCACert and uses it to create the// PEM-encoded public certificate and private key, respectivelyfunc (cr *CertRotator) CreateCertPEM(ca *KeyPairArtifacts, begin, end time.Time) ([]byte, []byte, error) {    // ...    key, err := rsa.GenerateKey(rand.Reader, 2048)    if err != nil {        return nil, nil, errors.Wrap(err, "generating key")    }    der, err := x509.CreateCertificate(rand.Reader, templ, ca.Cert, key.Public(), ca.Key)    if err != nil {        return nil, nil, errors.Wrap(err, "creating certificate")    }    certPEM, keyPEM, err := pemEncode(der, key)    if err != nil {        return nil, nil, errors.Wrap(err, "encoding PEM")    }    return certPEM, keyPEM, nil}

写入到 secret 中， 并期待 mount：

// ensureCertsMounted ensure the cert files exist.func (cr *CertRotator) ensureCertsMounted() {    checkFn := func() (bool, error) {        certFile := cr.CertDir + "/" + certName        _, err := os.Stat(certFile)        if err == nil {            return true, nil        }        return false, nil    }    if err := wait.ExponentialBackoff(wait.Backoff{        Duration: 1 * time.Second,        Factor:   2,        Jitter:   1,        Steps:    10,    }, checkFn); err != nil {        crLog.Error(err, "max retries for checking certs existence")        close(cr.certsNotMounted)        return    }    crLog.Info(fmt.Sprintf("certs are ready in %s", cr.CertDir))    close(cr.certsMounted)}

启动一个 Reconceile 监听 secret，如果有更新则将 caBundle 注入到 webhook config 中：

func injectCertToWebhook(wh *unstructured.Unstructured, certPem []byte) error {    webhooks, found, err := unstructured.NestedSlice(wh.Object, "webhooks")    if err != nil {        return err    }    if !found {        return errors.New("`webhooks` field not found in ValidatingWebhookConfiguration")    }    for i, h := range webhooks {        hook, ok := h.(map[string]interface{})        if !ok {            return errors.Errorf("webhook %d is not well-formed", i)        }        if err := unstructured.SetNestedField(hook, base64.StdEncoding.EncodeToString(certPem), "clientConfig", "caBundle"); err != nil {            return err        }        webhooks[i] = hook    }    if err := unstructured.SetNestedSlice(wh.Object, webhooks, "webhooks"); err != nil {        return err    }    return nil}