主题
自定义CRD开发
课程目标
通过本课程的学习,你将能够:
- 深入理解CRD的概念、结构和工作原理
- 掌握OpenAPI v3 schema验证配置
- 实现CRD多版本管理和版本转换
- 开发复杂的自定义控制器
- 使用Status子资源和Finalizer
- 应用CRD开发最佳实践
前置要求:已完成《Operator开发基础》课程,具备Operator SDK使用经验
一、CRD深入理解
1.1 CRD架构回顾
┌─────────────────────────────────────────────────────────────────┐
│ CRD架构详解 │
├─────────────────────────────────────────────────────────────────┤
│ │
│ CRD (CustomResourceDefinition) │
│ ├─ 定义新的Kubernetes资源类型 │
│ ├─ 扩展Kubernetes API │
│ └─ 提供声明式配置接口 │
│ │
│ 核心组成部分: │
│ ┌──────────────────────────────────────────────────────────┐ │
│ │ Group (API组) + Version (版本) + Kind (资源类型) │ │
│ │ 例如:apps.example.com/v1/MyApp │ │
│ └──────────────────────────────────────────────────────────┘ │
│ │
│ Schema定义: │
│ ├─ OpenAPI v3规范 │
│ ├─ 字段类型和验证 │
│ ├─ 默认值和必需字段 │
│ └─ 条件字段和依赖关系 │
│ │
└─────────────────────────────────────────────────────────────────┘1.2 CRD完整结构
yaml
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: myapps.apps.example.com # 格式: <plural>.<group>
spec:
# API组名
group: apps.example.com
# 命名范围
scope: Namespaced # 或 Cluster
# 名称定义
names:
plural: myapps # 复数形式,用于URL
singular: myapp # 单数形式
kind: MyApp # 资源类型名
shortNames: # 短名称
- ma
- mya
categories: # 分类
- all
# 版本定义
versions:
- name: v1
served: true # 是否提供服务
storage: true # 是否存储此版本
# Schema定义
schema:
openAPIV3Schema:
type: object
properties:
spec:
type: object
properties:
# 字段定义...
status:
type: object
properties:
# 状态定义...
# 子资源
subresources:
status: {} # 启用status子资源
scale: # 启用scale子资源
specReplicasPath: .spec.replicas
statusReplicasPath: .status.replicas
labelSelectorPath: .status.labelSelector
# 额外打印列
additionalPrinterColumns:
- name: Replicas
type: integer
jsonPath: .spec.replicas
- name: Status
type: string
jsonPath: .status.phase二、OpenAPI Schema验证
2.1 基础类型验证
go
// api/v1/myapp_types.go
type MyAppSpec struct {
// 字符串验证
// +kubebuilder:validation:MinLength=1
// +kubebuilder:validation:MaxLength=63
// +kubebuilder:validation:Pattern=^[a-z0-9]([-a-z0-9]*[a-z0-9])?$
Name string `json:"name"`
// 枚举验证
// +kubebuilder:validation:Enum=info;debug;warn;error
// +kubebuilder:default=info
LogLevel string `json:"logLevel,omitempty"`
// 整数验证
// +kubebuilder:validation:Minimum=1
// +kubebuilder:validation:Maximum=100
// +kubebuilder:default=1
Replicas int32 `json:"replicas,omitempty"`
// 布尔值
// +kubebuilder:default=false
Enabled bool `json:"enabled,omitempty"`
// 必需字段
// +kubebuilder:validation:Required
Image string `json:"image"`
}2.2 复杂类型验证
go
// 对象类型
type ServiceConfig struct {
// +kubebuilder:validation:Enum=ClusterIP;NodePort;LoadBalancer
Type string `json:"type,omitempty"`
// +kubebuilder:validation:Minimum=1
// +kubebuilder:validation:Maximum=65535
Port int32 `json:"port,omitempty"`
}
// 数组类型
type MyAppSpec struct {
// 字符串数组
// +kubebuilder:validation:MinItems=1
// +kubebuilder:validation:MaxItems=10
EnvVars []string `json:"envVars,omitempty"`
// 对象数组
// +listType=map
// +listMapKey=name
Volumes []VolumeConfig `json:"volumes,omitempty"`
// 嵌套对象
Service ServiceConfig `json:"service,omitempty"`
}
type VolumeConfig struct {
// +kubebuilder:validation:Required
Name string `json:"name"`
// +kubebuilder:validation:Enum=EmptyDir;HostPath;PVC
Type string `json:"type"`
Size string `json:"size,omitempty"`
}2.3 高级验证规则
go
// 条件验证(oneOf, anyOf, allOf)
// +kubebuilder:validation:XValidation:rule="self.type == 'EmptyDir' || self.size != ''",message="非EmptyDir类型必须指定size"
type VolumeConfig struct {
Type string `json:"type"`
Size string `json:"size,omitempty"`
}
// 字段依赖关系
// +kubebuilder:validation:XValidation:rule="!self.enabled || self.image != ''",message="启用时必须指定镜像"
type MyAppSpec struct {
Enabled bool `json:"enabled,omitempty"`
Image string `json:"image,omitempty"`
}
// 自定义CEL表达式验证(1.25+)
// +kubebuilder:validation:XValidation:rule="self.replicas >= self.minReplicas",message="副本数不能小于最小副本数"
// +kubebuilder:validation:XValidation:rule="self.replicas <= self.maxReplicas",message="副本数不能大于最大副本数"
type MyAppSpec struct {
Replicas int32 `json:"replicas"`
MinReplicas int32 `json:"minReplicas"`
MaxReplicas int32 `json:"maxReplicas"`
}三、多版本管理
3.1 版本策略
yaml
spec:
versions:
# v1alpha1 - 早期实验版本
- name: v1alpha1
served: true
storage: false
deprecated: true
deprecationWarning: "v1alpha1 is deprecated, use v1 instead"
schema:
openAPIV3Schema:
type: object
properties:
spec:
type: object
properties:
name:
type: string
# v1beta1 - 测试版本
- name: v1beta1
served: true
storage: false
schema:
openAPIV3Schema:
type: object
properties:
spec:
type: object
properties:
name:
type: string
replicas:
type: integer
# v1 - 稳定版本
- name: v1
served: true
storage: true # 只有一个是storage版本
schema:
openAPIV3Schema:
type: object
properties:
spec:
type: object
properties:
name:
type: string
replicas:
type: integer
config:
type: object3.2 版本转换(Webhook)
当API版本发生变化时,需要实现版本转换:
go
// api/v1alpha1/myapp_types.go
type MyAppSpec struct {
Name string `json:"name"` // 旧版本只有name字段
}
// api/v1/myapp_types.go
type MyAppSpec struct {
Name string `json:"name"`
Replicas int32 `json:"replicas"` // 新版本增加了replicas
}
// api/v1/myapp_conversion.go
package v1
import (
"sigs.k8s.io/controller-runtime/pkg/conversion"
oldv1 "github.com/example/myapp-operator/api/v1alpha1"
)
// ConvertTo converts this MyApp to the Hub version (v1)
func (src *MyApp) ConvertTo(dstRaw conversion.Hub) error {
dst := dstRaw.(*v1.MyApp)
// 转换metadata
dst.ObjectMeta = src.ObjectMeta
// 转换spec
dst.Spec.Name = src.Spec.Name
dst.Spec.Replicas = 1 // 默认值
return nil
}
// ConvertFrom converts from the Hub version (v1) to this version
func (dst *MyApp) ConvertFrom(srcRaw conversion.Hub) error {
src := srcRaw.(*v1.MyApp)
// 转换metadata
dst.ObjectMeta = src.ObjectMeta
// 转换spec
dst.Spec.Name = src.Spec.Name
// Replicas字段在v1alpha1中不存在,忽略
return nil
}3.3 在Kubebuilder中配置多版本
bash
# 创建v1版本
operator-sdk create api --group apps --version v1 --kind MyApp --resource --controller
# 创建v2版本(同一资源的新版本)
operator-sdk create api --group apps --version v2 --kind MyApp --resource
# 标记Hub版本(存储版本)
# 在v1/myapp_types.go中添加注释
// +kubebuilder:storageversion四、Status子资源
4.1 Status设计
go
// api/v1/myapp_types.go
// MyAppStatus defines the observed state of MyApp
type MyAppStatus struct {
// 当前阶段
// +optional
Phase AppPhase `json:"phase,omitempty"`
// 条件列表
// +patchMergeKey=type
// +patchStrategy=merge
// +listType=map
// +listMapKey=type
Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"`
// 可用副本数
// +optional
AvailableReplicas int32 `json:"availableReplicas,omitempty"`
// 观察到的资源版本
// +optional
ObservedGeneration int64 `json:"observedGeneration,omitempty"`
// 上次更新时间
// +optional
LastUpdateTime *metav1.Time `json:"lastUpdateTime,omitempty"`
}
// AppPhase 定义应用阶段
type AppPhase string
const (
AppPhasePending AppPhase = "Pending"
AppPhaseCreating AppPhase = "Creating"
AppPhaseRunning AppPhase = "Running"
AppPhaseFailed AppPhase = "Failed"
AppPhaseDeleting AppPhase = "Deleting"
)
// 条件类型常量
const (
ConditionTypeReady = "Ready"
ConditionTypeProgressing = "Progressing"
ConditionTypeDegraded = "Degraded"
)4.2 状态更新实现
go
// controllers/myapp_status.go
package controllers
import (
"context"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/ptr"
myappv1 "github.com/example/myapp-operator/api/v1"
)
// updateStatus 更新MyApp状态
func (r *MyAppReconciler) updateStatus(ctx context.Context, myapp *myappv1.MyApp) error {
// 获取Deployment状态
deployment := &appsv1.Deployment{}
err := r.Get(ctx, types.NamespacedName{Name: myapp.Name, Namespace: myapp.Namespace}, deployment)
// 更新观察到的generation
myapp.Status.ObservedGeneration = myapp.Generation
// 更新副本数
if err == nil {
myapp.Status.AvailableReplicas = deployment.Status.AvailableReplicas
}
// 确定阶段
switch {
case !myapp.DeletionTimestamp.IsZero():
myapp.Status.Phase = myappv1.AppPhaseDeleting
case err != nil:
myapp.Status.Phase = myappv1.AppPhaseFailed
case deployment.Status.AvailableReplicas == 0:
myapp.Status.Phase = myappv1.AppPhasePending
case deployment.Status.AvailableReplicas < *deployment.Spec.Replicas:
myapp.Status.Phase = myappv1.AppPhaseCreating
default:
myapp.Status.Phase = myappv1.AppPhaseRunning
}
// 更新条件
r.updateConditions(myapp, err)
// 更新最后更新时间
now := metav1.Now()
myapp.Status.LastUpdateTime = &now
return r.Status().Update(ctx, myapp)
}
// updateConditions 更新条件
func (r *MyAppReconciler) updateConditions(myapp *myappv1.MyApp, deploymentErr error) {
// Ready条件
readyCondition := metav1.Condition{
Type: myappv1.ConditionTypeReady,
LastTransitionTime: metav1.Now(),
}
if myapp.Status.Phase == myappv1.AppPhaseRunning {
readyCondition.Status = metav1.ConditionTrue
readyCondition.Reason = "AppRunning"
readyCondition.Message = "Application is running normally"
} else {
readyCondition.Status = metav1.ConditionFalse
readyCondition.Reason = "AppNotReady"
readyCondition.Message = string(myapp.Status.Phase)
}
if deploymentErr != nil {
readyCondition.Status = metav1.ConditionFalse
readyCondition.Reason = "DeploymentError"
readyCondition.Message = deploymentErr.Error()
}
metav1.SetStatusCondition(&myapp.Status.Conditions, readyCondition)
// Progressing条件
progressingCondition := metav1.Condition{
Type: myappv1.ConditionTypeProgressing,
LastTransitionTime: metav1.Now(),
}
if myapp.Status.Phase == myappv1.AppPhaseCreating {
progressingCondition.Status = metav1.ConditionTrue
progressingCondition.Reason = "CreatingResources"
progressingCondition.Message = "Creating application resources"
} else {
progressingCondition.Status = metav1.ConditionFalse
progressingCondition.Reason = "ProcessingComplete"
progressingCondition.Message = "Resource processing complete"
}
metav1.SetStatusCondition(&myapp.Status.Conditions, progressingCondition)
}五、Finalizer资源清理
5.1 Finalizer原理
┌─────────────────────────────────────────────────────────────────┐
│ Finalizer工作原理 │
├─────────────────────────────────────────────────────────────────┤
│ │
│ 删除CR时的流程: │
│ │
│ 1. 用户执行删除 │
│ kubectl delete myapp myapp-sample │
│ │ │
│ ▼ │
│ 2. K8s设置DeletionTimestamp │
│ CR进入"Terminating"状态 │
│ │ │
│ ▼ │
│ 3. Operator检测删除事件 │
│ 执行清理逻辑(删除外部资源) │
│ │ │
│ ▼ │
│ 4. 清理完成后移除Finalizer │
│ metadata.finalizers = [] │
│ │ │
│ ▼ │
│ 5. K8s真正删除CR │
│ │
└─────────────────────────────────────────────────────────────────┘5.2 Finalizer实现
go
// controllers/finalizer.go
package controllers
import (
"context"
"k8s.io/apimachinery/pkg/api/errors"
"controller-runtime/pkg/controller/controllerutil"
myappv1 "github.com/example/myapp-operator/api/v1"
)
const (
myAppFinalizer = "myapp.apps.example.com/finalizer"
)
// handleFinalizer 处理Finalizer逻辑
func (r *MyAppReconciler) handleFinalizer(ctx context.Context, myapp *myappv1.MyApp) (bool, error) {
log := log.FromContext(ctx)
// 检查是否正在删除
if !myapp.ObjectMeta.DeletionTimestamp.IsZero() {
// 正在删除,执行清理
if controllerutil.ContainsFinalizer(myapp, myAppFinalizer) {
log.Info("Performing Finalizer Operations")
// 执行清理操作
if err := r.cleanupResources(ctx, myapp); err != nil {
log.Error(err, "Failed to cleanup resources")
return false, err
}
// 移除finalizer
controllerutil.RemoveFinalizer(myapp, myAppFinalizer)
if err := r.Update(ctx, myapp); err != nil {
log.Error(err, "Failed to remove finalizer")
return false, err
}
}
// 停止调谐,等待K8s删除
return true, nil
}
// 未删除,确保有finalizer
if !controllerutil.ContainsFinalizer(myapp, myAppFinalizer) {
controllerutil.AddFinalizer(myapp, myAppFinalizer)
if err := r.Update(ctx, myapp); err != nil {
log.Error(err, "Failed to add finalizer")
return false, err
}
return true, nil // 需要重新调谐
}
return false, nil // 继续正常调谐
}
// cleanupResources 清理外部资源
func (r *MyAppReconciler) cleanupResources(ctx context.Context, myapp *myappv1.MyApp) error {
log := log.FromContext(ctx)
// 示例:删除外部数据库
if myapp.Spec.ExternalDatabase != nil {
log.Info("Cleaning up external database", "name", myapp.Spec.ExternalDatabase.Name)
if err := r.deleteExternalDatabase(myapp); err != nil {
return err
}
}
// 示例:删除备份数据
if myapp.Spec.BackupEnabled {
log.Info("Cleaning up backup data")
if err := r.deleteBackupData(ctx, myapp); err != nil {
return err
}
}
// 示例:通知外部系统
if err := r.notifyDeletion(myapp); err != nil {
log.Error(err, "Failed to notify external system")
// 根据需求决定是否返回错误
}
return nil
}
// 在Reconcile中调用
func (r *MyAppReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
myapp := &myappv1.MyApp{}
if err := r.Get(ctx, req.NamespacedName, myapp); err != nil {
return ctrl.Result{}, err
}
// 处理Finalizer
stop, err := r.handleFinalizer(ctx, myapp)
if err != nil {
return ctrl.Result{}, err
}
if stop {
return ctrl.Result{}, nil
}
// 正常调谐逻辑...
}六、高级控制器模式
6.1 外部资源管理
go
// 管理外部资源(如云数据库)
// ExternalResourceManager 外部资源管理器接口
type ExternalResourceManager interface {
Create(ctx context.Context, spec *ExternalResourceSpec) (*ExternalResourceStatus, error)
Delete(ctx context.Context, id string) error
Get(ctx context.Context, id string) (*ExternalResourceStatus, error)
Update(ctx context.Context, id string, spec *ExternalResourceSpec) error
}
// 在控制器中使用
func (r *MyAppReconciler) reconcileExternalResources(ctx context.Context, myapp *myappv1.MyApp) error {
if myapp.Spec.ExternalDatabase == nil {
return nil
}
// 检查是否已创建
if myapp.Status.ExternalDBID == "" {
// 创建外部数据库
status, err := r.ExternalDBManager.Create(ctx, &ExternalResourceSpec{
Name: myapp.Name,
Namespace: myapp.Namespace,
Engine: myapp.Spec.ExternalDatabase.Engine,
Version: myapp.Spec.ExternalDatabase.Version,
Size: myapp.Spec.ExternalDatabase.Size,
})
if err != nil {
return err
}
// 保存外部资源ID
myapp.Status.ExternalDBID = status.ID
myapp.Status.ExternalDBEndpoint = status.Endpoint
return r.Status().Update(ctx, myapp)
}
// 检查外部资源状态
status, err := r.ExternalDBManager.Get(ctx, myapp.Status.ExternalDBID)
if err != nil {
return err
}
if status.State != "available" {
return fmt.Errorf("external database not available: %s", status.State)
}
return nil
}6.2 事件和告警
go
// 发送K8s事件
func (r *MyAppReconciler) recordEvent(myapp *myappv1.MyApp, eventType, reason, message string) {
r.Recorder.Event(myapp, eventType, reason, message)
}
// 在控制器中使用
func (r *MyAppReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
// ...
// 创建成功
r.recordEvent(myapp, corev1.EventTypeNormal, "Created",
fmt.Sprintf("Successfully created deployment %s", deployment.Name))
// 发生错误
if err != nil {
r.recordEvent(myapp, corev1.EventTypeWarning, "Failed",
fmt.Sprintf("Failed to create deployment: %v", err))
}
}七、CRD最佳实践
7.1 设计原则
go
// ✅ 好的CRD设计
// 1. 清晰的版本策略
type MyAppSpec struct {
// 使用omitempty让字段可选
Replicas int32 `json:"replicas,omitempty"`
// 使用指针类型区分"未设置"和"零值"
Replicas *int32 `json:"replicas,omitempty"`
// 提供合理的默认值
// +kubebuilder:default=1
Replicas int32 `json:"replicas,omitempty"`
}
// 2. 状态与规格分离
// Spec: 用户期望的状态(可写)
// Status: 系统观察到的状态(只读)
// 3. 使用条件记录详细状态
type MyAppStatus struct {
Conditions []metav1.Condition `json:"conditions,omitempty"`
}
// 4. 资源引用使用标准格式
type ResourceReference struct {
Name string `json:"name"`
Namespace string `json:"namespace,omitempty"`
}7.2 完整示例
go
// api/v1/myapp_types.go
package v1
import (
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// +kubebuilder:object:root=true
// +kubebuilder:subresource:status
// +kubebuilder:subresource:scale:specpath=.spec.replicas,statuspath=.status.replicas
// +kubebuilder:resource:path=myapps,shortName=ma;mya,categories=all
// +kubebuilder:storageversion
// +kubebuilder:printcolumn:name="Replicas",type=integer,JSONPath=.spec.replicas
// +kubebuilder:printcolumn:name="Available",type=integer,JSONPath=.status.availableReplicas
// +kubebuilder:printcolumn:name="Status",type=string,JSONPath=.status.phase
// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=.metadata.creationTimestamp
type MyApp struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`
Spec MyAppSpec `json:"spec,omitempty"`
Status MyAppStatus `json:"status,omitempty"`
}
type MyAppSpec struct {
// +kubebuilder:validation:Minimum=0
// +kubebuilder:validation:Maximum=100
// +kubebuilder:default=1
Replicas *int32 `json:"replicas,omitempty"`
// +kubebuilder:validation:Required
// +kubebuilder:validation:MinLength=1
Image string `json:"image"`
// +kubebuilder:validation:Enum=Always;Never;IfNotPresent
// +kubebuilder:default=IfNotPresent
ImagePullPolicy corev1.PullPolicy `json:"imagePullPolicy,omitempty"`
Resources corev1.ResourceRequirements `json:"resources,omitempty"`
Env []corev1.EnvVar `json:"env,omitempty"`
Service ServiceConfig `json:"service,omitempty"`
// +kubebuilder:default=false
IngressEnabled bool `json:"ingressEnabled,omitempty"`
Storage *StorageConfig `json:"storage,omitempty"`
}
type ServiceConfig struct {
// +kubebuilder:validation:Enum=ClusterIP;NodePort;LoadBalancer
// +kubebuilder:default=ClusterIP
Type corev1.ServiceType `json:"type,omitempty"`
// +kubebuilder:validation:Minimum=1
// +kubebuilder:validation:Maximum=65535
// +kubebuilder:default=80
Port int32 `json:"port,omitempty"`
}
type StorageConfig struct {
// +kubebuilder:validation:Required
Size string `json:"size"`
// +kubebuilder:validation:Enum=standard;fast-ssd
// +kubebuilder:default=standard
StorageClassName string `json:"storageClassName,omitempty"`
}
type MyAppStatus struct {
Phase AppPhase `json:"phase,omitempty"`
AvailableReplicas int32 `json:"availableReplicas,omitempty"`
Conditions []metav1.Condition `json:"conditions,omitempty"`
ObservedGeneration int64 `json:"observedGeneration,omitempty"`
}
type AppPhase string
const (
AppPhasePending AppPhase = "Pending"
AppPhaseCreating AppPhase = "Creating"
AppPhaseRunning AppPhase = "Running"
AppPhaseFailed AppPhase = "Failed"
)
// +kubebuilder:object:root=true
type MyAppList struct {
metav1.TypeMeta `json:",inline"`
metav1.ListMeta `json:"metadata,omitempty"`
Items []MyApp `json:"items"`
}八、总结
核心概念回顾
CRD高级开发
├── Schema验证
│ ├── 基础类型验证(字符串、整数、枚举)
│ ├── 复杂类型(对象、数组、嵌套)
│ └── 高级验证(CEL表达式、条件验证)
│
├── 多版本管理
│ ├── 版本策略(alpha/beta/stable)
│ └── 版本转换(Webhook)
│
├── Status子资源
│ ├── Phase状态机
│ ├── Conditions条件
│ └── 观察到的状态
│
└── Finalizer
├── 资源清理流程
└── 外部资源管理下节预告
在《K8s客户端开发》中,我们将:
- 深入理解client-go架构
- 掌握Informer机制
- 使用WorkQueue实现异步处理
- 开发K8s管理工具和CLI
💡 学习建议:
- 实践多版本CRD的创建和转换
- 设计合理的Status结构,使用Conditions记录详细状态
- 为所有需要清理外部资源的CR添加Finalizer