feat(controller): develop s3,database, redis, reconcilier

.dockerignore

@@ -2,3 +2,5 @@
 # Ignore build and test binaries.
 bin/
 testbin/
+
+.vscode

.github/workflows/ci.yml

@@ -0,0 +1,147 @@
+# 定义工作流的名称
+name: CI
+
+on:
+  workflow_dispatch: # 允许手动触发工作流
+  push: # 当有代码推送事件发生时
+    branches:
+      - '*' # 只有推送到 main 分支时才触发
+    tags:
+      - '*'
+  pull_request:
+
+
+# 定义工作流的任务
+jobs:
+  build:
+    name: Set up Build
+    runs-on: ubuntu-latest
+
+    # 定义任务中的步骤
+    steps:
+      # 检出代码。只检出最新的1次提交
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 1
+
+      # 设置 QEMU 模拟器，这通常用于多平台的 Docker 构建
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      # 设置 Docker Buildx，用于构建多平台的 Docker 镜像
+      - name: Set up Build
+        uses: docker/setup-buildx-action@v2
+      - run: |
+          ls -l
+
+  linting:
+    name: Lint Code Base
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v3
+        with:
+          # Full git history is needed to get a proper list of changed files within `super-linter`
+          fetch-depth: 0
+      - name: Lint Code Base
+        uses: github/super-linter@v4
+        env:
+          VALIDATE_MARKDOWN: true
+          VALIDATE_ALL_CODEBASE: false
+          DEFAULT_BRANCH: main
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+          LINTER_RULES_PATH: /
+          MARKDOWN_CONFIG_FILE: .markdownlint.yml
+
+  go-test:
+    name: Go Test
+    runs-on: ubuntu-latest
+
+    # Service containers to run with `runner-job`
+    services:
+      # Label used to access the service container
+      postgres:
+        # Docker Hub image
+        image: postgres
+        # Provide the password for postgres
+        env:
+          POSTGRES_PASSWORD: postgres
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          # Maps tcp port 5432 on service container to the host
+          - 5432:5432
+
+      redis:
+        # Docker Hub image
+        image: redis
+        # Set health checks to wait until redis has started
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          # Maps port 6379 on service container to the host
+          - 6379:6379
+
+      mysql:
+        image: mysql:5.7
+        env:
+          MYSQL_DATABASE: test_db
+          MYSQL_USER: mysql
+          MYSQL_PASSWORD: mysql
+          MYSQL_ROOT_PASSWORD: mysql
+        ports:
+          - 33306:3306
+        options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3
+
+      minio:
+        # fixme: let's not depend on external unofficial image
+        image: bitnami/minio:latest
+        ports:
+          - 9000:9000
+          - 9090:9090
+        env:
+          MINIO_ROOT_USER: admin
+          MINIO_ROOT_PASSWORD: admin123456
+        options: --name=minio --health-cmd "curl http://localhost:9000/minio/health/live"
+
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v3
+        with:
+          # Full git history is needed to get a proper list of changed files within `super-linter`
+          fetch-depth: 0
+      - name: Go Test
+        run: |
+          docker ps -a
+          go test -v ./...
+        env:
+          # The hostname used to communicate with the PostgreSQL service container
+          PG_DB_HOST: "localhost"
+          PG_DB_PORT: "5432"
+          PG_DB_SSLMODE: "false"
+          PG_DB_USERNAME: "postgres"
+          PG_DB_PASSWORD: "postgres"
+
+          # The hostname used to communicate with the Redis service container
+          REDIS_ADDR: "localhost:6379"
+
+          # The hostname used to communicate with the Mysql service container
+          MYSQL_DB_HOST: "localhost"
+          MYSQL_DB_PORT: "33306"
+          MYSQL_DB_SSLMODE: "false"
+          MYSQL_DB_USERNAME: "root"
+          MYSQL_DB_PASSWORD: "mysql"
+
+          # The hostname used to communicate with the Minio service container
+          MINIO_ENDPOINT: "localhost:9000"
+          MINIO_ACCESS_KEY: "admin"
+          MINIO_SECRET_KEY: "admin123456"
+          MINIO_SSL: "false"

.gitignore

@@ -25,4 +25,5 @@ Dockerfile.cross
 *.swo
 *~
 
-.env
+.env
+.vscode

.markdownlint.yml

@@ -0,0 +1,20 @@
+MD007:
+  indent: 4
+  start_indented: false
+
+MD013:
+  line_length: 200
+  heading_line_length: 80
+  code_block_line_length: 200
+  code_blocks: true
+  tables: true
+  headings: true
+  headers: true
+  strict: false
+  stern: false
+
+MD012:
+  maximum: 20
+
+MD046: false
+MD051: false

PROJECT

@@ -29,4 +29,31 @@ resources:
   kind: Database
   path: github.com/zncdata-labs/zncdata-stack-operator/api/v1alpha1
   version: v1alpha1
+- api:
+    crdVersion: v1
+    namespaced: true
+  controller: true
+  domain: zncdata.net
+  group: stack
+  kind: S3Connection
+  path: github.com/zncdata-labs/zncdata-stack-operator/api/v1alpha1
+  version: v1alpha1
+- api:
+    crdVersion: v1
+    namespaced: true
+  controller: true
+  domain: zncdata.net
+  group: stack
+  kind: S3Bucket
+  path: github.com/zncdata-labs/zncdata-stack-operator/api/v1alpha1
+  version: v1alpha1
+- api:
+    crdVersion: v1
+    namespaced: true
+  controller: true
+  domain: zncdata.net
+  group: stack
+  kind: RedisConnection
+  path: github.com/zncdata-labs/zncdata-stack-operator/api/v1alpha1
+  version: v1alpha1
 version: "3"

api/v1alpha1/conditions.go

@@ -0,0 +1,15 @@
+package v1alpha1
+
+const (
+	ConditionTypeProgressing string = "Progressing"
+	ConditionTypeReconcile   string = "Reconcile"
+	ConditionTypeAvailable   string = "Available"
+
+	ConditionReasonPreparing           string = "Preparing"
+	ConditionReasonRunning             string = "Running"
+	ConditionReasonConfig              string = "Config"
+	ConditionReasonReconcilePVC        string = "ReconcilePVC"
+	ConditionReasonReconcileService    string = "ReconcileService"
+	ConditionReasonReconcileIngress    string = "ReconcileIngress"
+	ConditionReasonReconcileDeployment string = "ReconcileDeployment"
+)

api/v1alpha1/database_types.go

@@ -17,22 +17,23 @@ limitations under the License.
 package v1alpha1
 
 import (
+	apimeta "k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+const DatabaseFinalizer = "database.finalizers.stack.zncdata.net"
+
 // DatabaseSpec defines the desired state of Database
 type DatabaseSpec struct {
-	Name       string          `json:"name,omitempty"`
-	Connection *ConnectionSpec `json:"connection,omitempty"`
-	Credential *CredentialSpec `json:"credential,omitempty"`
+	Name       string                  `json:"name,omitempty"`
+	Reference  string                  `json:"reference,omitempty"`
+	Credential *DatabaseCredentialSpec `json:"credential,omitempty"`
 }
 
-type ConnectionSpec struct {
-	Reference string `json:"reference,omitempty"`
-	Driver    string `json:"driver,omitempty"`
-	Host      string `json:"host,omitempty"`
-	Port      int    `json:"port,omitempty"`
-	SSL       bool   `json:"ssl,omitempty"`
+type DatabaseCredentialSpec struct {
+	ExistSecret string `json:"existingSecret,omitempty"`
+	Username    string `json:"username,omitempty"`
+	Password    string `json:"password,omitempty"`
 }
 
 // DatabaseStatus defines the observed state of Database
@@ -53,6 +54,33 @@ type Database struct {
 	Status DatabaseStatus `json:"status,omitempty"`
 }
 
+func (in *Database) GetNameWithSuffix(s string) string {
+	return in.Name + "-" + s
+}
+
+func (in *Database) SetStatusCondition(condition metav1.Condition) {
+	// if the condition already exists, update it
+	existingCondition := apimeta.FindStatusCondition(in.Status.Conditions, condition.Type)
+	if existingCondition == nil {
+		condition.ObservedGeneration = in.GetGeneration()
+		condition.LastTransitionTime = metav1.Now()
+		in.Status.Conditions = append(in.Status.Conditions, condition)
+	} else if existingCondition.Status != condition.Status || existingCondition.Reason != condition.Reason || existingCondition.Message != condition.Message {
+		existingCondition.Status = condition.Status
+		existingCondition.Reason = condition.Reason
+		existingCondition.Message = condition.Message
+		existingCondition.ObservedGeneration = in.GetGeneration()
+		existingCondition.LastTransitionTime = metav1.Now()
+	}
+}
+
+func (in *Database) IsAvailable() bool {
+	if cond := apimeta.FindStatusCondition(in.Status.Conditions, ConditionTypeAvailable); cond != nil && cond.Status == metav1.ConditionTrue && string(cond.Status) == ConditionReasonRunning {
+		return true
+	}
+	return false
+}
+
 //+kubebuilder:object:root=true
 
 // DatabaseList contains a list of Database

api/v1alpha1/databaseconnection_types.go

@@ -17,6 +17,7 @@ limitations under the License.
 package v1alpha1
 
 import (
+	apimeta "k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -26,22 +27,21 @@ import (
 // DatabaseConnectionSpec defines the desired state of DatabaseConnection
 type DatabaseConnectionSpec struct {
 	Provider *ProviderSpec `json:"provider,omitempty"`
+	Default  bool          `json:"default,omitempty"`
 }
 
 type ProviderSpec struct {
 	// +kubebuilder:validation:Enum=mysql;postgres
 	// +kubebulider:default=postgres
-	Driver     string          `json:"driver,omitempty"`
-	Host       string          `json:"host,omitempty"`
-	Port       int             `json:"port,omitempty"`
-	SSL        bool            `json:"ssl,omitempty"`
-	Credential *CredentialSpec `json:"credential,omitempty"`
+	Driver     string                            `json:"driver,omitempty"`
+	Host       string                            `json:"host,omitempty"`
+	Port       int                               `json:"port,omitempty"`
+	SSL        bool                              `json:"ssl,omitempty"`
+	Credential *DatabaseConnectionCredentialSpec `json:"credential,omitempty"`
 }
 
-type CredentialSpec struct {
-	ExistSecret string `json:"existSecret,omitempty"`
-	Username    string `json:"username,omitempty"`
-	Password    string `json:"password,omitempty"`
+type DatabaseConnectionCredentialSpec struct {
+	ExistSecret string `json:"existingSecret,omitempty"`
 }
 
 // DatabaseConnectionStatus defines the observed state of DatabaseConnection
@@ -74,3 +74,50 @@ type DatabaseConnectionList struct {
 func init() {
 	SchemeBuilder.Register(&DatabaseConnection{}, &DatabaseConnectionList{})
 }
+
+// SetStatusCondition updates the status condition using the provided arguments.
+// If the condition already exists, it updates the condition; otherwise, it appends the condition.
+// If the condition status has changed, it updates the condition's LastTransitionTime.
+func (r *DatabaseConnection) SetStatusCondition(condition metav1.Condition) {
+	// if the condition already exists, update it
+	existingCondition := apimeta.FindStatusCondition(r.Status.Conditions, condition.Type)
+	if existingCondition == nil {
+		condition.ObservedGeneration = r.GetGeneration()
+		condition.LastTransitionTime = metav1.Now()
+		r.Status.Conditions = append(r.Status.Conditions, condition)
+	} else if existingCondition.Status != condition.Status || existingCondition.Reason != condition.Reason || existingCondition.Message != condition.Message {
+		existingCondition.Status = condition.Status
+		existingCondition.Reason = condition.Reason
+		existingCondition.Message = condition.Message
+		existingCondition.ObservedGeneration = r.GetGeneration()
+		existingCondition.LastTransitionTime = metav1.Now()
+	}
+}
+
+// InitStatusConditions initializes the status conditions to the provided conditions.
+func (r *DatabaseConnection) InitStatusConditions() {
+	r.Status.Conditions = []metav1.Condition{}
+	r.SetStatusCondition(metav1.Condition{
+		Type:               ConditionTypeProgressing,
+		Status:             metav1.ConditionTrue,
+		Reason:             ConditionReasonPreparing,
+		Message:            "DatabaseConnection is preparing",
+		ObservedGeneration: r.GetGeneration(),
+		LastTransitionTime: metav1.Now(),
+	})
+	r.SetStatusCondition(metav1.Condition{
+		Type:               ConditionTypeAvailable,
+		Status:             metav1.ConditionFalse,
+		Reason:             ConditionReasonPreparing,
+		Message:            "DatabaseConnection is preparing",
+		ObservedGeneration: r.GetGeneration(),
+		LastTransitionTime: metav1.Now(),
+	})
+}
+
+func (r *DatabaseConnection) IsAvailable() bool {
+	if cond := apimeta.FindStatusCondition(r.Status.Conditions, ConditionTypeAvailable); cond != nil && cond.Status == metav1.ConditionTrue && string(cond.Status) == ConditionReasonRunning {
+		return true
+	}
+	return false
+}