From 0d1a34ab3d9160035523187a68f7a809386bf3dd Mon Sep 17 00:00:00 2001 From: GreatRiver Date: Wed, 4 Feb 2026 11:07:48 +0800 Subject: [PATCH 01/18] feat(gc): add sync protection mechanism for cross-cluster sync This commit implements a sync protection mechanism to prevent GC from deleting files that are being synchronized across clusters. Key changes: - Add SyncProtectionManager to manage sync protection entries - Use index.BloomFilter (xorfilter-based, deterministic) instead of bloomfilter.BloomFilter (wyhash-based, non-deterministic) - Integrate sync protection check into MakeBloomfilterCoarseFilter so protected files stay in filesNotGC (recorded in GC metadata) - Add mo_ctl handlers for register/renew/unregister sync protection - Add mo-tool sync-protection command for testing The protection is applied at the coarse filter stage to ensure: 1. Protected files are recorded in GC window metadata 2. Protected files are not deleted during GC 3. After protection is released, files can be GC'd normally --- cmd/mo-tool/main.go | 1 + cmd/mo-tool/sync_protection.go | 485 ++++++++++++++++++ pkg/common/bloomfilter/bloomfilter.go | 93 ++++ pkg/sql/plan/function/ctl/cmd_disk_cleaner.go | 32 +- pkg/vm/engine/cmd_util/operations.go | 8 + pkg/vm/engine/cmd_util/operations.proto | 8 + pkg/vm/engine/cmd_util/type.go | 5 + pkg/vm/engine/tae/db/gc/v3/checkpoint.go | 59 +++ pkg/vm/engine/tae/db/gc/v3/exec_v1.go | 52 +- pkg/vm/engine/tae/db/gc/v3/mock_cleaner.go | 4 + pkg/vm/engine/tae/db/gc/v3/sync_protection.go | 399 ++++++++++++++ .../tae/db/gc/v3/sync_protection_test.go | 381 ++++++++++++++ pkg/vm/engine/tae/db/gc/v3/types.go | 3 + pkg/vm/engine/tae/db/gc/v3/window.go | 2 + pkg/vm/engine/tae/rpc/handle_debug.go | 87 ++++ 15 files changed, 1598 insertions(+), 21 deletions(-) create mode 100644 cmd/mo-tool/sync_protection.go create mode 100644 pkg/vm/engine/tae/db/gc/v3/sync_protection.go create mode 100644 pkg/vm/engine/tae/db/gc/v3/sync_protection_test.go diff --git a/cmd/mo-tool/main.go b/cmd/mo-tool/main.go index 8df6e75f29d51..a5eaa4663e902 100644 --- a/cmd/mo-tool/main.go +++ b/cmd/mo-tool/main.go @@ -37,6 +37,7 @@ func main() { rootCmd.AddCommand(dashboard.PrepareCommand()) rootCmd.AddCommand(object.PrepareCommand()) rootCmd.AddCommand(ckp.PrepareCommand()) + rootCmd.AddCommand(PrepareSyncProtectionCommand()) if err := rootCmd.Execute(); err != nil { os.Exit(1) diff --git a/cmd/mo-tool/sync_protection.go b/cmd/mo-tool/sync_protection.go new file mode 100644 index 0000000000000..9715a40c9a6dd --- /dev/null +++ b/cmd/mo-tool/sync_protection.go @@ -0,0 +1,485 @@ +// Copyright 2021 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "database/sql" + "encoding/base64" + "encoding/json" + "fmt" + "math/rand" + "os" + "path/filepath" + "strings" + "time" + + "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/containers" + "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/index" + + _ "github.com/go-sql-driver/mysql" + "github.com/spf13/cobra" +) + +// SyncProtectionRequest represents a sync protection request +type SyncProtectionRequest struct { + JobID string `json:"job_id"` + BF string `json:"bf"` // Base64 encoded BloomFilter + ValidTS int64 `json:"valid_ts"` + TestObject string `json:"test_object"` // Test object name (for debugging) +} + +// SyncProtectionTester tests the sync protection mechanism +type SyncProtectionTester struct { + db *sql.DB + dataDir string + jobID string + protectedFiles []string + sampleCount int + verbose bool + waitTime int +} + +func NewSyncProtectionTester(dsn, dataDir string, sampleCount int, verbose bool, waitTime int) (*SyncProtectionTester, error) { + db, err := sql.Open("mysql", dsn) + if err != nil { + return nil, fmt.Errorf("failed to connect to database: %w", err) + } + + if err := db.Ping(); err != nil { + return nil, fmt.Errorf("failed to ping database: %w", err) + } + + return &SyncProtectionTester{ + db: db, + dataDir: dataDir, + jobID: fmt.Sprintf("sync-test-%d", time.Now().UnixNano()), + sampleCount: sampleCount, + verbose: verbose, + waitTime: waitTime, + }, nil +} + +func (t *SyncProtectionTester) Close() { + if t.db != nil { + t.db.Close() + } +} + +// ScanObjectFiles scans the directory for object files +func (t *SyncProtectionTester) ScanObjectFiles() ([]string, error) { + var objects []string + + err := filepath.Walk(t.dataDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + + // Match object file name pattern + name := info.Name() + // MatrixOne object files are typically UUID format with underscore + // Format: 019c226d-9e98-7ecc-9662-712ff0edcbfb_00000 (42 characters) + if len(name) == 42 && strings.Contains(name, "_") && strings.Count(name, "-") == 4 { + objects = append(objects, name) + } + return nil + }) + + if err != nil { + return nil, fmt.Errorf("failed to scan directory: %w", err) + } + + return objects, nil +} + +// SelectRandomObjects randomly selects objects +func (t *SyncProtectionTester) SelectRandomObjects(objects []string, count int) []string { + if len(objects) <= count { + return objects + } + + // Copy slice to avoid modifying original data + copied := make([]string, len(objects)) + copy(copied, objects) + + // Shuffle randomly + rand.Shuffle(len(copied), func(i, j int) { + copied[i], copied[j] = copied[j], copied[i] + }) + + return copied[:count] +} + +// BuildBloomFilter builds a BloomFilter using xorfilter (deterministic hash) +func (t *SyncProtectionTester) BuildBloomFilter(objects []string) (string, error) { + // Create a containers.Vector with all object names + vec := containers.MakeVector(types.T_varchar.ToType(), mpool.MustNewZero()) + defer vec.Close() + + for _, obj := range objects { + vec.Append([]byte(obj), false) + } + + // Create BloomFilter using index.NewBloomFilter (xorfilter based) + bf, err := index.NewBloomFilter(vec) + if err != nil { + return "", fmt.Errorf("failed to create BloomFilter: %w", err) + } + + // Marshal BloomFilter + data, err := bf.Marshal() + if err != nil { + return "", fmt.Errorf("failed to marshal BloomFilter: %w", err) + } + + // Base64 encode + base64Data := base64.StdEncoding.EncodeToString(data) + + if t.verbose { + fmt.Printf(" BloomFilter: %d objects, %d bytes, base64 len=%d\n", len(objects), len(data), len(base64Data)) + } + + return base64Data, nil +} + +// RegisterProtection registers protection +func (t *SyncProtectionTester) RegisterProtection(objects []string) error { + // Build BloomFilter + bfData, err := t.BuildBloomFilter(objects) + if err != nil { + return fmt.Errorf("failed to build BloomFilter: %w", err) + } + + // Send first protected object name for testing + testObject := "" + if len(objects) > 0 { + testObject = objects[0] + } + + req := SyncProtectionRequest{ + JobID: t.jobID, + BF: bfData, + ValidTS: time.Now().UnixNano(), + TestObject: testObject, + } + + jsonData, err := json.Marshal(req) + if err != nil { + return fmt.Errorf("failed to marshal request: %w", err) + } + + query := fmt.Sprintf("SELECT mo_ctl('dn', 'diskcleaner', 'register_sync_protection.%s')", string(jsonData)) + + if t.verbose { + fmt.Printf(" SQL length: %d\n", len(query)) + } + + var result string + err = t.db.QueryRow(query).Scan(&result) + if err != nil { + return fmt.Errorf("failed to register protection: %w", err) + } + + if t.verbose { + fmt.Printf(" Result: %s\n", result) + } + + // Check if successful + if strings.Contains(strings.ToLower(result), "error") { + return fmt.Errorf("register protection returned error: %s", result) + } + + t.protectedFiles = objects + return nil +} + +// RenewProtection renews protection +func (t *SyncProtectionTester) RenewProtection() error { + req := SyncProtectionRequest{ + JobID: t.jobID, + ValidTS: time.Now().UnixNano(), + } + + jsonData, err := json.Marshal(req) + if err != nil { + return fmt.Errorf("failed to marshal request: %w", err) + } + + query := fmt.Sprintf("SELECT mo_ctl('dn', 'diskcleaner', 'renew_sync_protection.%s')", string(jsonData)) + + if t.verbose { + fmt.Printf(" SQL: %s\n", query) + } + + var result string + err = t.db.QueryRow(query).Scan(&result) + if err != nil { + return fmt.Errorf("failed to renew protection: %w", err) + } + + if t.verbose { + fmt.Printf(" Result: %s\n", result) + } + + return nil +} + +// UnregisterProtection unregisters protection +func (t *SyncProtectionTester) UnregisterProtection() error { + req := SyncProtectionRequest{ + JobID: t.jobID, + } + + jsonData, err := json.Marshal(req) + if err != nil { + return fmt.Errorf("failed to marshal request: %w", err) + } + + query := fmt.Sprintf("SELECT mo_ctl('dn', 'diskcleaner', 'unregister_sync_protection.%s')", string(jsonData)) + + if t.verbose { + fmt.Printf(" SQL: %s\n", query) + } + + var result string + err = t.db.QueryRow(query).Scan(&result) + if err != nil { + return fmt.Errorf("failed to unregister protection: %w", err) + } + + if t.verbose { + fmt.Printf(" Result: %s\n", result) + } + + return nil +} + +// TriggerGC triggers GC +func (t *SyncProtectionTester) TriggerGC() error { + query := "SELECT mo_ctl('dn', 'diskcleaner', 'force_gc')" + + if t.verbose { + fmt.Printf(" SQL: %s\n", query) + } + + var result string + err := t.db.QueryRow(query).Scan(&result) + if err != nil { + return fmt.Errorf("failed to trigger GC: %w", err) + } + + if t.verbose { + fmt.Printf(" Result: %s\n", result) + } + + return nil +} + +// CheckFilesExist checks if files exist +func (t *SyncProtectionTester) CheckFilesExist() (existing, deleted []string) { + for _, file := range t.protectedFiles { + // Search for file in data directory + found := false + filepath.Walk(t.dataDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return nil + } + if info.Name() == file { + found = true + return filepath.SkipAll + } + return nil + }) + if found { + existing = append(existing, file) + } else { + deleted = append(deleted, file) + } + } + return +} + +// RunTest runs the test +func (t *SyncProtectionTester) RunTest() error { + fmt.Println("========================================") + fmt.Println("Sync Protection Test (BloomFilter)") + fmt.Println("========================================") + fmt.Printf("Job ID: %s\n", t.jobID) + fmt.Printf("Data directory: %s\n", t.dataDir) + fmt.Printf("Sample count: %d\n", t.sampleCount) + fmt.Printf("Wait time: %d seconds\n", t.waitTime) + fmt.Println() + + // Step 1: Scan object files + fmt.Println("[Step 1] Scanning object files...") + objects, err := t.ScanObjectFiles() + if err != nil { + return err + } + fmt.Printf(" Found %d object files\n", len(objects)) + + if len(objects) == 0 { + return fmt.Errorf("no object files found, please check data directory: %s", t.dataDir) + } + + // Step 2: Randomly select objects + fmt.Println("[Step 2] Randomly selecting objects...") + selected := t.SelectRandomObjects(objects, t.sampleCount) + fmt.Printf(" Selected %d objects:\n", len(selected)) + for i, obj := range selected { + if i < 5 { + fmt.Printf(" - %s\n", obj) + } else if i == 5 { + fmt.Printf(" - ... (%d more)\n", len(selected)-5) + break + } + } + + // Step 3: Build BloomFilter and register protection + fmt.Println("[Step 3] Building BloomFilter and registering sync protection...") + if err := t.RegisterProtection(selected); err != nil { + return fmt.Errorf("failed to register protection: %w", err) + } + fmt.Println(" ✓ Registration successful!") + + // Step 4: Check initial file status + fmt.Println("[Step 4] Checking initial file status...") + existingBefore, deletedBefore := t.CheckFilesExist() + fmt.Printf(" Existing: %d, Deleted: %d\n", len(existingBefore), len(deletedBefore)) + + // Step 5: Trigger GC + fmt.Println("[Step 5] Triggering GC...") + if err := t.TriggerGC(); err != nil { + fmt.Printf(" ⚠ Warning: Failed to trigger GC: %v\n", err) + } else { + fmt.Println(" ✓ GC triggered successfully!") + } + + // Wait for GC to complete + fmt.Printf("[Step 6] Waiting for GC to complete (%d seconds)...\n", t.waitTime) + time.Sleep(time.Duration(t.waitTime) * time.Second) + + // Step 7: Check file protection status + fmt.Println("[Step 7] Checking file protection status...") + existingAfter, deletedAfter := t.CheckFilesExist() + fmt.Printf(" Existing: %d, Deleted: %d\n", len(existingAfter), len(deletedAfter)) + + // Compare results + newlyDeleted := len(deletedAfter) - len(deletedBefore) + if newlyDeleted > 0 { + fmt.Printf(" ✗ [FAILED] %d protected files were deleted!\n", newlyDeleted) + for _, f := range deletedAfter { + found := false + for _, bf := range deletedBefore { + if f == bf { + found = true + break + } + } + if !found { + fmt.Printf(" - Deleted: %s\n", f) + } + } + // Validation failed, stop test + return fmt.Errorf("protection mechanism validation failed: %d protected files were deleted", newlyDeleted) + } else { + fmt.Println(" ✓ [SUCCESS] All protected files were not deleted!") + } + + // Step 8: Test renewal + fmt.Println("[Step 8] Testing renewal...") + if err := t.RenewProtection(); err != nil { + fmt.Printf(" ⚠ Warning: Renewal failed: %v\n", err) + } else { + fmt.Println(" ✓ Renewal successful!") + } + + // Step 9: Unregister protection + fmt.Println("[Step 9] Unregistering protection (soft delete)...") + if err := t.UnregisterProtection(); err != nil { + fmt.Printf(" ⚠ Warning: Unregister failed: %v\n", err) + } else { + fmt.Println(" ✓ Unregister successful!") + } + + // Step 10: Trigger GC again + fmt.Println("[Step 10] Triggering GC again...") + if err := t.TriggerGC(); err != nil { + fmt.Printf(" ⚠ Warning: Failed to trigger GC: %v\n", err) + } else { + fmt.Println(" ✓ GC triggered successfully!") + } + + // Wait for GC to complete + fmt.Printf("[Step 11] Waiting for GC to complete (%d seconds)...\n", t.waitTime) + time.Sleep(time.Duration(t.waitTime) * time.Second) + + // Step 12: Final check + fmt.Println("[Step 12] Final check...") + existingFinal, deletedFinal := t.CheckFilesExist() + fmt.Printf(" Existing: %d, Deleted: %d\n", len(existingFinal), len(deletedFinal)) + + fmt.Println() + fmt.Println("========================================") + fmt.Println("Test completed!") + fmt.Println("========================================") + + return nil +} + +// PrepareSyncProtectionCommand prepares the sync protection test command +func PrepareSyncProtectionCommand() *cobra.Command { + var ( + dsn string + dataDir string + sampleCount int + verbose bool + waitTime int + ) + + cmd := &cobra.Command{ + Use: "sync-protection", + Short: "Test sync protection mechanism", + Long: `Test cross-cluster sync protection mechanism. + +This command will: +1. Scan the specified directory for object files +2. Randomly select some objects to build a BloomFilter +3. Register BloomFilter protection +4. Trigger GC and verify protected files are not deleted +5. Test renewal and unregister functionality`, + RunE: func(cmd *cobra.Command, args []string) error { + tester, err := NewSyncProtectionTester(dsn, dataDir, sampleCount, verbose, waitTime) + if err != nil { + return err + } + defer tester.Close() + + return tester.RunTest() + }, + } + + cmd.Flags().StringVar(&dsn, "dsn", "root:111@tcp(127.0.0.1:6001)/", "Database connection string") + cmd.Flags().StringVar(&dataDir, "data-dir", "./mo-data/shared", "Data directory path") + cmd.Flags().IntVar(&sampleCount, "sample", 10, "Number of objects to sample") + cmd.Flags().BoolVar(&verbose, "verbose", false, "Show verbose output") + cmd.Flags().IntVar(&waitTime, "wait", 30, "Time to wait for GC to complete (seconds)") + + return cmd +} diff --git a/pkg/common/bloomfilter/bloomfilter.go b/pkg/common/bloomfilter/bloomfilter.go index f00e96a82bac5..5d2eab4c31339 100644 --- a/pkg/common/bloomfilter/bloomfilter.go +++ b/pkg/common/bloomfilter/bloomfilter.go @@ -183,6 +183,59 @@ func (bf *BloomFilter) TestRow(v *vector.Vector, row int) bool { return true } +// TestRowDebug tests if a single row might be in the bloom filter with debug info. +// Returns (result, encodedKey, hashVals) +func (bf *BloomFilter) TestRowDebug(v *vector.Vector, row int) (bool, []byte, []uint64) { + if row < 0 || row >= v.Length() { + return false, nil, nil + } + + bf.keys[0] = bf.keys[0][:0] + encodeHashKeys(bf.keys[:1], v, row, 1) + + // Copy the encoded key for debugging + encodedKey := make([]byte, len(bf.keys[0])) + copy(encodedKey, bf.keys[0]) + + bitSize := uint64(bf.shared.bitmap.Len()) + lastSeed := len(bf.shared.hashSeed) - 1 + + getIdxVal := func(v uint64) uint64 { + if bitSize == 0 || v < bitSize { + return v + } else { + return v % bitSize + } + } + + for k := 0; k < lastSeed; k++ { + hashtable.BytesBatchGenHashStatesWithSeed(&bf.keys[0], &bf.states[0], 1, bf.shared.hashSeed[k]) + idx := k * 3 + bf.vals[0][idx] = getIdxVal(bf.states[0][0]) + bf.vals[0][idx+1] = getIdxVal(bf.states[0][1]) + bf.vals[0][idx+2] = getIdxVal(bf.states[0][2]) + } + hashtable.BytesBatchGenHashStatesWithSeed(&bf.keys[0], &bf.states[0], 1, bf.shared.hashSeed[lastSeed]) + idx := lastSeed * 3 + bf.vals[0][idx] = getIdxVal(bf.states[0][0]) + bf.vals[0][idx+1] = getIdxVal(bf.states[0][1]) + bf.vals[0][idx+2] = getIdxVal(bf.states[0][2]) + + bf.keys[0] = bf.keys[0][:0] + + // Copy hash values for debugging + hashVals := make([]uint64, bf.valLength) + copy(hashVals, bf.vals[0][:bf.valLength]) + + vals := bf.vals[0] + for j := 0; j < bf.valLength; j++ { + if !bf.shared.bitmap.Contains(vals[j]) { + return false, encodedKey, hashVals + } + } + return true, encodedKey, hashVals +} + func (bf *BloomFilter) TestAndAdd(v *vector.Vector, callBack func(bool, int)) { bf.handle(v, func(idx, beginIdx int) { var contains bool @@ -273,6 +326,46 @@ func (bf *BloomFilter) Unmarshal(data []byte) error { return nil } +// GetBitmapLen returns the length of the bitmap for debugging +func (bf *BloomFilter) GetBitmapLen() int64 { + if bf == nil || bf.shared == nil { + return 0 + } + return bf.shared.bitmap.Len() +} + +// GetSeedCount returns the number of hash seeds for debugging +func (bf *BloomFilter) GetSeedCount() int { + if bf == nil || bf.shared == nil { + return 0 + } + return len(bf.shared.hashSeed) +} + +// GetFirstSeed returns the first hash seed for debugging +func (bf *BloomFilter) GetFirstSeed() uint64 { + if bf == nil || bf.shared == nil || len(bf.shared.hashSeed) == 0 { + return 0 + } + return bf.shared.hashSeed[0] +} + +// GetAllSeeds returns all hash seeds for debugging +func (bf *BloomFilter) GetAllSeeds() []uint64 { + if bf == nil || bf.shared == nil { + return nil + } + return bf.shared.hashSeed +} + +// GetBitmapCount returns the number of bits set in the bitmap for debugging +func (bf *BloomFilter) GetBitmapCount() int { + if bf == nil || bf.shared == nil { + return 0 + } + return bf.shared.bitmap.Count() +} + // handle computes the hash value of each element and executes the callback. func (bf *BloomFilter) handle(v *vector.Vector, callBack func(int, int)) { length := v.Length() diff --git a/pkg/sql/plan/function/ctl/cmd_disk_cleaner.go b/pkg/sql/plan/function/ctl/cmd_disk_cleaner.go index 24c65f17cfbd4..514d2b2365c65 100644 --- a/pkg/sql/plan/function/ctl/cmd_disk_cleaner.go +++ b/pkg/sql/plan/function/ctl/cmd_disk_cleaner.go @@ -18,21 +18,26 @@ import ( "github.com/fagongzi/util/protoc" "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/logutil" "github.com/matrixorigin/matrixone/pkg/pb/api" "github.com/matrixorigin/matrixone/pkg/vm/engine/cmd_util" "github.com/matrixorigin/matrixone/pkg/vm/process" + "go.uber.org/zap" "strings" ) func IsValidArg(parameter string, proc *process.Process) (*cmd_util.DiskCleaner, error) { parameters := strings.Split(parameter, ".") - if len(parameters) > 3 || len(parameters) < 1 { + if len(parameters) < 1 { return nil, moerr.NewInternalError(proc.Ctx, "handleDiskCleaner: invalid argument!") } op := parameters[0] switch op { case cmd_util.AddChecker, cmd_util.RemoveChecker: - break + // These operations need key validation, check parameter count later + if len(parameters) > 3 { + return nil, moerr.NewInternalError(proc.Ctx, "handleDiskCleaner: invalid argument!") + } case cmd_util.StopGC, cmd_util.StartGC: return &cmd_util.DiskCleaner{ Op: op, @@ -52,6 +57,29 @@ func IsValidArg(parameter string, proc *process.Process) (*cmd_util.DiskCleaner, Op: op, Key: cmd_util.GCVerify, }, nil + case cmd_util.RegisterSyncProtection, cmd_util.RenewSyncProtection, cmd_util.UnregisterSyncProtection: + // Sync protection operations expect JSON value in the second parameter + // Format: register_sync_protection.{"job_id":"xxx","objects":["obj1"],"valid_ts":123} + // Note: JSON may contain dots, so we join all remaining parts + value := "" + if len(parameters) > 1 { + // Join remaining parts as JSON value (in case JSON contains dots) + value = strings.Join(parameters[1:], ".") + } + + // Debug: print parameter info + logutil.Info( + "GC-Sync-Protection-CMD-Parse", + zap.String("op", op), + zap.Int("parameter-len", len(parameter)), + zap.Int("parts-count", len(parameters)), + zap.Int("value-len", len(value)), + ) + + return &cmd_util.DiskCleaner{ + Op: op, + Value: value, + }, nil default: return nil, moerr.NewInternalError(proc.Ctx, "handleDiskCleaner: invalid operation!") } diff --git a/pkg/vm/engine/cmd_util/operations.go b/pkg/vm/engine/cmd_util/operations.go index f6345f368a9a7..0710135490033 100644 --- a/pkg/vm/engine/cmd_util/operations.go +++ b/pkg/vm/engine/cmd_util/operations.go @@ -424,3 +424,11 @@ func (f *FaultInjectReq) MarshalBinary() ([]byte, error) { func (f *FaultInjectReq) UnmarshalBinary(data []byte) error { return f.Unmarshal(data) } + +// SyncProtection is the request for sync protection operations +type SyncProtection struct { + JobID string `json:"job_id"` // Sync job ID + BF string `json:"bf"` // Base64 encoded BloomFilter data (for register) + ValidTS int64 `json:"valid_ts"` // Valid timestamp in nanoseconds (for register and renew) + TestObject string `json:"test_object"` // Test object name for debugging (optional) +} diff --git a/pkg/vm/engine/cmd_util/operations.proto b/pkg/vm/engine/cmd_util/operations.proto index f85001fcef50a..9d8bc19fd47bc 100644 --- a/pkg/vm/engine/cmd_util/operations.proto +++ b/pkg/vm/engine/cmd_util/operations.proto @@ -200,4 +200,12 @@ message FaultInjectReq { option (gogoproto.typedecl) = false; string method = 1; string parameter = 2; +} + +message SyncProtection { + option (gogoproto.typedecl) = false; + string Op = 1; // Operation: register_sync_protection, renew_sync_protection, unregister_sync_protection + string JobID = 2; // Sync job ID + repeated string Objects = 3; // Protected object names (for register) + int64 ValidTS = 4; // Valid timestamp in nanoseconds (for register and renew) } \ No newline at end of file diff --git a/pkg/vm/engine/cmd_util/type.go b/pkg/vm/engine/cmd_util/type.go index d638abba3229c..555b164098f59 100644 --- a/pkg/vm/engine/cmd_util/type.go +++ b/pkg/vm/engine/cmd_util/type.go @@ -33,4 +33,9 @@ const ( GCDetails = "details" GCVerify = "verify" + + // Sync protection operations for cross-cluster sync + RegisterSyncProtection = "register_sync_protection" + RenewSyncProtection = "renew_sync_protection" + UnregisterSyncProtection = "unregister_sync_protection" ) diff --git a/pkg/vm/engine/tae/db/gc/v3/checkpoint.go b/pkg/vm/engine/tae/db/gc/v3/checkpoint.go index b163088cb2377..74f27b8b96534 100644 --- a/pkg/vm/engine/tae/db/gc/v3/checkpoint.go +++ b/pkg/vm/engine/tae/db/gc/v3/checkpoint.go @@ -116,6 +116,9 @@ type checkpointCleaner struct { isActive bool } + // syncProtection is the sync protection manager for cross-cluster sync + syncProtection *SyncProtectionManager + mutation struct { sync.Mutex taskState struct { @@ -220,9 +223,15 @@ func NewCheckpointCleaner( cleaner.mutation.metaFiles = make(map[string]ioutil.TSRangeFile) cleaner.mutation.snapshotMeta = logtail.NewSnapshotMeta() cleaner.backupProtection.isActive = false + cleaner.syncProtection = NewSyncProtectionManager() return cleaner } +// GetSyncProtectionManager returns the sync protection manager +func (c *checkpointCleaner) GetSyncProtectionManager() *SyncProtectionManager { + return c.syncProtection +} + func (c *checkpointCleaner) Stop() { c.mutation.Lock() defer c.mutation.Unlock() @@ -1200,6 +1209,36 @@ func (c *checkpointCleaner) tryGCAgainstGCKPLocked( extraErrMsg = "doGCAgainstGlobalCheckpointLocked failed" return } + + // Filter out files protected by sync protection before deletion + // This ensures cross-cluster sync operations don't lose their referenced objects + originalCount := len(filesToGC) + + // Debug: print first few files to be deleted + if originalCount > 0 { + logutil.Info( + "GC-Files-To-Delete-Before-Filter", + zap.Int("count", originalCount), + zap.Strings("sample-files", func() []string { + if originalCount <= 5 { + return filesToGC + } + return filesToGC[:5] + }()), + ) + } + + filesToGC = c.syncProtection.FilterProtectedFiles(filesToGC) + if originalCount != len(filesToGC) { + logutil.Info( + "GC-Sync-Protection-Filtered", + zap.String("task", c.TaskNameLocked()), + zap.Int("original-count", originalCount), + zap.Int("filtered-count", len(filesToGC)), + zap.Int("protected-count", originalCount-len(filesToGC)), + ) + } + // Delete files after doGCAgainstGlobalCheckpointLocked // TODO:Requires Physical Removal Policy // Note: Data files are GC'ed normally even when backup protection is active. @@ -1224,6 +1263,15 @@ func (c *checkpointCleaner) tryGCAgainstGCKPLocked( v2.GCCheckpointDeleteDurationHistogram.Observe(time.Since(deleteStart).Seconds()) v2.GCSnapshotDeleteDurationHistogram.Observe(time.Since(deleteStart).Seconds()) } + + // Cleanup soft-deleted sync protections when checkpoint watermark > validTS + // This ensures protections are only removed after the checkpoint has recorded the commit + gcWaterMarkEntry := c.GetGCWaterMark() + if gcWaterMarkEntry != nil { + checkpointWatermark := gcWaterMarkEntry.GetEnd().ToTimestamp().PhysicalTime + c.syncProtection.CleanupSoftDeleted(checkpointWatermark) + } + if c.GetGCWaterMark() == nil { return nil } @@ -1308,6 +1356,7 @@ func (c *checkpointCleaner) doGCAgainstGlobalCheckpointLocked( c.mutation.snapshotMeta, iscp, c.checkpointCli, + c.syncProtection, memoryBuffer, c.config.canGCCacheSize, c.config.estimateRows, @@ -1483,6 +1532,7 @@ func (c *checkpointCleaner) DoCheck(ctx context.Context) error { c.mutation.snapshotMeta, iscp, c.checkpointCli, + c.syncProtection, buffer, c.config.canGCCacheSize, c.config.estimateRows, @@ -1507,6 +1557,7 @@ func (c *checkpointCleaner) DoCheck(ctx context.Context) error { c.mutation.snapshotMeta, iscp, c.checkpointCli, + c.syncProtection, buffer, c.config.canGCCacheSize, c.config.estimateRows, @@ -1624,6 +1675,14 @@ func (c *checkpointCleaner) Process( c.StartMutationTask("gc-process") defer c.StopMutationTask() + // Set GC running state for sync protection + // This prevents new sync protections from being registered during GC + c.syncProtection.SetGCRunning(true) + defer c.syncProtection.SetGCRunning(false) + + // Cleanup expired sync protections (TTL exceeded, handles crashed sync jobs) + c.syncProtection.CleanupExpired() + // Check backup protection state and create a snapshot at the start of GC // This snapshot will be used throughout the GC process to ensure consistency c.backupProtection.Lock() diff --git a/pkg/vm/engine/tae/db/gc/v3/exec_v1.go b/pkg/vm/engine/tae/db/gc/v3/exec_v1.go index 99fd054969ecc..d2a674788af1e 100644 --- a/pkg/vm/engine/tae/db/gc/v3/exec_v1.go +++ b/pkg/vm/engine/tae/db/gc/v3/exec_v1.go @@ -65,15 +65,16 @@ type CheckpointBasedGCJob struct { coarseProbility float64 canGCCacheSize int } - sourcer engine.BaseReader - snapshotMeta *logtail.SnapshotMeta - snapshots *logtail.SnapshotInfo - iscpTables map[uint64]types.TS - pitr *logtail.PitrInfo - ts *types.TS - globalCkpLoc objectio.Location - globalCkpVer uint32 - checkpointCli checkpoint.Runner // Added to access catalog + sourcer engine.BaseReader + snapshotMeta *logtail.SnapshotMeta + snapshots *logtail.SnapshotInfo + iscpTables map[uint64]types.TS + pitr *logtail.PitrInfo + ts *types.TS + globalCkpLoc objectio.Location + globalCkpVer uint32 + checkpointCli checkpoint.Runner // Added to access catalog + syncProtection *SyncProtectionManager // Sync protection manager for cross-cluster sync result struct { vecToGC *vector.Vector @@ -91,6 +92,7 @@ func NewCheckpointBasedGCJob( iscpTables map[uint64]types.TS, snapshotMeta *logtail.SnapshotMeta, checkpointCli checkpoint.Runner, + syncProtection *SyncProtectionManager, buffer *containers.OneSchemaBatchBuffer, isOwner bool, mp *mpool.MPool, @@ -99,15 +101,16 @@ func NewCheckpointBasedGCJob( opts ...GCJobExecutorOption, ) *CheckpointBasedGCJob { e := &CheckpointBasedGCJob{ - sourcer: sourcer, - snapshotMeta: snapshotMeta, - snapshots: snapshots, - pitr: pitr, - ts: ts, - globalCkpLoc: globalCkpLoc, - globalCkpVer: gckpVersion, - iscpTables: iscpTables, - checkpointCli: checkpointCli, + sourcer: sourcer, + snapshotMeta: snapshotMeta, + snapshots: snapshots, + pitr: pitr, + ts: ts, + globalCkpLoc: globalCkpLoc, + globalCkpVer: gckpVersion, + iscpTables: iscpTables, + checkpointCli: checkpointCli, + syncProtection: syncProtection, } for _, opt := range opts { opt(e) @@ -164,6 +167,7 @@ func (e *CheckpointBasedGCJob) Execute(ctx context.Context) error { e.globalCkpVer, e.ts, &transObjects, + e.syncProtection, e.mp, e.fs, ) @@ -235,6 +239,7 @@ func MakeBloomfilterCoarseFilter( ckpVersion uint32, ts *types.TS, transObjects *map[string]map[uint64]*ObjectEntry, + syncProtection *SyncProtectionManager, mp *mpool.MPool, fs fileservice.FileService, ) ( @@ -291,10 +296,19 @@ func MakeBloomfilterCoarseFilter( if !createTS.LT(ts) || !dropTS.LT(ts) { return } - bm.Add(uint64(i)) + + // Check if the object is protected by sync protection + // If protected, skip marking it for GC so it stays in filesNotGC buf := bat.Vecs[0].GetRawBytesAt(i) stats := (objectio.ObjectStats)(buf) name := stats.ObjectName().UnsafeString() + + if syncProtection != nil && syncProtection.IsProtected(name) { + // Protected file: don't mark for GC, it will stay in filesNotGC + return + } + + bm.Add(uint64(i)) tid := tableIDs[i] if (*transObjects)[name] == nil || (*transObjects)[name][tableIDs[i]] == nil { diff --git a/pkg/vm/engine/tae/db/gc/v3/mock_cleaner.go b/pkg/vm/engine/tae/db/gc/v3/mock_cleaner.go index b5a5147d1f60e..48759fdc25cee 100644 --- a/pkg/vm/engine/tae/db/gc/v3/mock_cleaner.go +++ b/pkg/vm/engine/tae/db/gc/v3/mock_cleaner.go @@ -171,3 +171,7 @@ func (c *MockCleaner) RemoveBackupProtection() { func (c *MockCleaner) GetBackupProtection() (protectedTS types.TS, lastUpdateTime time.Time, isActive bool) { return types.TS{}, time.Time{}, false } + +func (c *MockCleaner) GetSyncProtectionManager() *SyncProtectionManager { + return nil +} diff --git a/pkg/vm/engine/tae/db/gc/v3/sync_protection.go b/pkg/vm/engine/tae/db/gc/v3/sync_protection.go new file mode 100644 index 0000000000000..03b71f0d1ad66 --- /dev/null +++ b/pkg/vm/engine/tae/db/gc/v3/sync_protection.go @@ -0,0 +1,399 @@ +// Copyright 2021 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gc + +import ( + "encoding/base64" + "fmt" + "sync" + "sync/atomic" + "time" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/logutil" + "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/index" + "go.uber.org/zap" +) + +const ( + // DefaultSyncProtectionTTL is the default TTL for sync protection + // If a protection is not renewed within this duration, it will be force cleaned + DefaultSyncProtectionTTL = 20 * time.Minute + + // DefaultMaxSyncProtections is the default maximum number of sync protections + DefaultMaxSyncProtections = 100 +) + +// SyncProtection represents a single sync protection entry +type SyncProtection struct { + JobID string // Sync job ID + BF index.BloomFilter // BloomFilter for protected objects (using xorfilter, deterministic) + ValidTS int64 // Valid timestamp (nanoseconds), needs to be renewed + SoftDelete bool // Whether soft deleted + CreateTime time.Time // Creation time for logging +} + +// SyncProtectionManager manages sync protection entries +type SyncProtectionManager struct { + sync.RWMutex + protections map[string]*SyncProtection // jobID -> protection + gcRunning atomic.Bool // Whether GC is running + ttl time.Duration // TTL for non-soft-deleted protections + maxCount int // Maximum number of protections + mp *mpool.MPool // Memory pool for vector operations +} + +// NewSyncProtectionManager creates a new SyncProtectionManager +func NewSyncProtectionManager() *SyncProtectionManager { + mp, _ := mpool.NewMPool("sync_protection", 0, mpool.NoFixed) + return &SyncProtectionManager{ + protections: make(map[string]*SyncProtection), + ttl: DefaultSyncProtectionTTL, + maxCount: DefaultMaxSyncProtections, + mp: mp, + } +} + +// SetGCRunning sets the GC running state +func (m *SyncProtectionManager) SetGCRunning(running bool) { + m.gcRunning.Store(running) + logutil.Debug( + "GC-Sync-Protection-GC-State-Changed", + zap.Bool("running", running), + ) +} + +// IsGCRunning returns whether GC is running +func (m *SyncProtectionManager) IsGCRunning() bool { + return m.gcRunning.Load() +} + +// RegisterSyncProtection registers a new sync protection with BloomFilter +// bfData is base64 encoded BloomFilter bytes (using index.BloomFilter/xorfilter format) +// Returns error if GC is running or job already exists +func (m *SyncProtectionManager) RegisterSyncProtection( + jobID string, + bfData string, + validTS int64, +) error { + m.Lock() + defer m.Unlock() + + // Check if GC is running + if m.gcRunning.Load() { + logutil.Warn( + "GC-Sync-Protection-Register-Rejected-GC-Running", + zap.String("job-id", jobID), + ) + return moerr.NewInternalErrorNoCtx("GC is running, please retry later") + } + + // Check if job already exists + if _, ok := m.protections[jobID]; ok { + logutil.Warn( + "GC-Sync-Protection-Register-Already-Exists", + zap.String("job-id", jobID), + ) + return moerr.NewInternalErrorNoCtx(fmt.Sprintf("sync protection already exists: %s", jobID)) + } + + // Check max count + if len(m.protections) >= m.maxCount { + logutil.Warn( + "GC-Sync-Protection-Register-Max-Count-Reached", + zap.String("job-id", jobID), + zap.Int("current-count", len(m.protections)), + zap.Int("max-count", m.maxCount), + ) + return moerr.NewInternalErrorNoCtx(fmt.Sprintf("sync protection max count reached: %d", m.maxCount)) + } + + // Decode base64 BloomFilter data + bfBytes, err := base64.StdEncoding.DecodeString(bfData) + if err != nil { + logutil.Error( + "GC-Sync-Protection-Register-Decode-Error", + zap.String("job-id", jobID), + zap.Error(err), + ) + return moerr.NewInternalErrorNoCtx(fmt.Sprintf("failed to decode bloom filter: %v", err)) + } + + // Unmarshal BloomFilter (using index.BloomFilter which is based on xorfilter - deterministic) + // Use recover to handle panic from invalid data + var bf index.BloomFilter + var unmarshalErr error + func() { + defer func() { + if r := recover(); r != nil { + unmarshalErr = moerr.NewInternalErrorNoCtx(fmt.Sprintf("failed to unmarshal bloom filter (panic): %v", r)) + } + }() + unmarshalErr = bf.Unmarshal(bfBytes) + }() + if unmarshalErr != nil { + logutil.Error( + "GC-Sync-Protection-Register-Unmarshal-Error", + zap.String("job-id", jobID), + zap.Error(unmarshalErr), + ) + return moerr.NewInternalErrorNoCtx(fmt.Sprintf("failed to unmarshal bloom filter: %v", unmarshalErr)) + } + + m.protections[jobID] = &SyncProtection{ + JobID: jobID, + BF: bf, + ValidTS: validTS, + SoftDelete: false, + CreateTime: time.Now(), + } + + logutil.Info( + "GC-Sync-Protection-Registered", + zap.String("job-id", jobID), + zap.Int64("valid-ts", validTS), + zap.Int("bf-size", len(bfBytes)), + zap.Int("total-protections", len(m.protections)), + ) + return nil +} + +// RenewSyncProtection renews the valid timestamp of a sync protection +func (m *SyncProtectionManager) RenewSyncProtection(jobID string, validTS int64) error { + m.Lock() + defer m.Unlock() + + p, ok := m.protections[jobID] + if !ok { + logutil.Warn( + "GC-Sync-Protection-Renew-Not-Found", + zap.String("job-id", jobID), + ) + return moerr.NewInternalErrorNoCtx(fmt.Sprintf("sync protection not found: %s", jobID)) + } + + if p.SoftDelete { + logutil.Warn( + "GC-Sync-Protection-Renew-Already-Soft-Deleted", + zap.String("job-id", jobID), + ) + return moerr.NewInternalErrorNoCtx(fmt.Sprintf("sync protection is soft deleted: %s", jobID)) + } + + oldValidTS := p.ValidTS + p.ValidTS = validTS + + logutil.Debug( + "GC-Sync-Protection-Renewed", + zap.String("job-id", jobID), + zap.Int64("old-valid-ts", oldValidTS), + zap.Int64("new-valid-ts", validTS), + ) + return nil +} + +// UnregisterSyncProtection soft deletes a sync protection +// Returns error if job not found (sync job needs to handle rollback) +func (m *SyncProtectionManager) UnregisterSyncProtection(jobID string) error { + m.Lock() + defer m.Unlock() + + p, ok := m.protections[jobID] + if !ok { + logutil.Warn( + "GC-Sync-Protection-Unregister-Not-Found", + zap.String("job-id", jobID), + ) + return moerr.NewInternalErrorNoCtx(fmt.Sprintf("sync protection not found: %s", jobID)) + } + + p.SoftDelete = true + + logutil.Info( + "GC-Sync-Protection-Soft-Deleted", + zap.String("job-id", jobID), + zap.Int64("valid-ts", p.ValidTS), + ) + return nil +} + +// CleanupSoftDeleted cleans up soft-deleted protections when checkpoint watermark > validTS +// This should be called during GC when processing checkpoints +func (m *SyncProtectionManager) CleanupSoftDeleted(checkpointWatermark int64) { + m.Lock() + defer m.Unlock() + + for jobID, p := range m.protections { + // Condition: soft delete state AND checkpoint watermark > validTS + if p.SoftDelete && checkpointWatermark > p.ValidTS { + delete(m.protections, jobID) + logutil.Info( + "GC-Sync-Protection-Cleaned-Soft-Deleted", + zap.String("job-id", jobID), + zap.Int64("valid-ts", p.ValidTS), + zap.Int64("checkpoint-watermark", checkpointWatermark), + ) + } + } +} + +// CleanupExpired cleans up expired protections (TTL exceeded and not soft deleted) +// This handles crashed sync jobs that didn't unregister +func (m *SyncProtectionManager) CleanupExpired() { + m.Lock() + defer m.Unlock() + + now := time.Now() + for jobID, p := range m.protections { + validTime := time.Unix(0, p.ValidTS) + + // Non soft delete state, but TTL exceeded without renewal + if !p.SoftDelete && now.Sub(validTime) > m.ttl { + delete(m.protections, jobID) + logutil.Warn( + "GC-Sync-Protection-Force-Cleaned-Expired", + zap.String("job-id", jobID), + zap.Int64("valid-ts", p.ValidTS), + zap.Duration("age", now.Sub(validTime)), + zap.Duration("ttl", m.ttl), + ) + } + } +} + +// GetProtectionCount returns the number of protections +func (m *SyncProtectionManager) GetProtectionCount() int { + m.RLock() + defer m.RUnlock() + return len(m.protections) +} + +// GetProtectionCountByState returns the count of protections by state +func (m *SyncProtectionManager) GetProtectionCountByState() (active, softDeleted int) { + m.RLock() + defer m.RUnlock() + + for _, p := range m.protections { + if p.SoftDelete { + softDeleted++ + } else { + active++ + } + } + return +} + +// HasProtection checks if a job has protection +func (m *SyncProtectionManager) HasProtection(jobID string) bool { + m.RLock() + defer m.RUnlock() + _, ok := m.protections[jobID] + return ok +} + +// IsProtected checks if an object name is protected by any BloomFilter +func (m *SyncProtectionManager) IsProtected(objectName string) bool { + m.RLock() + defer m.RUnlock() + + if len(m.protections) == 0 { + return false + } + + for _, p := range m.protections { + // Use MayContainsKey for single element test + if result, err := p.BF.MayContainsKey([]byte(objectName)); err == nil && result { + return true + } + } + return false +} + +// FilterProtectedFiles filters out protected files from the list +// Returns files that are NOT protected (can be deleted) +func (m *SyncProtectionManager) FilterProtectedFiles(files []string) []string { + m.RLock() + defer m.RUnlock() + + if len(m.protections) == 0 || len(files) == 0 { + return files + } + + // Collect all BloomFilters + type bfEntry struct { + jobID string + bf *index.BloomFilter + } + var bfs []bfEntry + for jobID, p := range m.protections { + bfs = append(bfs, bfEntry{jobID: jobID, bf: &p.BF}) + } + + if len(bfs) == 0 { + return files + } + + // Build result: files that are NOT protected + result := make([]string, 0, len(files)) + protectedCount := 0 + + for _, f := range files { + protected := false + + // Check against each BloomFilter + for _, entry := range bfs { + if contains, err := entry.bf.MayContainsKey([]byte(f)); err == nil && contains { + protected = true + break + } + } + + if protected { + protectedCount++ + } else { + result = append(result, f) + } + } + + if protectedCount > 0 { + logutil.Info( + "GC-Sync-Protection-Filtered", + zap.Int("total", len(files)), + zap.Int("can-delete", len(result)), + zap.Int("protected", protectedCount), + ) + } + + return result +} + +// DebugTestFile tests if a single file is protected (for debugging purposes) +func (m *SyncProtectionManager) DebugTestFile(fileName string) bool { + m.RLock() + defer m.RUnlock() + + if len(m.protections) == 0 { + return false + } + + for _, p := range m.protections { + if result, err := p.BF.MayContainsKey([]byte(fileName)); err == nil && result { + return true + } + } + + return false +} diff --git a/pkg/vm/engine/tae/db/gc/v3/sync_protection_test.go b/pkg/vm/engine/tae/db/gc/v3/sync_protection_test.go new file mode 100644 index 0000000000000..6c92765453a98 --- /dev/null +++ b/pkg/vm/engine/tae/db/gc/v3/sync_protection_test.go @@ -0,0 +1,381 @@ +// Copyright 2021 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gc + +import ( + "encoding/base64" + "testing" + "time" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/common" + "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/containers" + "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/index" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// buildTestBF creates a BloomFilter from object names and returns base64 encoded data +// Uses index.BloomFilter (xorfilter based) which is deterministic across processes +func buildTestBF(t *testing.T, objects []string) string { + // Create a containers.Vector with all object names + vec := containers.MakeVector(types.T_varchar.ToType(), common.DefaultAllocator) + defer vec.Close() + + for _, obj := range objects { + vec.Append([]byte(obj), false) + } + + // Create BloomFilter using index.NewBloomFilter (xorfilter based) + bf, err := index.NewBloomFilter(vec) + require.NoError(t, err) + + data, err := bf.Marshal() + require.NoError(t, err) + + return base64.StdEncoding.EncodeToString(data) +} + +func TestSyncProtectionManager_RegisterAndUnregister(t *testing.T) { + mgr := NewSyncProtectionManager() + + jobID := "test-job-1" + objects := []string{"object1", "object2", "object3"} + bfData := buildTestBF(t, objects) + validTS := time.Now().UnixNano() + + // Test register + err := mgr.RegisterSyncProtection(jobID, bfData, validTS) + require.NoError(t, err) + assert.Equal(t, 1, mgr.GetProtectionCount()) + assert.True(t, mgr.HasProtection(jobID)) + + // Test duplicate register + err = mgr.RegisterSyncProtection(jobID, bfData, validTS) + require.Error(t, err) + assert.Contains(t, err.Error(), "already exists") + + // Test unregister (soft delete) + err = mgr.UnregisterSyncProtection(jobID) + require.NoError(t, err) + assert.Equal(t, 1, mgr.GetProtectionCount()) // Still exists, just soft deleted + + active, softDeleted := mgr.GetProtectionCountByState() + assert.Equal(t, 0, active) + assert.Equal(t, 1, softDeleted) + + // Test unregister non-existent job + err = mgr.UnregisterSyncProtection("non-existent") + require.Error(t, err) + assert.Contains(t, err.Error(), "not found") +} + +func TestSyncProtectionManager_GCRunningBlock(t *testing.T) { + mgr := NewSyncProtectionManager() + + jobID := "test-job-1" + bfData := buildTestBF(t, []string{"object1"}) + validTS := time.Now().UnixNano() + + // Set GC running + mgr.SetGCRunning(true) + assert.True(t, mgr.IsGCRunning()) + + // Register should fail when GC is running + err := mgr.RegisterSyncProtection(jobID, bfData, validTS) + require.Error(t, err) + assert.Contains(t, err.Error(), "GC is running") + + // Set GC not running + mgr.SetGCRunning(false) + assert.False(t, mgr.IsGCRunning()) + + // Register should succeed now + err = mgr.RegisterSyncProtection(jobID, bfData, validTS) + require.NoError(t, err) +} + +func TestSyncProtectionManager_Renew(t *testing.T) { + mgr := NewSyncProtectionManager() + + jobID := "test-job-1" + bfData := buildTestBF(t, []string{"object1"}) + validTS1 := time.Now().UnixNano() + + // Register + err := mgr.RegisterSyncProtection(jobID, bfData, validTS1) + require.NoError(t, err) + + // Renew + validTS2 := time.Now().Add(time.Minute).UnixNano() + err = mgr.RenewSyncProtection(jobID, validTS2) + require.NoError(t, err) + + // Renew non-existent job + err = mgr.RenewSyncProtection("non-existent", validTS2) + require.Error(t, err) + assert.Contains(t, err.Error(), "not found") + + // Soft delete and try to renew + err = mgr.UnregisterSyncProtection(jobID) + require.NoError(t, err) + + err = mgr.RenewSyncProtection(jobID, validTS2) + require.Error(t, err) + assert.Contains(t, err.Error(), "soft deleted") +} + +func TestSyncProtectionManager_CleanupSoftDeleted(t *testing.T) { + mgr := NewSyncProtectionManager() + + // Register multiple jobs + job1 := "job-1" + job2 := "job-2" + + validTS1 := int64(1000) + validTS2 := int64(2000) + + bfData1 := buildTestBF(t, []string{"obj1"}) + bfData2 := buildTestBF(t, []string{"obj2"}) + + err := mgr.RegisterSyncProtection(job1, bfData1, validTS1) + require.NoError(t, err) + err = mgr.RegisterSyncProtection(job2, bfData2, validTS2) + require.NoError(t, err) + + // Soft delete both + err = mgr.UnregisterSyncProtection(job1) + require.NoError(t, err) + err = mgr.UnregisterSyncProtection(job2) + require.NoError(t, err) + + assert.Equal(t, 2, mgr.GetProtectionCount()) + + // Cleanup with watermark = 1500 (only job1 should be cleaned) + mgr.CleanupSoftDeleted(1500) + assert.Equal(t, 1, mgr.GetProtectionCount()) + assert.False(t, mgr.HasProtection(job1)) + assert.True(t, mgr.HasProtection(job2)) + + // Cleanup with watermark = 2500 (job2 should be cleaned) + mgr.CleanupSoftDeleted(2500) + assert.Equal(t, 0, mgr.GetProtectionCount()) +} + +func TestSyncProtectionManager_CleanupExpired(t *testing.T) { + mgr := NewSyncProtectionManager() + mgr.ttl = 100 * time.Millisecond // Short TTL for testing + + jobID := "test-job-1" + bfData := buildTestBF(t, []string{"object1"}) + + // Register with old validTS + oldValidTS := time.Now().Add(-200 * time.Millisecond).UnixNano() + err := mgr.RegisterSyncProtection(jobID, bfData, oldValidTS) + require.NoError(t, err) + + assert.Equal(t, 1, mgr.GetProtectionCount()) + + // Cleanup expired + mgr.CleanupExpired() + assert.Equal(t, 0, mgr.GetProtectionCount()) +} + +func TestSyncProtectionManager_CleanupExpired_NotSoftDeleted(t *testing.T) { + mgr := NewSyncProtectionManager() + mgr.ttl = 100 * time.Millisecond + + jobID := "test-job-1" + bfData := buildTestBF(t, []string{"object1"}) + + // Register with old validTS + oldValidTS := time.Now().Add(-200 * time.Millisecond).UnixNano() + err := mgr.RegisterSyncProtection(jobID, bfData, oldValidTS) + require.NoError(t, err) + + // Soft delete it + err = mgr.UnregisterSyncProtection(jobID) + require.NoError(t, err) + + // CleanupExpired should NOT clean soft-deleted entries + mgr.CleanupExpired() + assert.Equal(t, 1, mgr.GetProtectionCount()) // Still exists +} + +func TestSyncProtectionManager_IsProtected(t *testing.T) { + mgr := NewSyncProtectionManager() + + jobID := "job-1" + bfData := buildTestBF(t, []string{"protected-obj"}) + validTS := time.Now().UnixNano() + + err := mgr.RegisterSyncProtection(jobID, bfData, validTS) + require.NoError(t, err) + + assert.True(t, mgr.IsProtected("protected-obj")) + // Note: BloomFilter may have false positives, so we can't assert False for unprotected +} + +func TestSyncProtectionManager_FilterProtectedFiles(t *testing.T) { + mgr := NewSyncProtectionManager() + + jobID := "job-1" + bfData := buildTestBF(t, []string{"protected1", "protected2"}) + validTS := time.Now().UnixNano() + + err := mgr.RegisterSyncProtection(jobID, bfData, validTS) + require.NoError(t, err) + + files := []string{"protected1", "protected2", "unprotected1", "unprotected2"} + canDelete := mgr.FilterProtectedFiles(files) + + // Protected files should be filtered out + assert.NotContains(t, canDelete, "protected1") + assert.NotContains(t, canDelete, "protected2") + // Unprotected files should remain (unless false positive) + // Note: Due to BloomFilter false positives, we can't guarantee unprotected files are in result +} + +func TestSyncProtectionManager_FilterProtectedFiles_NoProtection(t *testing.T) { + mgr := NewSyncProtectionManager() + + files := []string{"file1", "file2", "file3"} + canDelete := mgr.FilterProtectedFiles(files) + + assert.Equal(t, files, canDelete) +} + +func TestSyncProtectionManager_MaxCount(t *testing.T) { + mgr := NewSyncProtectionManager() + mgr.maxCount = 2 // Set low max for testing + + validTS := time.Now().UnixNano() + + // Register up to max + bfData1 := buildTestBF(t, []string{"obj1"}) + bfData2 := buildTestBF(t, []string{"obj2"}) + bfData3 := buildTestBF(t, []string{"obj3"}) + + err := mgr.RegisterSyncProtection("job-1", bfData1, validTS) + require.NoError(t, err) + err = mgr.RegisterSyncProtection("job-2", bfData2, validTS) + require.NoError(t, err) + + // Should fail when max reached + err = mgr.RegisterSyncProtection("job-3", bfData3, validTS) + require.Error(t, err) + assert.Contains(t, err.Error(), "max count reached") +} + +func TestSyncProtectionManager_ConcurrentAccess(t *testing.T) { + mgr := NewSyncProtectionManager() + + // Concurrent register/unregister + done := make(chan bool) + for i := 0; i < 10; i++ { + go func(id int) { + jobID := "job-" + string(rune('0'+id)) + bfData := buildTestBF(t, []string{"obj"}) + validTS := time.Now().UnixNano() + + _ = mgr.RegisterSyncProtection(jobID, bfData, validTS) + _ = mgr.RenewSyncProtection(jobID, validTS+1000) + _ = mgr.UnregisterSyncProtection(jobID) + _ = mgr.IsProtected("obj") + done <- true + }(i) + } + + for i := 0; i < 10; i++ { + <-done + } +} + +func TestSyncProtectionManager_FullWorkflow(t *testing.T) { + mgr := NewSyncProtectionManager() + + // Simulate sync job workflow + jobID := "sync-job-123" + objects := []string{"table1/obj1", "table1/obj2", "table2/obj1"} + bfData := buildTestBF(t, objects) + validTS := time.Now().UnixNano() + + // Step 1: Check GC not running, register protection + assert.False(t, mgr.IsGCRunning()) + err := mgr.RegisterSyncProtection(jobID, bfData, validTS) + require.NoError(t, err) + + // Step 2: Simulate GC starts (should not affect existing protection) + mgr.SetGCRunning(true) + + // Step 3: GC tries to delete files - protected files should be filtered + filesToDelete := []string{"table1/obj1", "table1/obj2", "table2/obj1", "table3/obj1"} + canDelete := mgr.FilterProtectedFiles(filesToDelete) + // Protected files should be filtered out + assert.NotContains(t, canDelete, "table1/obj1") + assert.NotContains(t, canDelete, "table1/obj2") + assert.NotContains(t, canDelete, "table2/obj1") + + // Step 4: GC ends + mgr.SetGCRunning(false) + + // Step 5: Sync job completes, soft delete protection + err = mgr.UnregisterSyncProtection(jobID) + require.NoError(t, err) + + // Step 6: Next GC cleans up soft-deleted protection when watermark > validTS + checkpointWatermark := validTS + 1000000 // Watermark > validTS + mgr.CleanupSoftDeleted(checkpointWatermark) + assert.Equal(t, 0, mgr.GetProtectionCount()) +} + +func TestSyncProtectionManager_CheckpointWatermarkEdgeCase(t *testing.T) { + mgr := NewSyncProtectionManager() + + jobID := "job-1" + bfData := buildTestBF(t, []string{"obj"}) + validTS := int64(1000) + + err := mgr.RegisterSyncProtection(jobID, bfData, validTS) + require.NoError(t, err) + err = mgr.UnregisterSyncProtection(jobID) + require.NoError(t, err) + + // Watermark == validTS: should NOT be cleaned (need strictly greater) + mgr.CleanupSoftDeleted(1000) + assert.Equal(t, 1, mgr.GetProtectionCount()) + + // Watermark > validTS: should be cleaned + mgr.CleanupSoftDeleted(1001) + assert.Equal(t, 0, mgr.GetProtectionCount()) +} + +func TestSyncProtectionManager_InvalidBFData(t *testing.T) { + mgr := NewSyncProtectionManager() + + jobID := "job-1" + validTS := time.Now().UnixNano() + + // Test invalid base64 + err := mgr.RegisterSyncProtection(jobID, "invalid-base64!!!", validTS) + require.Error(t, err) + assert.Contains(t, err.Error(), "decode") + + // Test invalid BloomFilter data + invalidBF := base64.StdEncoding.EncodeToString([]byte("not a bloom filter")) + err = mgr.RegisterSyncProtection(jobID, invalidBF, validTS) + require.Error(t, err) + assert.Contains(t, err.Error(), "unmarshal") +} diff --git a/pkg/vm/engine/tae/db/gc/v3/types.go b/pkg/vm/engine/tae/db/gc/v3/types.go index da23e712dec3e..7f0df8fd22f5c 100644 --- a/pkg/vm/engine/tae/db/gc/v3/types.go +++ b/pkg/vm/engine/tae/db/gc/v3/types.go @@ -159,6 +159,9 @@ type Cleaner interface { RemoveBackupProtection() GetBackupProtection() (protectedTS types.TS, lastUpdateTime time.Time, isActive bool) + // Sync protection methods (for cross-cluster sync) + GetSyncProtectionManager() *SyncProtectionManager + // For testing GetTablePK(tableId uint64) string } diff --git a/pkg/vm/engine/tae/db/gc/v3/window.go b/pkg/vm/engine/tae/db/gc/v3/window.go index d9303844283be..923dc4fe036a3 100644 --- a/pkg/vm/engine/tae/db/gc/v3/window.go +++ b/pkg/vm/engine/tae/db/gc/v3/window.go @@ -126,6 +126,7 @@ func (w *GCWindow) ExecuteGlobalCheckpointBasedGC( snapshotMeta *logtail.SnapshotMeta, iscpTables map[uint64]types.TS, checkpointCli checkpoint.Runner, + syncProtection *SyncProtectionManager, buffer *containers.OneSchemaBatchBuffer, cacheSize int, estimateRows int, @@ -149,6 +150,7 @@ func (w *GCWindow) ExecuteGlobalCheckpointBasedGC( iscpTables, snapshotMeta, checkpointCli, + syncProtection, buffer, false, mp, diff --git a/pkg/vm/engine/tae/rpc/handle_debug.go b/pkg/vm/engine/tae/rpc/handle_debug.go index 93975d4a1d438..306298265a311 100644 --- a/pkg/vm/engine/tae/rpc/handle_debug.go +++ b/pkg/vm/engine/tae/rpc/handle_debug.go @@ -17,6 +17,7 @@ package rpc import ( "bytes" "context" + "encoding/json" "fmt" "github.com/matrixorigin/matrixone/pkg/clusterservice" "github.com/matrixorigin/matrixone/pkg/pb/metadata" @@ -793,6 +794,92 @@ func (h *Handle) HandleDiskCleaner( case cmd_util.GCVerify: resp.ReturnStr = h.db.DiskCleaner.Verify(ctx) return + case cmd_util.RegisterSyncProtection: + // Register sync protection for cross-cluster sync + // value format: JSON {"job_id": "xxx", "bf": "base64_encoded_bloomfilter", "valid_ts": 1234567890} + if value == "" { + return nil, moerr.NewInvalidArgNoCtx(op, "empty value") + } + + // Debug: print received value info + logutil.Info( + "GC-Sync-Protection-Register-Value-Received", + zap.Int("value-len", len(value)), + zap.String("value-prefix", func() string { + if len(value) > 200 { + return value[:200] + "..." + } + return value + }()), + ) + + var req cmd_util.SyncProtection + if err = json.Unmarshal([]byte(value), &req); err != nil { + logutil.Error( + "GC-Sync-Protection-Register-Parse-Error", + zap.String("value", value), + zap.Error(err), + ) + return nil, moerr.NewInvalidArgNoCtx(op, value) + } + + // Debug: print parsed request info + logutil.Info( + "GC-Sync-Protection-Register-Parsed", + zap.String("job-id", req.JobID), + zap.Int("bf-len", len(req.BF)), + zap.Int64("valid-ts", req.ValidTS), + zap.String("test-object", req.TestObject), + ) + + syncMgr := h.db.DiskCleaner.GetCleaner().GetSyncProtectionManager() + if err = syncMgr.RegisterSyncProtection(req.JobID, req.BF, req.ValidTS); err != nil { + return nil, err + } + resp.ReturnStr = `{"status": "ok"}` + return + case cmd_util.RenewSyncProtection: + // Renew sync protection valid timestamp + // value format: JSON {"job_id": "xxx", "valid_ts": 1234567890} + if value == "" { + return nil, moerr.NewInvalidArgNoCtx(op, "empty value") + } + var req cmd_util.SyncProtection + if err = json.Unmarshal([]byte(value), &req); err != nil { + logutil.Error( + "GC-Sync-Protection-Renew-Parse-Error", + zap.String("value", value), + zap.Error(err), + ) + return nil, moerr.NewInvalidArgNoCtx(op, value) + } + syncMgr := h.db.DiskCleaner.GetCleaner().GetSyncProtectionManager() + if err = syncMgr.RenewSyncProtection(req.JobID, req.ValidTS); err != nil { + return nil, err + } + resp.ReturnStr = `{"status": "ok"}` + return + case cmd_util.UnregisterSyncProtection: + // Unregister (soft delete) sync protection + // value format: JSON {"job_id": "xxx"} + if value == "" { + return nil, moerr.NewInvalidArgNoCtx(op, "empty value") + } + var req cmd_util.SyncProtection + if err = json.Unmarshal([]byte(value), &req); err != nil { + logutil.Error( + "GC-Sync-Protection-Unregister-Parse-Error", + zap.String("value", value), + zap.Error(err), + ) + return nil, moerr.NewInvalidArgNoCtx(op, value) + } + syncMgr := h.db.DiskCleaner.GetCleaner().GetSyncProtectionManager() + if err = syncMgr.UnregisterSyncProtection(req.JobID); err != nil { + return nil, err + } + resp.ReturnStr = `{"status": "ok"}` + return case cmd_util.AddChecker: break } From 75ec851294c9c595b9e771fe90c7a07726253d8b Mon Sep 17 00:00:00 2001 From: GreatRiver Date: Wed, 4 Feb 2026 11:32:49 +0800 Subject: [PATCH 02/18] remove mo-tool and add gc-tool --- .../tae/db/gc/v3/sync_protection_test.go | 127 ++++++++++++++++++ .../vm/engine/tae/db/gc/v3/tool}/main.go | 16 +-- .../tae/db/gc/v3/tool}/sync_protection.go | 16 +++ 3 files changed, 146 insertions(+), 13 deletions(-) rename {cmd/mo-tool => pkg/vm/engine/tae/db/gc/v3/tool}/main.go (56%) rename {cmd/mo-tool => pkg/vm/engine/tae/db/gc/v3/tool}/sync_protection.go (96%) diff --git a/pkg/vm/engine/tae/db/gc/v3/sync_protection_test.go b/pkg/vm/engine/tae/db/gc/v3/sync_protection_test.go index 6c92765453a98..c1ced04eeb039 100644 --- a/pkg/vm/engine/tae/db/gc/v3/sync_protection_test.go +++ b/pkg/vm/engine/tae/db/gc/v3/sync_protection_test.go @@ -379,3 +379,130 @@ func TestSyncProtectionManager_InvalidBFData(t *testing.T) { require.Error(t, err) assert.Contains(t, err.Error(), "unmarshal") } + + +func TestSyncProtectionManager_DebugTestFile(t *testing.T) { + mgr := NewSyncProtectionManager() + + // Test with no protections + assert.False(t, mgr.DebugTestFile("any-file")) + + // Register protection + jobID := "job-1" + bfData := buildTestBF(t, []string{"protected-file"}) + validTS := time.Now().UnixNano() + + err := mgr.RegisterSyncProtection(jobID, bfData, validTS) + require.NoError(t, err) + + // Test protected file + assert.True(t, mgr.DebugTestFile("protected-file")) + + // Test unprotected file (may have false positives due to BloomFilter) + // We can't assert False here due to BloomFilter characteristics +} + +func TestSyncProtectionManager_FilterProtectedFiles_EmptyFiles(t *testing.T) { + mgr := NewSyncProtectionManager() + + jobID := "job-1" + bfData := buildTestBF(t, []string{"protected1"}) + validTS := time.Now().UnixNano() + + err := mgr.RegisterSyncProtection(jobID, bfData, validTS) + require.NoError(t, err) + + // Test with empty files list + result := mgr.FilterProtectedFiles([]string{}) + assert.Empty(t, result) +} + +func TestSyncProtectionManager_MultipleProtections(t *testing.T) { + mgr := NewSyncProtectionManager() + + // Register multiple protections + bfData1 := buildTestBF(t, []string{"obj1", "obj2"}) + bfData2 := buildTestBF(t, []string{"obj3", "obj4"}) + + validTS := time.Now().UnixNano() + + err := mgr.RegisterSyncProtection("job-1", bfData1, validTS) + require.NoError(t, err) + err = mgr.RegisterSyncProtection("job-2", bfData2, validTS) + require.NoError(t, err) + + // All objects should be protected + assert.True(t, mgr.IsProtected("obj1")) + assert.True(t, mgr.IsProtected("obj2")) + assert.True(t, mgr.IsProtected("obj3")) + assert.True(t, mgr.IsProtected("obj4")) + + // Filter should protect all + files := []string{"obj1", "obj2", "obj3", "obj4", "obj5"} + canDelete := mgr.FilterProtectedFiles(files) + assert.NotContains(t, canDelete, "obj1") + assert.NotContains(t, canDelete, "obj2") + assert.NotContains(t, canDelete, "obj3") + assert.NotContains(t, canDelete, "obj4") +} + +func TestSyncProtectionManager_IsProtected_NoProtections(t *testing.T) { + mgr := NewSyncProtectionManager() + + // No protections registered + assert.False(t, mgr.IsProtected("any-file")) +} + +func TestSyncProtectionManager_SetGCRunning(t *testing.T) { + mgr := NewSyncProtectionManager() + + // Initial state + assert.False(t, mgr.IsGCRunning()) + + // Set running + mgr.SetGCRunning(true) + assert.True(t, mgr.IsGCRunning()) + + // Set not running + mgr.SetGCRunning(false) + assert.False(t, mgr.IsGCRunning()) +} + +func TestSyncProtectionManager_GetProtectionCountByState_Empty(t *testing.T) { + mgr := NewSyncProtectionManager() + + active, softDeleted := mgr.GetProtectionCountByState() + assert.Equal(t, 0, active) + assert.Equal(t, 0, softDeleted) +} + +func TestSyncProtectionManager_CleanupExpired_NoExpired(t *testing.T) { + mgr := NewSyncProtectionManager() + mgr.ttl = time.Hour // Long TTL + + jobID := "test-job-1" + bfData := buildTestBF(t, []string{"object1"}) + validTS := time.Now().UnixNano() + + err := mgr.RegisterSyncProtection(jobID, bfData, validTS) + require.NoError(t, err) + + // Cleanup should not remove anything + mgr.CleanupExpired() + assert.Equal(t, 1, mgr.GetProtectionCount()) +} + +func TestSyncProtectionManager_CleanupSoftDeleted_NoSoftDeleted(t *testing.T) { + mgr := NewSyncProtectionManager() + + jobID := "job-1" + bfData := buildTestBF(t, []string{"obj"}) + validTS := int64(1000) + + err := mgr.RegisterSyncProtection(jobID, bfData, validTS) + require.NoError(t, err) + + // Not soft deleted, cleanup should not remove + mgr.CleanupSoftDeleted(2000) + assert.Equal(t, 1, mgr.GetProtectionCount()) +} diff --git a/cmd/mo-tool/main.go b/pkg/vm/engine/tae/db/gc/v3/tool/main.go similarity index 56% rename from cmd/mo-tool/main.go rename to pkg/vm/engine/tae/db/gc/v3/tool/main.go index a5eaa4663e902..7a58204406370 100644 --- a/cmd/mo-tool/main.go +++ b/pkg/vm/engine/tae/db/gc/v3/tool/main.go @@ -17,26 +17,16 @@ package main import ( "os" - dashboard "github.com/matrixorigin/matrixone/cmd/mo-dashboard" - debug "github.com/matrixorigin/matrixone/cmd/mo-debug" - inspect "github.com/matrixorigin/matrixone/cmd/mo-inspect" - object "github.com/matrixorigin/matrixone/cmd/mo-object-tool" - ckp "github.com/matrixorigin/matrixone/cmd/mo-object-tool/ckp" "github.com/spf13/cobra" ) func main() { var rootCmd = &cobra.Command{ - Use: "mo-tool", - Short: "Mo tool", - Long: "Mo tool is a multifunctional development tool", + Use: "gc-tool", + Short: "GC testing tool", + Long: "GC testing tool for MatrixOne garbage collection", } - rootCmd.AddCommand(debug.PrepareCommand()) - rootCmd.AddCommand(inspect.PrepareCommand()) - rootCmd.AddCommand(dashboard.PrepareCommand()) - rootCmd.AddCommand(object.PrepareCommand()) - rootCmd.AddCommand(ckp.PrepareCommand()) rootCmd.AddCommand(PrepareSyncProtectionCommand()) if err := rootCmd.Execute(); err != nil { diff --git a/cmd/mo-tool/sync_protection.go b/pkg/vm/engine/tae/db/gc/v3/tool/sync_protection.go similarity index 96% rename from cmd/mo-tool/sync_protection.go rename to pkg/vm/engine/tae/db/gc/v3/tool/sync_protection.go index 9715a40c9a6dd..bed9fe9f6cd8d 100644 --- a/cmd/mo-tool/sync_protection.go +++ b/pkg/vm/engine/tae/db/gc/v3/tool/sync_protection.go @@ -326,6 +326,20 @@ func (t *SyncProtectionTester) RunTest() error { fmt.Printf("Wait time: %d seconds\n", t.waitTime) fmt.Println() + // Ensure cleanup on exit + registered := false + defer func() { + if registered { + fmt.Println("[Cleanup] Ensuring protection is unregistered...") + if err := t.UnregisterProtection(); err != nil { + // Ignore error if already unregistered + fmt.Printf(" (Already unregistered or error: %v)\n", err) + } else { + fmt.Println(" ✓ Cleanup successful!") + } + } + }() + // Step 1: Scan object files fmt.Println("[Step 1] Scanning object files...") objects, err := t.ScanObjectFiles() @@ -356,6 +370,7 @@ func (t *SyncProtectionTester) RunTest() error { if err := t.RegisterProtection(selected); err != nil { return fmt.Errorf("failed to register protection: %w", err) } + registered = true fmt.Println(" ✓ Registration successful!") // Step 4: Check initial file status @@ -415,6 +430,7 @@ func (t *SyncProtectionTester) RunTest() error { if err := t.UnregisterProtection(); err != nil { fmt.Printf(" ⚠ Warning: Unregister failed: %v\n", err) } else { + registered = false // Mark as unregistered so defer won't try again fmt.Println(" ✓ Unregister successful!") } From c00be97be62d56565b9b7b51537466263ebdf95e Mon Sep 17 00:00:00 2001 From: GreatRiver Date: Wed, 4 Feb 2026 11:34:40 +0800 Subject: [PATCH 03/18] add bvt --- .../mo_ctl/mo_ctl_sync_protection.result | 37 +++++++++++++++++++ .../mo_ctl/mo_ctl_sync_protection.test | 30 +++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 test/distributed/cases/function/mo_ctl/mo_ctl_sync_protection.result create mode 100644 test/distributed/cases/function/mo_ctl/mo_ctl_sync_protection.test diff --git a/test/distributed/cases/function/mo_ctl/mo_ctl_sync_protection.result b/test/distributed/cases/function/mo_ctl/mo_ctl_sync_protection.result new file mode 100644 index 0000000000000..1eb6ec1d6bc92 --- /dev/null +++ b/test/distributed/cases/function/mo_ctl/mo_ctl_sync_protection.result @@ -0,0 +1,37 @@ +-- Test sync protection mo_ctl commands +-- This test verifies the sync protection mechanism for cross-cluster sync + +-- Test 1: Register sync protection with invalid JSON (should fail gracefully) +select mo_ctl('dn', 'diskcleaner', 'register_sync_protection.invalid_json'); +mo_ctl(dn, diskcleaner, register_sync_protection.invalid_json) +invalid sync protection request: invalid character 'i' looking for beginning of value + +-- Test 2: Register sync protection with missing fields (should fail) +select mo_ctl('dn', 'diskcleaner', 'register_sync_protection.{"job_id":"test-job-1"}'); +mo_ctl(dn, diskcleaner, register_sync_protection.{"job_id":"test-job-1"}) +invalid sync protection request: bf is empty + +-- Test 3: Register sync protection with invalid base64 BF (should fail) +select mo_ctl('dn', 'diskcleaner', 'register_sync_protection.{"job_id":"test-job-2","bf":"invalid!!!","valid_ts":1234567890}'); +mo_ctl(dn, diskcleaner, register_sync_protection.{"job_id":"test-job-2","bf":"invalid!!!","valid_ts":1234567890}) +register sync protection failed: internal error: failed to decode bloom filter: illegal base64 data at input byte 7 + +-- Test 4: Renew non-existent protection (should fail) +select mo_ctl('dn', 'diskcleaner', 'renew_sync_protection.{"job_id":"non-existent","valid_ts":1234567890}'); +mo_ctl(dn, diskcleaner, renew_sync_protection.{"job_id":"non-existent","valid_ts":1234567890}) +renew sync protection failed: internal error: sync protection not found: non-existent + +-- Test 5: Unregister non-existent protection (should fail) +select mo_ctl('dn', 'diskcleaner', 'unregister_sync_protection.{"job_id":"non-existent"}'); +mo_ctl(dn, diskcleaner, unregister_sync_protection.{"job_id":"non-existent"}) +unregister sync protection failed: internal error: sync protection not found: non-existent + +-- Test 6: Test with empty command +select mo_ctl('dn', 'diskcleaner', 'register_sync_protection.'); +mo_ctl(dn, diskcleaner, register_sync_protection.) +invalid sync protection request: unexpected end of JSON input + +-- Test 7: Test unknown sync protection command +select mo_ctl('dn', 'diskcleaner', 'unknown_sync_protection.{}'); +mo_ctl(dn, diskcleaner, unknown_sync_protection.{}) +unknown command: unknown_sync_protection diff --git a/test/distributed/cases/function/mo_ctl/mo_ctl_sync_protection.test b/test/distributed/cases/function/mo_ctl/mo_ctl_sync_protection.test new file mode 100644 index 0000000000000..1a6aa65fc6581 --- /dev/null +++ b/test/distributed/cases/function/mo_ctl/mo_ctl_sync_protection.test @@ -0,0 +1,30 @@ +-- Test sync protection mo_ctl commands +-- This test verifies the sync protection mechanism for cross-cluster sync + +-- Test 1: Register sync protection with invalid JSON (should fail gracefully) +-- @ignore:0 +select mo_ctl('dn', 'diskcleaner', 'register_sync_protection.invalid_json'); + +-- Test 2: Register sync protection with missing fields (should fail) +-- @ignore:0 +select mo_ctl('dn', 'diskcleaner', 'register_sync_protection.{"job_id":"test-job-1"}'); + +-- Test 3: Register sync protection with invalid base64 BF (should fail) +-- @ignore:0 +select mo_ctl('dn', 'diskcleaner', 'register_sync_protection.{"job_id":"test-job-2","bf":"invalid!!!","valid_ts":1234567890}'); + +-- Test 4: Renew non-existent protection (should fail) +-- @ignore:0 +select mo_ctl('dn', 'diskcleaner', 'renew_sync_protection.{"job_id":"non-existent","valid_ts":1234567890}'); + +-- Test 5: Unregister non-existent protection (should fail) +-- @ignore:0 +select mo_ctl('dn', 'diskcleaner', 'unregister_sync_protection.{"job_id":"non-existent"}'); + +-- Test 6: Test with empty command +-- @ignore:0 +select mo_ctl('dn', 'diskcleaner', 'register_sync_protection.'); + +-- Test 7: Test unknown sync protection command +-- @ignore:0 +select mo_ctl('dn', 'diskcleaner', 'unknown_sync_protection.{}'); From b6e3a246c95694dbdd77fce1ac5b24fc4cab8c65 Mon Sep 17 00:00:00 2001 From: GreatRiver Date: Wed, 4 Feb 2026 11:45:32 +0800 Subject: [PATCH 04/18] U --- pkg/vm/engine/tae/db/gc/v3/sync_protection.go | 9 ++++++ .../tae/db/gc/v3/sync_protection_test.go | 7 ++++- .../mo_ctl/mo_ctl_sync_protection.result | 30 +++++-------------- 3 files changed, 22 insertions(+), 24 deletions(-) diff --git a/pkg/vm/engine/tae/db/gc/v3/sync_protection.go b/pkg/vm/engine/tae/db/gc/v3/sync_protection.go index 03b71f0d1ad66..8a048360d1397 100644 --- a/pkg/vm/engine/tae/db/gc/v3/sync_protection.go +++ b/pkg/vm/engine/tae/db/gc/v3/sync_protection.go @@ -121,6 +121,15 @@ func (m *SyncProtectionManager) RegisterSyncProtection( return moerr.NewInternalErrorNoCtx(fmt.Sprintf("sync protection max count reached: %d", m.maxCount)) } + // Check if BF data is empty + if bfData == "" { + logutil.Error( + "GC-Sync-Protection-Register-Empty-BF", + zap.String("job-id", jobID), + ) + return moerr.NewInternalErrorNoCtx("bf is empty") + } + // Decode base64 BloomFilter data bfBytes, err := base64.StdEncoding.DecodeString(bfData) if err != nil { diff --git a/pkg/vm/engine/tae/db/gc/v3/sync_protection_test.go b/pkg/vm/engine/tae/db/gc/v3/sync_protection_test.go index c1ced04eeb039..1596d4b305ab2 100644 --- a/pkg/vm/engine/tae/db/gc/v3/sync_protection_test.go +++ b/pkg/vm/engine/tae/db/gc/v3/sync_protection_test.go @@ -368,8 +368,13 @@ func TestSyncProtectionManager_InvalidBFData(t *testing.T) { jobID := "job-1" validTS := time.Now().UnixNano() + // Test empty BF + err := mgr.RegisterSyncProtection(jobID, "", validTS) + require.Error(t, err) + assert.Contains(t, err.Error(), "bf is empty") + // Test invalid base64 - err := mgr.RegisterSyncProtection(jobID, "invalid-base64!!!", validTS) + err = mgr.RegisterSyncProtection(jobID, "invalid-base64!!!", validTS) require.Error(t, err) assert.Contains(t, err.Error(), "decode") diff --git a/test/distributed/cases/function/mo_ctl/mo_ctl_sync_protection.result b/test/distributed/cases/function/mo_ctl/mo_ctl_sync_protection.result index 1eb6ec1d6bc92..a11298ce146b2 100644 --- a/test/distributed/cases/function/mo_ctl/mo_ctl_sync_protection.result +++ b/test/distributed/cases/function/mo_ctl/mo_ctl_sync_protection.result @@ -1,37 +1,21 @@ --- Test sync protection mo_ctl commands --- This test verifies the sync protection mechanism for cross-cluster sync - --- Test 1: Register sync protection with invalid JSON (should fail gracefully) select mo_ctl('dn', 'diskcleaner', 'register_sync_protection.invalid_json'); mo_ctl(dn, diskcleaner, register_sync_protection.invalid_json) -invalid sync protection request: invalid character 'i' looking for beginning of value - --- Test 2: Register sync protection with missing fields (should fail) +invalid argument register_sync_protection, bad value invalid_json select mo_ctl('dn', 'diskcleaner', 'register_sync_protection.{"job_id":"test-job-1"}'); mo_ctl(dn, diskcleaner, register_sync_protection.{"job_id":"test-job-1"}) -invalid sync protection request: bf is empty - --- Test 3: Register sync protection with invalid base64 BF (should fail) +internal error: bf is empty select mo_ctl('dn', 'diskcleaner', 'register_sync_protection.{"job_id":"test-job-2","bf":"invalid!!!","valid_ts":1234567890}'); mo_ctl(dn, diskcleaner, register_sync_protection.{"job_id":"test-job-2","bf":"invalid!!!","valid_ts":1234567890}) -register sync protection failed: internal error: failed to decode bloom filter: illegal base64 data at input byte 7 - --- Test 4: Renew non-existent protection (should fail) +internal error: failed to decode bloom filter: illegal base64 data at input byte 7 select mo_ctl('dn', 'diskcleaner', 'renew_sync_protection.{"job_id":"non-existent","valid_ts":1234567890}'); mo_ctl(dn, diskcleaner, renew_sync_protection.{"job_id":"non-existent","valid_ts":1234567890}) -renew sync protection failed: internal error: sync protection not found: non-existent - --- Test 5: Unregister non-existent protection (should fail) +internal error: sync protection not found: non-existent select mo_ctl('dn', 'diskcleaner', 'unregister_sync_protection.{"job_id":"non-existent"}'); mo_ctl(dn, diskcleaner, unregister_sync_protection.{"job_id":"non-existent"}) -unregister sync protection failed: internal error: sync protection not found: non-existent - --- Test 6: Test with empty command +internal error: sync protection not found: non-existent select mo_ctl('dn', 'diskcleaner', 'register_sync_protection.'); mo_ctl(dn, diskcleaner, register_sync_protection.) -invalid sync protection request: unexpected end of JSON input - --- Test 7: Test unknown sync protection command +invalid argument register_sync_protection, bad value select mo_ctl('dn', 'diskcleaner', 'unknown_sync_protection.{}'); mo_ctl(dn, diskcleaner, unknown_sync_protection.{}) -unknown command: unknown_sync_protection +internal error: handleDiskCleaner: invalid operation! From c7256b33ff5477b1e021527eee00386fce065e77 Mon Sep 17 00:00:00 2001 From: GreatRiver Date: Wed, 4 Feb 2026 11:50:31 +0800 Subject: [PATCH 05/18] U --- .../cases/function/mo_ctl/mo_ctl_sync_protection.result | 2 +- .../cases/function/mo_ctl/mo_ctl_sync_protection.test | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/test/distributed/cases/function/mo_ctl/mo_ctl_sync_protection.result b/test/distributed/cases/function/mo_ctl/mo_ctl_sync_protection.result index a11298ce146b2..672a78bae1533 100644 --- a/test/distributed/cases/function/mo_ctl/mo_ctl_sync_protection.result +++ b/test/distributed/cases/function/mo_ctl/mo_ctl_sync_protection.result @@ -15,7 +15,7 @@ mo_ctl(dn, diskcleaner, unregister_sync_protection.{"job_id":"non-existent"}) internal error: sync protection not found: non-existent select mo_ctl('dn', 'diskcleaner', 'register_sync_protection.'); mo_ctl(dn, diskcleaner, register_sync_protection.) -invalid argument register_sync_protection, bad value +invalid argument register_sync_protection, bad value empty value select mo_ctl('dn', 'diskcleaner', 'unknown_sync_protection.{}'); mo_ctl(dn, diskcleaner, unknown_sync_protection.{}) internal error: handleDiskCleaner: invalid operation! diff --git a/test/distributed/cases/function/mo_ctl/mo_ctl_sync_protection.test b/test/distributed/cases/function/mo_ctl/mo_ctl_sync_protection.test index 1a6aa65fc6581..5838e459cfd07 100644 --- a/test/distributed/cases/function/mo_ctl/mo_ctl_sync_protection.test +++ b/test/distributed/cases/function/mo_ctl/mo_ctl_sync_protection.test @@ -2,29 +2,22 @@ -- This test verifies the sync protection mechanism for cross-cluster sync -- Test 1: Register sync protection with invalid JSON (should fail gracefully) --- @ignore:0 select mo_ctl('dn', 'diskcleaner', 'register_sync_protection.invalid_json'); -- Test 2: Register sync protection with missing fields (should fail) --- @ignore:0 select mo_ctl('dn', 'diskcleaner', 'register_sync_protection.{"job_id":"test-job-1"}'); -- Test 3: Register sync protection with invalid base64 BF (should fail) --- @ignore:0 select mo_ctl('dn', 'diskcleaner', 'register_sync_protection.{"job_id":"test-job-2","bf":"invalid!!!","valid_ts":1234567890}'); -- Test 4: Renew non-existent protection (should fail) --- @ignore:0 select mo_ctl('dn', 'diskcleaner', 'renew_sync_protection.{"job_id":"non-existent","valid_ts":1234567890}'); -- Test 5: Unregister non-existent protection (should fail) --- @ignore:0 select mo_ctl('dn', 'diskcleaner', 'unregister_sync_protection.{"job_id":"non-existent"}'); -- Test 6: Test with empty command --- @ignore:0 select mo_ctl('dn', 'diskcleaner', 'register_sync_protection.'); -- Test 7: Test unknown sync protection command --- @ignore:0 select mo_ctl('dn', 'diskcleaner', 'unknown_sync_protection.{}'); From 240026837631c19c5ad9f5b5574528709359beea Mon Sep 17 00:00:00 2001 From: GreatRiver Date: Wed, 4 Feb 2026 12:05:55 +0800 Subject: [PATCH 06/18] update --- pkg/common/moerr/error.go | 16 ++++++++++++ pkg/common/moerr/error_no_ctx.go | 25 ++++++++++++++++++ pkg/vm/engine/tae/db/gc/v3/sync_protection.go | 26 +++++++++++-------- .../tae/db/gc/v3/sync_protection_test.go | 6 ++--- .../mo_ctl/mo_ctl_sync_protection.result | 7 ----- 5 files changed, 59 insertions(+), 21 deletions(-) diff --git a/pkg/common/moerr/error.go b/pkg/common/moerr/error.go index 1d1ee693f702b..bb866b88ae4d2 100644 --- a/pkg/common/moerr/error.go +++ b/pkg/common/moerr/error.go @@ -249,6 +249,14 @@ const ( ErrTxnControl uint16 = 20639 ErrOfflineTxnWrite uint16 = 20640 + // GC sync protection errors + ErrGCIsRunning uint16 = 20641 + ErrSyncProtectionNotFound uint16 = 20642 + ErrSyncProtectionExists uint16 = 20643 + ErrSyncProtectionMaxCount uint16 = 20644 + ErrSyncProtectionSoftDelete uint16 = 20645 + ErrSyncProtectionInvalid uint16 = 20646 + // Group 7: lock service // ErrDeadLockDetected lockservice has detected a deadlock and should abort the transaction if it receives this error ErrDeadLockDetected uint16 = 20701 @@ -502,6 +510,14 @@ var errorMsgRefer = map[uint16]moErrorMsgItem{ ErrTxnControl: {ER_UNKNOWN_ERROR, []string{MySQLDefaultSqlState}, "txn control error: %s"}, ErrOfflineTxnWrite: {ER_UNKNOWN_ERROR, []string{MySQLDefaultSqlState}, "write offline txn: %s"}, + // GC sync protection errors + ErrGCIsRunning: {ER_UNKNOWN_ERROR, []string{MySQLDefaultSqlState}, "GC is running, please retry later"}, + ErrSyncProtectionNotFound: {ER_UNKNOWN_ERROR, []string{MySQLDefaultSqlState}, "sync protection not found: %s"}, + ErrSyncProtectionExists: {ER_UNKNOWN_ERROR, []string{MySQLDefaultSqlState}, "sync protection already exists: %s"}, + ErrSyncProtectionMaxCount: {ER_UNKNOWN_ERROR, []string{MySQLDefaultSqlState}, "sync protection max count reached: %d"}, + ErrSyncProtectionSoftDelete: {ER_UNKNOWN_ERROR, []string{MySQLDefaultSqlState}, "sync protection is soft deleted: %s"}, + ErrSyncProtectionInvalid: {ER_UNKNOWN_ERROR, []string{MySQLDefaultSqlState}, "invalid sync protection request"}, + // Group 7: lock service ErrDeadLockDetected: {ER_UNKNOWN_ERROR, []string{MySQLDefaultSqlState}, "deadlock detected"}, ErrLockTableBindChanged: {ER_UNKNOWN_ERROR, []string{MySQLDefaultSqlState}, "lock table bind changed"}, diff --git a/pkg/common/moerr/error_no_ctx.go b/pkg/common/moerr/error_no_ctx.go index 8d130e221f270..d63fa284e1de6 100644 --- a/pkg/common/moerr/error_no_ctx.go +++ b/pkg/common/moerr/error_no_ctx.go @@ -495,3 +495,28 @@ func NewReplicaNotMatch(current, received string) *Error { func NewCantCompileForPrepareNoCtx() *Error { return newError(Context(), ErrCantCompileForPrepare) } + +// GC sync protection errors +func NewGCIsRunningNoCtx() *Error { + return newError(Context(), ErrGCIsRunning) +} + +func NewSyncProtectionNotFoundNoCtx(jobID string) *Error { + return newError(Context(), ErrSyncProtectionNotFound, jobID) +} + +func NewSyncProtectionExistsNoCtx(jobID string) *Error { + return newError(Context(), ErrSyncProtectionExists, jobID) +} + +func NewSyncProtectionMaxCountNoCtx(maxCount int) *Error { + return newError(Context(), ErrSyncProtectionMaxCount, maxCount) +} + +func NewSyncProtectionSoftDeleteNoCtx(jobID string) *Error { + return newError(Context(), ErrSyncProtectionSoftDelete, jobID) +} + +func NewSyncProtectionInvalidNoCtx() *Error { + return newError(Context(), ErrSyncProtectionInvalid) +} diff --git a/pkg/vm/engine/tae/db/gc/v3/sync_protection.go b/pkg/vm/engine/tae/db/gc/v3/sync_protection.go index 8a048360d1397..a15164f786443 100644 --- a/pkg/vm/engine/tae/db/gc/v3/sync_protection.go +++ b/pkg/vm/engine/tae/db/gc/v3/sync_protection.go @@ -16,7 +16,6 @@ package gc import ( "encoding/base64" - "fmt" "sync" "sync/atomic" "time" @@ -98,7 +97,7 @@ func (m *SyncProtectionManager) RegisterSyncProtection( "GC-Sync-Protection-Register-Rejected-GC-Running", zap.String("job-id", jobID), ) - return moerr.NewInternalErrorNoCtx("GC is running, please retry later") + return moerr.NewGCIsRunningNoCtx() } // Check if job already exists @@ -107,7 +106,7 @@ func (m *SyncProtectionManager) RegisterSyncProtection( "GC-Sync-Protection-Register-Already-Exists", zap.String("job-id", jobID), ) - return moerr.NewInternalErrorNoCtx(fmt.Sprintf("sync protection already exists: %s", jobID)) + return moerr.NewSyncProtectionExistsNoCtx(jobID) } // Check max count @@ -118,7 +117,7 @@ func (m *SyncProtectionManager) RegisterSyncProtection( zap.Int("current-count", len(m.protections)), zap.Int("max-count", m.maxCount), ) - return moerr.NewInternalErrorNoCtx(fmt.Sprintf("sync protection max count reached: %d", m.maxCount)) + return moerr.NewSyncProtectionMaxCountNoCtx(m.maxCount) } // Check if BF data is empty @@ -127,7 +126,7 @@ func (m *SyncProtectionManager) RegisterSyncProtection( "GC-Sync-Protection-Register-Empty-BF", zap.String("job-id", jobID), ) - return moerr.NewInternalErrorNoCtx("bf is empty") + return moerr.NewSyncProtectionInvalidNoCtx() } // Decode base64 BloomFilter data @@ -138,7 +137,7 @@ func (m *SyncProtectionManager) RegisterSyncProtection( zap.String("job-id", jobID), zap.Error(err), ) - return moerr.NewInternalErrorNoCtx(fmt.Sprintf("failed to decode bloom filter: %v", err)) + return moerr.NewSyncProtectionInvalidNoCtx() } // Unmarshal BloomFilter (using index.BloomFilter which is based on xorfilter - deterministic) @@ -148,7 +147,12 @@ func (m *SyncProtectionManager) RegisterSyncProtection( func() { defer func() { if r := recover(); r != nil { - unmarshalErr = moerr.NewInternalErrorNoCtx(fmt.Sprintf("failed to unmarshal bloom filter (panic): %v", r)) + logutil.Error( + "GC-Sync-Protection-Register-Unmarshal-Panic", + zap.String("job-id", jobID), + zap.Any("panic", r), + ) + unmarshalErr = moerr.NewSyncProtectionInvalidNoCtx() } }() unmarshalErr = bf.Unmarshal(bfBytes) @@ -159,7 +163,7 @@ func (m *SyncProtectionManager) RegisterSyncProtection( zap.String("job-id", jobID), zap.Error(unmarshalErr), ) - return moerr.NewInternalErrorNoCtx(fmt.Sprintf("failed to unmarshal bloom filter: %v", unmarshalErr)) + return unmarshalErr } m.protections[jobID] = &SyncProtection{ @@ -191,7 +195,7 @@ func (m *SyncProtectionManager) RenewSyncProtection(jobID string, validTS int64) "GC-Sync-Protection-Renew-Not-Found", zap.String("job-id", jobID), ) - return moerr.NewInternalErrorNoCtx(fmt.Sprintf("sync protection not found: %s", jobID)) + return moerr.NewSyncProtectionNotFoundNoCtx(jobID) } if p.SoftDelete { @@ -199,7 +203,7 @@ func (m *SyncProtectionManager) RenewSyncProtection(jobID string, validTS int64) "GC-Sync-Protection-Renew-Already-Soft-Deleted", zap.String("job-id", jobID), ) - return moerr.NewInternalErrorNoCtx(fmt.Sprintf("sync protection is soft deleted: %s", jobID)) + return moerr.NewSyncProtectionSoftDeleteNoCtx(jobID) } oldValidTS := p.ValidTS @@ -226,7 +230,7 @@ func (m *SyncProtectionManager) UnregisterSyncProtection(jobID string) error { "GC-Sync-Protection-Unregister-Not-Found", zap.String("job-id", jobID), ) - return moerr.NewInternalErrorNoCtx(fmt.Sprintf("sync protection not found: %s", jobID)) + return moerr.NewSyncProtectionNotFoundNoCtx(jobID) } p.SoftDelete = true diff --git a/pkg/vm/engine/tae/db/gc/v3/sync_protection_test.go b/pkg/vm/engine/tae/db/gc/v3/sync_protection_test.go index 1596d4b305ab2..c46a3c5ebaac9 100644 --- a/pkg/vm/engine/tae/db/gc/v3/sync_protection_test.go +++ b/pkg/vm/engine/tae/db/gc/v3/sync_protection_test.go @@ -371,18 +371,18 @@ func TestSyncProtectionManager_InvalidBFData(t *testing.T) { // Test empty BF err := mgr.RegisterSyncProtection(jobID, "", validTS) require.Error(t, err) - assert.Contains(t, err.Error(), "bf is empty") + assert.Contains(t, err.Error(), "invalid sync protection") // Test invalid base64 err = mgr.RegisterSyncProtection(jobID, "invalid-base64!!!", validTS) require.Error(t, err) - assert.Contains(t, err.Error(), "decode") + assert.Contains(t, err.Error(), "invalid sync protection") // Test invalid BloomFilter data invalidBF := base64.StdEncoding.EncodeToString([]byte("not a bloom filter")) err = mgr.RegisterSyncProtection(jobID, invalidBF, validTS) require.Error(t, err) - assert.Contains(t, err.Error(), "unmarshal") + assert.Contains(t, err.Error(), "invalid sync protection") } diff --git a/test/distributed/cases/function/mo_ctl/mo_ctl_sync_protection.result b/test/distributed/cases/function/mo_ctl/mo_ctl_sync_protection.result index 672a78bae1533..ef9e1ad9467c3 100644 --- a/test/distributed/cases/function/mo_ctl/mo_ctl_sync_protection.result +++ b/test/distributed/cases/function/mo_ctl/mo_ctl_sync_protection.result @@ -1,21 +1,14 @@ select mo_ctl('dn', 'diskcleaner', 'register_sync_protection.invalid_json'); -mo_ctl(dn, diskcleaner, register_sync_protection.invalid_json) invalid argument register_sync_protection, bad value invalid_json select mo_ctl('dn', 'diskcleaner', 'register_sync_protection.{"job_id":"test-job-1"}'); -mo_ctl(dn, diskcleaner, register_sync_protection.{"job_id":"test-job-1"}) internal error: bf is empty select mo_ctl('dn', 'diskcleaner', 'register_sync_protection.{"job_id":"test-job-2","bf":"invalid!!!","valid_ts":1234567890}'); -mo_ctl(dn, diskcleaner, register_sync_protection.{"job_id":"test-job-2","bf":"invalid!!!","valid_ts":1234567890}) internal error: failed to decode bloom filter: illegal base64 data at input byte 7 select mo_ctl('dn', 'diskcleaner', 'renew_sync_protection.{"job_id":"non-existent","valid_ts":1234567890}'); -mo_ctl(dn, diskcleaner, renew_sync_protection.{"job_id":"non-existent","valid_ts":1234567890}) internal error: sync protection not found: non-existent select mo_ctl('dn', 'diskcleaner', 'unregister_sync_protection.{"job_id":"non-existent"}'); -mo_ctl(dn, diskcleaner, unregister_sync_protection.{"job_id":"non-existent"}) internal error: sync protection not found: non-existent select mo_ctl('dn', 'diskcleaner', 'register_sync_protection.'); -mo_ctl(dn, diskcleaner, register_sync_protection.) invalid argument register_sync_protection, bad value empty value select mo_ctl('dn', 'diskcleaner', 'unknown_sync_protection.{}'); -mo_ctl(dn, diskcleaner, unknown_sync_protection.{}) internal error: handleDiskCleaner: invalid operation! From 39530c03e04640af597ee72e710790bca97734d3 Mon Sep 17 00:00:00 2001 From: GreatRiver Date: Wed, 4 Feb 2026 14:07:09 +0800 Subject: [PATCH 07/18] U --- .../cases/function/mo_ctl/mo_ctl_sync_protection.result | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/distributed/cases/function/mo_ctl/mo_ctl_sync_protection.result b/test/distributed/cases/function/mo_ctl/mo_ctl_sync_protection.result index ef9e1ad9467c3..7426a6a9eb1e1 100644 --- a/test/distributed/cases/function/mo_ctl/mo_ctl_sync_protection.result +++ b/test/distributed/cases/function/mo_ctl/mo_ctl_sync_protection.result @@ -1,13 +1,13 @@ select mo_ctl('dn', 'diskcleaner', 'register_sync_protection.invalid_json'); invalid argument register_sync_protection, bad value invalid_json select mo_ctl('dn', 'diskcleaner', 'register_sync_protection.{"job_id":"test-job-1"}'); -internal error: bf is empty +invalid sync protection request select mo_ctl('dn', 'diskcleaner', 'register_sync_protection.{"job_id":"test-job-2","bf":"invalid!!!","valid_ts":1234567890}'); -internal error: failed to decode bloom filter: illegal base64 data at input byte 7 +invalid sync protection request select mo_ctl('dn', 'diskcleaner', 'renew_sync_protection.{"job_id":"non-existent","valid_ts":1234567890}'); -internal error: sync protection not found: non-existent +sync protection not found: non-existent select mo_ctl('dn', 'diskcleaner', 'unregister_sync_protection.{"job_id":"non-existent"}'); -internal error: sync protection not found: non-existent +sync protection not found: non-existent select mo_ctl('dn', 'diskcleaner', 'register_sync_protection.'); invalid argument register_sync_protection, bad value empty value select mo_ctl('dn', 'diskcleaner', 'unknown_sync_protection.{}'); From 0941fa3b0781dc9b3c7b1ec8df6dda87d324a672 Mon Sep 17 00:00:00 2001 From: GreatRiver Date: Wed, 4 Feb 2026 14:25:28 +0800 Subject: [PATCH 08/18] U --- pkg/common/bloomfilter/bloomfilter.go | 93 ------------------- pkg/vm/engine/tae/db/gc/v3/sync_protection.go | 40 +------- .../tae/db/gc/v3/sync_protection_test.go | 22 ----- 3 files changed, 2 insertions(+), 153 deletions(-) diff --git a/pkg/common/bloomfilter/bloomfilter.go b/pkg/common/bloomfilter/bloomfilter.go index 5d2eab4c31339..f00e96a82bac5 100644 --- a/pkg/common/bloomfilter/bloomfilter.go +++ b/pkg/common/bloomfilter/bloomfilter.go @@ -183,59 +183,6 @@ func (bf *BloomFilter) TestRow(v *vector.Vector, row int) bool { return true } -// TestRowDebug tests if a single row might be in the bloom filter with debug info. -// Returns (result, encodedKey, hashVals) -func (bf *BloomFilter) TestRowDebug(v *vector.Vector, row int) (bool, []byte, []uint64) { - if row < 0 || row >= v.Length() { - return false, nil, nil - } - - bf.keys[0] = bf.keys[0][:0] - encodeHashKeys(bf.keys[:1], v, row, 1) - - // Copy the encoded key for debugging - encodedKey := make([]byte, len(bf.keys[0])) - copy(encodedKey, bf.keys[0]) - - bitSize := uint64(bf.shared.bitmap.Len()) - lastSeed := len(bf.shared.hashSeed) - 1 - - getIdxVal := func(v uint64) uint64 { - if bitSize == 0 || v < bitSize { - return v - } else { - return v % bitSize - } - } - - for k := 0; k < lastSeed; k++ { - hashtable.BytesBatchGenHashStatesWithSeed(&bf.keys[0], &bf.states[0], 1, bf.shared.hashSeed[k]) - idx := k * 3 - bf.vals[0][idx] = getIdxVal(bf.states[0][0]) - bf.vals[0][idx+1] = getIdxVal(bf.states[0][1]) - bf.vals[0][idx+2] = getIdxVal(bf.states[0][2]) - } - hashtable.BytesBatchGenHashStatesWithSeed(&bf.keys[0], &bf.states[0], 1, bf.shared.hashSeed[lastSeed]) - idx := lastSeed * 3 - bf.vals[0][idx] = getIdxVal(bf.states[0][0]) - bf.vals[0][idx+1] = getIdxVal(bf.states[0][1]) - bf.vals[0][idx+2] = getIdxVal(bf.states[0][2]) - - bf.keys[0] = bf.keys[0][:0] - - // Copy hash values for debugging - hashVals := make([]uint64, bf.valLength) - copy(hashVals, bf.vals[0][:bf.valLength]) - - vals := bf.vals[0] - for j := 0; j < bf.valLength; j++ { - if !bf.shared.bitmap.Contains(vals[j]) { - return false, encodedKey, hashVals - } - } - return true, encodedKey, hashVals -} - func (bf *BloomFilter) TestAndAdd(v *vector.Vector, callBack func(bool, int)) { bf.handle(v, func(idx, beginIdx int) { var contains bool @@ -326,46 +273,6 @@ func (bf *BloomFilter) Unmarshal(data []byte) error { return nil } -// GetBitmapLen returns the length of the bitmap for debugging -func (bf *BloomFilter) GetBitmapLen() int64 { - if bf == nil || bf.shared == nil { - return 0 - } - return bf.shared.bitmap.Len() -} - -// GetSeedCount returns the number of hash seeds for debugging -func (bf *BloomFilter) GetSeedCount() int { - if bf == nil || bf.shared == nil { - return 0 - } - return len(bf.shared.hashSeed) -} - -// GetFirstSeed returns the first hash seed for debugging -func (bf *BloomFilter) GetFirstSeed() uint64 { - if bf == nil || bf.shared == nil || len(bf.shared.hashSeed) == 0 { - return 0 - } - return bf.shared.hashSeed[0] -} - -// GetAllSeeds returns all hash seeds for debugging -func (bf *BloomFilter) GetAllSeeds() []uint64 { - if bf == nil || bf.shared == nil { - return nil - } - return bf.shared.hashSeed -} - -// GetBitmapCount returns the number of bits set in the bitmap for debugging -func (bf *BloomFilter) GetBitmapCount() int { - if bf == nil || bf.shared == nil { - return 0 - } - return bf.shared.bitmap.Count() -} - // handle computes the hash value of each element and executes the callback. func (bf *BloomFilter) handle(v *vector.Vector, callBack func(int, int)) { length := v.Length() diff --git a/pkg/vm/engine/tae/db/gc/v3/sync_protection.go b/pkg/vm/engine/tae/db/gc/v3/sync_protection.go index a15164f786443..e5b1f8df8641f 100644 --- a/pkg/vm/engine/tae/db/gc/v3/sync_protection.go +++ b/pkg/vm/engine/tae/db/gc/v3/sync_protection.go @@ -21,7 +21,6 @@ import ( "time" "github.com/matrixorigin/matrixone/pkg/common/moerr" - "github.com/matrixorigin/matrixone/pkg/common/mpool" "github.com/matrixorigin/matrixone/pkg/logutil" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/index" "go.uber.org/zap" @@ -52,17 +51,14 @@ type SyncProtectionManager struct { gcRunning atomic.Bool // Whether GC is running ttl time.Duration // TTL for non-soft-deleted protections maxCount int // Maximum number of protections - mp *mpool.MPool // Memory pool for vector operations } // NewSyncProtectionManager creates a new SyncProtectionManager func NewSyncProtectionManager() *SyncProtectionManager { - mp, _ := mpool.NewMPool("sync_protection", 0, mpool.NoFixed) return &SyncProtectionManager{ protections: make(map[string]*SyncProtection), ttl: DefaultSyncProtectionTTL, maxCount: DefaultMaxSyncProtections, - mp: mp, } } @@ -345,20 +341,6 @@ func (m *SyncProtectionManager) FilterProtectedFiles(files []string) []string { return files } - // Collect all BloomFilters - type bfEntry struct { - jobID string - bf *index.BloomFilter - } - var bfs []bfEntry - for jobID, p := range m.protections { - bfs = append(bfs, bfEntry{jobID: jobID, bf: &p.BF}) - } - - if len(bfs) == 0 { - return files - } - // Build result: files that are NOT protected result := make([]string, 0, len(files)) protectedCount := 0 @@ -367,8 +349,8 @@ func (m *SyncProtectionManager) FilterProtectedFiles(files []string) []string { protected := false // Check against each BloomFilter - for _, entry := range bfs { - if contains, err := entry.bf.MayContainsKey([]byte(f)); err == nil && contains { + for _, p := range m.protections { + if contains, err := p.BF.MayContainsKey([]byte(f)); err == nil && contains { protected = true break } @@ -392,21 +374,3 @@ func (m *SyncProtectionManager) FilterProtectedFiles(files []string) []string { return result } - -// DebugTestFile tests if a single file is protected (for debugging purposes) -func (m *SyncProtectionManager) DebugTestFile(fileName string) bool { - m.RLock() - defer m.RUnlock() - - if len(m.protections) == 0 { - return false - } - - for _, p := range m.protections { - if result, err := p.BF.MayContainsKey([]byte(fileName)); err == nil && result { - return true - } - } - - return false -} diff --git a/pkg/vm/engine/tae/db/gc/v3/sync_protection_test.go b/pkg/vm/engine/tae/db/gc/v3/sync_protection_test.go index c46a3c5ebaac9..fef0500e767fa 100644 --- a/pkg/vm/engine/tae/db/gc/v3/sync_protection_test.go +++ b/pkg/vm/engine/tae/db/gc/v3/sync_protection_test.go @@ -385,28 +385,6 @@ func TestSyncProtectionManager_InvalidBFData(t *testing.T) { assert.Contains(t, err.Error(), "invalid sync protection") } - -func TestSyncProtectionManager_DebugTestFile(t *testing.T) { - mgr := NewSyncProtectionManager() - - // Test with no protections - assert.False(t, mgr.DebugTestFile("any-file")) - - // Register protection - jobID := "job-1" - bfData := buildTestBF(t, []string{"protected-file"}) - validTS := time.Now().UnixNano() - - err := mgr.RegisterSyncProtection(jobID, bfData, validTS) - require.NoError(t, err) - - // Test protected file - assert.True(t, mgr.DebugTestFile("protected-file")) - - // Test unprotected file (may have false positives due to BloomFilter) - // We can't assert False here due to BloomFilter characteristics -} - func TestSyncProtectionManager_FilterProtectedFiles_EmptyFiles(t *testing.T) { mgr := NewSyncProtectionManager() From 54104a4c49b37903ac7a2ce147bb043fe4401cce Mon Sep 17 00:00:00 2001 From: GreatRiver Date: Wed, 4 Feb 2026 14:29:47 +0800 Subject: [PATCH 09/18] U --- cmd/mo-tool/main.go | 44 ++ pkg/sql/plan/function/ctl/cmd_disk_cleaner.go | 4 +- pkg/vm/engine/cmd_util/operations.pb.go | 459 +++++++++++++++--- pkg/vm/engine/tae/db/gc/v3/exec_v1.go | 2 +- .../tae/db/gc/v3/tool/sync_protection.go | 2 +- pkg/vm/engine/tae/rpc/handle_debug.go | 8 +- 6 files changed, 435 insertions(+), 84 deletions(-) create mode 100644 cmd/mo-tool/main.go diff --git a/cmd/mo-tool/main.go b/cmd/mo-tool/main.go new file mode 100644 index 0000000000000..8df6e75f29d51 --- /dev/null +++ b/cmd/mo-tool/main.go @@ -0,0 +1,44 @@ +// Copyright 2021 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + + dashboard "github.com/matrixorigin/matrixone/cmd/mo-dashboard" + debug "github.com/matrixorigin/matrixone/cmd/mo-debug" + inspect "github.com/matrixorigin/matrixone/cmd/mo-inspect" + object "github.com/matrixorigin/matrixone/cmd/mo-object-tool" + ckp "github.com/matrixorigin/matrixone/cmd/mo-object-tool/ckp" + "github.com/spf13/cobra" +) + +func main() { + var rootCmd = &cobra.Command{ + Use: "mo-tool", + Short: "Mo tool", + Long: "Mo tool is a multifunctional development tool", + } + + rootCmd.AddCommand(debug.PrepareCommand()) + rootCmd.AddCommand(inspect.PrepareCommand()) + rootCmd.AddCommand(dashboard.PrepareCommand()) + rootCmd.AddCommand(object.PrepareCommand()) + rootCmd.AddCommand(ckp.PrepareCommand()) + + if err := rootCmd.Execute(); err != nil { + os.Exit(1) + } +} diff --git a/pkg/sql/plan/function/ctl/cmd_disk_cleaner.go b/pkg/sql/plan/function/ctl/cmd_disk_cleaner.go index 514d2b2365c65..f7693071db408 100644 --- a/pkg/sql/plan/function/ctl/cmd_disk_cleaner.go +++ b/pkg/sql/plan/function/ctl/cmd_disk_cleaner.go @@ -66,7 +66,7 @@ func IsValidArg(parameter string, proc *process.Process) (*cmd_util.DiskCleaner, // Join remaining parts as JSON value (in case JSON contains dots) value = strings.Join(parameters[1:], ".") } - + // Debug: print parameter info logutil.Info( "GC-Sync-Protection-CMD-Parse", @@ -75,7 +75,7 @@ func IsValidArg(parameter string, proc *process.Process) (*cmd_util.DiskCleaner, zap.Int("parts-count", len(parameters)), zap.Int("value-len", len(value)), ) - + return &cmd_util.DiskCleaner{ Op: op, Value: value, diff --git a/pkg/vm/engine/cmd_util/operations.pb.go b/pkg/vm/engine/cmd_util/operations.pb.go index acfcab731f067..dea678cd84b9d 100644 --- a/pkg/vm/engine/cmd_util/operations.pb.go +++ b/pkg/vm/engine/cmd_util/operations.pb.go @@ -1343,6 +1343,67 @@ func (m *FaultInjectReq) GetParameter() string { return "" } +func (m *SyncProtection) Reset() { *m = SyncProtection{} } +func (m *SyncProtection) String() string { return proto.CompactTextString(m) } +func (*SyncProtection) ProtoMessage() {} +func (*SyncProtection) Descriptor() ([]byte, []int) { + return fileDescriptor_1b4a5877375e491e, []int{23} +} +func (m *SyncProtection) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *SyncProtection) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_SyncProtection.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *SyncProtection) XXX_Merge(src proto.Message) { + xxx_messageInfo_SyncProtection.Merge(m, src) +} +func (m *SyncProtection) XXX_Size() int { + return m.ProtoSize() +} +func (m *SyncProtection) XXX_DiscardUnknown() { + xxx_messageInfo_SyncProtection.DiscardUnknown(m) +} + +var xxx_messageInfo_SyncProtection proto.InternalMessageInfo + +func (m *SyncProtection) GetOp() string { + if m != nil { + return m.Op + } + return "" +} + +func (m *SyncProtection) GetJobID() string { + if m != nil { + return m.JobID + } + return "" +} + +func (m *SyncProtection) GetObjects() []string { + if m != nil { + return m.Objects + } + return nil +} + +func (m *SyncProtection) GetValidTS() int64 { + if m != nil { + return m.ValidTS + } + return 0 +} + func init() { proto.RegisterEnum("cmd_util.ChangedListType", ChangedListType_name, ChangedListType_value) proto.RegisterType((*AccessInfo)(nil), "cmd_util.AccessInfo") @@ -1368,87 +1429,91 @@ func init() { proto.RegisterType((*GetChangedTableListReq)(nil), "cmd_util.GetChangedTableListReq") proto.RegisterType((*GetChangedTableListResp)(nil), "cmd_util.GetChangedTableListResp") proto.RegisterType((*FaultInjectReq)(nil), "cmd_util.FaultInjectReq") + proto.RegisterType((*SyncProtection)(nil), "cmd_util.SyncProtection") } func init() { proto.RegisterFile("operations.proto", fileDescriptor_1b4a5877375e491e) } var fileDescriptor_1b4a5877375e491e = []byte{ - // 1195 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x56, 0xbd, 0xaf, 0x13, 0x47, - 0x10, 0xf7, 0xf9, 0xce, 0xf6, 0xf3, 0xf8, 0x7d, 0x38, 0x0b, 0x01, 0x83, 0x88, 0x9f, 0x85, 0x50, - 0xf2, 0x82, 0xe0, 0x01, 0x46, 0x08, 0x89, 0x54, 0xd8, 0x86, 0xc8, 0xe2, 0xe3, 0xa1, 0xb5, 0x21, - 0x25, 0x59, 0x9f, 0x17, 0xfb, 0xf0, 0xf9, 0xee, 0x72, 0xbb, 0x0e, 0x71, 0xfe, 0x82, 0x94, 0xa9, - 0x52, 0x27, 0x65, 0xfe, 0x92, 0x50, 0x52, 0xa6, 0x22, 0x09, 0xaf, 0x48, 0x15, 0x29, 0x35, 0x55, - 0xb4, 0xb3, 0x7b, 0x1f, 0x46, 0xc6, 0x4d, 0x94, 0x22, 0xdd, 0xfe, 0x66, 0x66, 0x67, 0x7f, 0xbf, - 0x99, 0xb9, 0xdd, 0x83, 0x7a, 0x18, 0xf1, 0x98, 0x49, 0x2f, 0x0c, 0xc4, 0x61, 0x14, 0x87, 0x32, - 0x24, 0x5b, 0xee, 0x7c, 0xfc, 0x74, 0x21, 0x3d, 0xff, 0xec, 0xe5, 0x89, 0x27, 0xa7, 0x8b, 0xd1, - 0xa1, 0x1b, 0xce, 0xaf, 0x4c, 0xc2, 0x49, 0x78, 0x05, 0x03, 0x46, 0x8b, 0x67, 0x88, 0x10, 0xe0, - 0x4a, 0x6f, 0x3c, 0xbb, 0x27, 0xbd, 0x39, 0x17, 0x92, 0xcd, 0x23, 0x6d, 0x38, 0xff, 0x25, 0xc0, - 0x6d, 0xd7, 0xe5, 0x42, 0xf4, 0x83, 0x67, 0x21, 0x39, 0x07, 0xd5, 0xdb, 0xae, 0x1b, 0x2e, 0x02, - 0xd9, 0xef, 0x35, 0xac, 0x96, 0x75, 0xb0, 0x43, 0x33, 0x03, 0x39, 0x05, 0xe5, 0xc7, 0x82, 0xc7, - 0xfd, 0x5e, 0xa3, 0x88, 0x2e, 0x83, 0x94, 0x9d, 0x86, 0x3e, 0xef, 0xf7, 0x1a, 0xb6, 0xb6, 0x6b, - 0x74, 0xcb, 0xf9, 0xfb, 0xa7, 0xfd, 0xc2, 0xf9, 0xef, 0x2c, 0x80, 0xbb, 0xfe, 0x42, 0x4c, 0x87, - 0x6c, 0xe4, 0x73, 0x72, 0x2b, 0x7f, 0x20, 0x9e, 0x51, 0x6b, 0x9f, 0x3c, 0x4c, 0xf4, 0x1c, 0x66, - 0xbe, 0x8e, 0xf3, 0xf2, 0xf5, 0x7e, 0x81, 0xe6, 0xe9, 0x35, 0x01, 0x7a, 0x4c, 0xb2, 0x11, 0x13, - 0xdc, 0x90, 0x70, 0x68, 0xce, 0x42, 0x1a, 0x50, 0xc1, 0x43, 0x0c, 0x13, 0x87, 0x26, 0xd0, 0x50, - 0xb9, 0x07, 0xb5, 0x9e, 0x27, 0x66, 0x5d, 0x9f, 0xb3, 0x80, 0xc7, 0x64, 0x17, 0x8a, 0x47, 0x11, - 0x52, 0xa8, 0xd2, 0xe2, 0x51, 0x44, 0xea, 0x60, 0xdf, 0xe3, 0x4b, 0xcc, 0x5b, 0xa5, 0x6a, 0x49, - 0x4e, 0x42, 0xe9, 0x09, 0xf3, 0x17, 0x1c, 0xd3, 0x55, 0xa9, 0x06, 0x69, 0x32, 0xe8, 0x4e, 0xb9, - 0x3b, 0x8b, 0x42, 0x2f, 0x90, 0xe4, 0x26, 0xec, 0xa0, 0xc8, 0xde, 0x42, 0x77, 0x0a, 0xd3, 0xda, - 0x9d, 0x0f, 0xde, 0xbe, 0xde, 0xdf, 0x51, 0x35, 0x3f, 0x4c, 0x1c, 0x74, 0x35, 0xce, 0x24, 0xbb, - 0x01, 0x7b, 0xfd, 0x40, 0xf2, 0xd8, 0xe5, 0x91, 0xec, 0x86, 0xf3, 0xb9, 0x27, 0x55, 0x2f, 0x90, - 0xfd, 0x43, 0x36, 0xe7, 0x86, 0x64, 0x66, 0x30, 0xdb, 0x66, 0x50, 0xed, 0x07, 0x22, 0xe2, 0xae, - 0x1c, 0x3e, 0xfc, 0x57, 0x95, 0x3d, 0x07, 0xd5, 0xa3, 0x64, 0xc8, 0x4c, 0x01, 0x32, 0x83, 0x39, - 0x6c, 0x04, 0x35, 0x73, 0x18, 0xe5, 0x22, 0x22, 0x67, 0xc0, 0x1e, 0x2e, 0x75, 0xf9, 0x4a, 0x9d, - 0xca, 0xdb, 0xd7, 0xfb, 0xb6, 0x17, 0x48, 0xaa, 0x6c, 0xaa, 0x0f, 0x0f, 0xb8, 0x10, 0x6c, 0xc2, - 0x4d, 0xae, 0x04, 0x2a, 0xcf, 0x23, 0xb6, 0xf4, 0x43, 0x36, 0xc6, 0x92, 0x6e, 0xd3, 0x04, 0x9a, - 0x33, 0x1e, 0x41, 0xad, 0xcb, 0x24, 0xf3, 0xc3, 0x09, 0x9e, 0x41, 0xc0, 0xe9, 0x4b, 0x3e, 0x37, - 0xf2, 0x71, 0x4d, 0x3e, 0x01, 0x7b, 0xb0, 0x18, 0x35, 0x8a, 0x2d, 0xfb, 0xa0, 0xd6, 0xfe, 0x30, - 0xd3, 0x97, 0xdb, 0x47, 0x55, 0x84, 0xc9, 0xf8, 0x83, 0x1a, 0x3f, 0xb6, 0xf0, 0xe5, 0x23, 0xec, - 0x13, 0x01, 0x27, 0x57, 0x50, 0x5c, 0x2b, 0xdb, 0xdd, 0x98, 0x7f, 0x65, 0xb8, 0xe2, 0x5a, 0xcd, - 0xf4, 0x6d, 0x17, 0xab, 0xa1, 0x5b, 0x6f, 0x10, 0x32, 0x62, 0xf1, 0xa4, 0xe1, 0xa8, 0xf6, 0x52, - 0x5c, 0x2b, 0xdb, 0x40, 0xd9, 0x4a, 0x7a, 0xbf, 0x5a, 0x93, 0xb3, 0xb0, 0xd5, 0x0d, 0x03, 0x21, - 0x59, 0x20, 0x1b, 0xe5, 0x96, 0x75, 0xb0, 0x45, 0x53, 0x6c, 0x88, 0x7d, 0x01, 0xd5, 0x61, 0xcc, - 0x5c, 0x3e, 0x88, 0x58, 0xa0, 0x46, 0xcf, 0x9d, 0x8f, 0x0d, 0x2b, 0xb5, 0x54, 0xa3, 0x27, 0x22, - 0x16, 0x08, 0xc3, 0x4a, 0x03, 0xd5, 0x27, 0x39, 0x8d, 0xb9, 0x98, 0x86, 0xbe, 0xae, 0xa0, 0x4d, - 0x33, 0x83, 0x49, 0xfc, 0x29, 0xec, 0x74, 0xfc, 0xd0, 0x9d, 0x3d, 0xe0, 0x92, 0x61, 0x73, 0x09, - 0x38, 0x9e, 0x1e, 0x09, 0xfb, 0xc0, 0xa1, 0xb8, 0x36, 0xa1, 0x7d, 0xa8, 0x75, 0x67, 0x51, 0x1a, - 0xd8, 0x80, 0xca, 0xd7, 0x3c, 0x16, 0xc9, 0xf8, 0xee, 0xd0, 0x04, 0x2a, 0x39, 0x7e, 0xe8, 0x66, - 0xe3, 0xb1, 0x4d, 0x53, 0x6c, 0x52, 0xfd, 0x6c, 0xc1, 0x89, 0x81, 0x0c, 0x63, 0x36, 0xe1, 0x8f, - 0x55, 0xab, 0x55, 0x1f, 0x9e, 0x3e, 0xb9, 0xaa, 0x72, 0x0e, 0x16, 0xae, 0xcb, 0xb9, 0x56, 0xb7, - 0x45, 0x13, 0x48, 0x6e, 0x00, 0x74, 0x67, 0xd1, 0x9d, 0x40, 0xc6, 0x1e, 0x17, 0x6b, 0xfa, 0x99, - 0x11, 0xa3, 0xb9, 0x40, 0xf2, 0x19, 0x6c, 0xa3, 0xbc, 0x64, 0xa3, 0x8d, 0x1b, 0x4f, 0x67, 0x1b, - 0x57, 0xc4, 0xd3, 0x95, 0x60, 0xc3, 0xf5, 0x0a, 0xec, 0xad, 0x52, 0x35, 0xfd, 0x76, 0xfb, 0x63, - 0x81, 0x55, 0xb2, 0xa9, 0x41, 0x66, 0xc3, 0x72, 0x9d, 0xb6, 0x6b, 0x1b, 0xb4, 0x65, 0xe9, 0x8a, - 0xf9, 0x74, 0xaa, 0xab, 0x03, 0xef, 0x5b, 0xc3, 0xda, 0xa1, 0x1a, 0x28, 0xeb, 0x03, 0x36, 0xf1, - 0x5c, 0x9c, 0x2a, 0x87, 0x6a, 0x60, 0x8e, 0xfe, 0x65, 0x6d, 0x5d, 0xdb, 0xff, 0xed, 0xd9, 0x2a, - 0xfb, 0xd1, 0xe8, 0x79, 0x37, 0x90, 0xa2, 0x51, 0xc2, 0xe8, 0x04, 0x2a, 0x4f, 0xc7, 0x9f, 0xa1, - 0xa7, 0xac, 0x3d, 0x06, 0x2a, 0x0f, 0x0d, 0x5f, 0xa0, 0xa7, 0xa2, 0x3d, 0x06, 0x1a, 0x25, 0x7f, - 0xad, 0x55, 0x72, 0xfd, 0xff, 0xa4, 0x84, 0x5c, 0x80, 0x9d, 0x41, 0xc0, 0x22, 0x31, 0x0d, 0xa5, - 0x66, 0xb0, 0x85, 0xfe, 0x55, 0x63, 0xfa, 0x71, 0xed, 0x25, 0x66, 0xca, 0xd9, 0x58, 0x4d, 0xd9, - 0x55, 0xd8, 0x4a, 0x4c, 0xe9, 0x05, 0x9d, 0xbd, 0xc8, 0xc3, 0x64, 0x45, 0xd3, 0xa8, 0xf4, 0x9e, - 0xaf, 0xaf, 0xa6, 0x12, 0xd1, 0x86, 0xb2, 0xdd, 0x84, 0xca, 0xea, 0x57, 0xf5, 0x51, 0xee, 0xab, - 0x4a, 0x9f, 0x2c, 0x15, 0xb2, 0xc4, 0xdb, 0xb2, 0xb2, 0xfa, 0x75, 0x1c, 0x5b, 0x70, 0x62, 0x4d, - 0x18, 0xb9, 0x08, 0xa5, 0x81, 0x64, 0xf1, 0x66, 0xe6, 0x3a, 0x84, 0x7c, 0x0c, 0xf6, 0x9d, 0x60, - 0x8c, 0x57, 0xc5, 0xfb, 0x22, 0x55, 0x80, 0xba, 0xcf, 0xee, 0x9b, 0x7b, 0xe4, 0x9a, 0x79, 0x11, - 0x32, 0x43, 0xde, 0xdb, 0xc6, 0xae, 0xe6, 0xbc, 0x6d, 0xe5, 0x45, 0x72, 0xc3, 0x65, 0xc4, 0xf1, - 0xee, 0x2d, 0xd1, 0xcc, 0xa0, 0xca, 0xf3, 0xc4, 0xdc, 0x65, 0x65, 0x7d, 0x97, 0x19, 0x68, 0x54, - 0xfe, 0x66, 0xc1, 0xa9, 0xcf, 0xb9, 0xec, 0x4e, 0x59, 0x30, 0xe1, 0x63, 0x7c, 0x58, 0xef, 0x7b, - 0x42, 0xaa, 0x2e, 0x5d, 0x80, 0xe2, 0x70, 0x60, 0x4a, 0xb7, 0x9e, 0x7b, 0x71, 0x38, 0xc8, 0x0d, - 0xa7, 0x9e, 0xc2, 0x64, 0x38, 0x5b, 0x50, 0x4b, 0x7f, 0x49, 0xc6, 0xa2, 0xe1, 0xa0, 0x33, 0x6f, - 0x52, 0x97, 0xa9, 0xfe, 0x2f, 0x19, 0x27, 0x33, 0x99, 0x62, 0x35, 0xc4, 0x77, 0xbe, 0x91, 0x31, - 0x43, 0xd2, 0xdb, 0x54, 0x03, 0x72, 0x19, 0x1c, 0x54, 0x59, 0x69, 0x59, 0x07, 0xbb, 0xed, 0x33, - 0xf9, 0x76, 0x22, 0x7d, 0xc5, 0x5c, 0x05, 0x50, 0x0c, 0x33, 0x0a, 0xff, 0xb4, 0xe0, 0xf4, 0x5a, - 0x85, 0x22, 0x22, 0x97, 0xa0, 0xfc, 0x90, 0xbf, 0xe0, 0x62, 0x73, 0x33, 0x4d, 0xcc, 0x3b, 0xdf, - 0xe1, 0x7b, 0xa5, 0xda, 0x9b, 0xa5, 0x3a, 0xef, 0x93, 0x5a, 0xca, 0x4b, 0xbd, 0x04, 0xe5, 0x23, - 0x7f, 0xac, 0x98, 0x95, 0x37, 0x31, 0xd3, 0x31, 0x46, 0xe9, 0x7d, 0xd8, 0xc5, 0x27, 0xbe, 0x1f, - 0x3c, 0xc7, 0xbf, 0x13, 0xbc, 0xce, 0xe7, 0x5c, 0x4e, 0xc3, 0xe4, 0x49, 0x35, 0x48, 0xcd, 0x4c, - 0xc4, 0x62, 0x36, 0xe7, 0x92, 0xc7, 0xc9, 0x7f, 0x4e, 0x6a, 0xd0, 0xd9, 0x2e, 0xde, 0x84, 0xbd, - 0x77, 0xca, 0x4a, 0xea, 0xb0, 0x8d, 0x5f, 0x84, 0xb1, 0xd7, 0x0b, 0x84, 0xc0, 0x6e, 0x37, 0xf4, - 0x7d, 0xee, 0x26, 0xf5, 0xad, 0x5b, 0x9d, 0xd6, 0xab, 0x3f, 0x9a, 0x85, 0x97, 0x6f, 0x9a, 0xd6, - 0xab, 0x37, 0x4d, 0xeb, 0xf7, 0x37, 0xcd, 0xc2, 0xf7, 0xc7, 0xcd, 0xc2, 0x8f, 0xc7, 0x4d, 0xeb, - 0xd5, 0x71, 0xb3, 0xf0, 0xeb, 0x71, 0xb3, 0x30, 0x2a, 0xe3, 0x4f, 0xf7, 0xf5, 0x7f, 0x02, 0x00, - 0x00, 0xff, 0xff, 0xce, 0x43, 0x55, 0xaa, 0xd2, 0x0b, 0x00, 0x00, + // 1243 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x56, 0x3b, 0x93, 0x13, 0xc7, + 0x13, 0xd7, 0x6a, 0x57, 0xd2, 0xa9, 0x75, 0xaf, 0xff, 0xc0, 0x1f, 0x04, 0x85, 0x75, 0x57, 0x14, + 0x65, 0x9f, 0x29, 0x38, 0xe0, 0x28, 0x8a, 0x2a, 0x1c, 0x71, 0x3a, 0x70, 0xc9, 0x3c, 0xee, 0x6a, + 0x24, 0x70, 0x88, 0x47, 0xab, 0x41, 0x5a, 0xb4, 0xda, 0x59, 0xef, 0x8c, 0x8c, 0xe5, 0x4f, 0xe0, + 0xd0, 0x91, 0x63, 0x3b, 0xf4, 0x27, 0x31, 0x21, 0xa1, 0x23, 0x6c, 0x73, 0x81, 0x23, 0x57, 0x39, + 0x26, 0x72, 0x4d, 0xcf, 0xec, 0x43, 0x94, 0x4e, 0x89, 0xcb, 0x81, 0xb3, 0xf9, 0x75, 0xf7, 0xf4, + 0xfc, 0xfa, 0x31, 0x3d, 0x03, 0x9b, 0x22, 0xe6, 0x09, 0x53, 0x81, 0x88, 0xe4, 0x6e, 0x9c, 0x08, + 0x25, 0xc8, 0x8a, 0x3f, 0x19, 0x3c, 0x9b, 0xaa, 0x20, 0x3c, 0x7f, 0x75, 0x18, 0xa8, 0xd1, 0xb4, + 0xbf, 0xeb, 0x8b, 0xc9, 0xb5, 0xa1, 0x18, 0x8a, 0x6b, 0x68, 0xd0, 0x9f, 0x3e, 0x47, 0x84, 0x00, + 0x57, 0x66, 0xe3, 0xf9, 0x0d, 0x15, 0x4c, 0xb8, 0x54, 0x6c, 0x12, 0x1b, 0xc1, 0xc5, 0x2f, 0x00, + 0xee, 0xfa, 0x3e, 0x97, 0xb2, 0x13, 0x3d, 0x17, 0xe4, 0x02, 0xd4, 0xef, 0xfa, 0xbe, 0x98, 0x46, + 0xaa, 0x73, 0xd0, 0x74, 0xb6, 0x9d, 0x9d, 0x35, 0x9a, 0x0b, 0xc8, 0x19, 0xa8, 0x3e, 0x91, 0x3c, + 0xe9, 0x1c, 0x34, 0xcb, 0xa8, 0xb2, 0x48, 0xcb, 0xa9, 0x08, 0x79, 0xe7, 0xa0, 0xe9, 0x1a, 0xb9, + 0x41, 0x77, 0xbc, 0xbf, 0x7e, 0xdc, 0x2a, 0x5d, 0xfc, 0xd6, 0x01, 0xb8, 0x1f, 0x4e, 0xe5, 0xa8, + 0xc7, 0xfa, 0x21, 0x27, 0x77, 0x8a, 0x07, 0xe2, 0x19, 0x8d, 0xbd, 0xd3, 0xbb, 0x69, 0x3c, 0xbb, + 0xb9, 0x6e, 0xdf, 0x7b, 0xf5, 0x66, 0xab, 0x44, 0x8b, 0xf4, 0x5a, 0x00, 0x07, 0x4c, 0xb1, 0x3e, + 0x93, 0xdc, 0x92, 0xf0, 0x68, 0x41, 0x42, 0x9a, 0x50, 0xc3, 0x43, 0x2c, 0x13, 0x8f, 0xa6, 0xd0, + 0x52, 0x79, 0x00, 0x8d, 0x83, 0x40, 0x8e, 0xdb, 0x21, 0x67, 0x11, 0x4f, 0xc8, 0x3a, 0x94, 0x0f, + 0x63, 0xa4, 0x50, 0xa7, 0xe5, 0xc3, 0x98, 0x6c, 0x82, 0xfb, 0x80, 0xcf, 0xd0, 0x6f, 0x9d, 0xea, + 0x25, 0x39, 0x0d, 0x95, 0xa7, 0x2c, 0x9c, 0x72, 0x74, 0x57, 0xa7, 0x06, 0x64, 0xce, 0xa0, 0x3d, + 0xe2, 0xfe, 0x38, 0x16, 0x41, 0xa4, 0xc8, 0x6d, 0x58, 0xc3, 0x20, 0x0f, 0xa6, 0xa6, 0x52, 0xe8, + 0xd6, 0xdd, 0xff, 0xdf, 0xbb, 0x37, 0x5b, 0x6b, 0x3a, 0xe7, 0xbb, 0xa9, 0x82, 0xce, 0xdb, 0x59, + 0x67, 0xb7, 0x60, 0xa3, 0x13, 0x29, 0x9e, 0xf8, 0x3c, 0x56, 0x6d, 0x31, 0x99, 0x04, 0x4a, 0xd7, + 0x02, 0xd9, 0x3f, 0x66, 0x13, 0x6e, 0x49, 0xe6, 0x02, 0xbb, 0x6d, 0x0c, 0xf5, 0x4e, 0x24, 0x63, + 0xee, 0xab, 0xde, 0xe3, 0x7f, 0x94, 0xd9, 0x0b, 0x50, 0x3f, 0x4c, 0x9b, 0xcc, 0x26, 0x20, 0x17, + 0xd8, 0xc3, 0xfa, 0xd0, 0xb0, 0x87, 0x51, 0x2e, 0x63, 0x72, 0x0e, 0xdc, 0xde, 0xcc, 0xa4, 0xaf, + 0xb2, 0x5f, 0x7b, 0xf7, 0x66, 0xcb, 0x0d, 0x22, 0x45, 0xb5, 0x4c, 0xd7, 0xe1, 0x11, 0x97, 0x92, + 0x0d, 0xb9, 0xf5, 0x95, 0x42, 0xad, 0x39, 0x62, 0xb3, 0x50, 0xb0, 0x01, 0xa6, 0x74, 0x95, 0xa6, + 0xd0, 0x9e, 0x71, 0x04, 0x8d, 0x36, 0x53, 0x2c, 0x14, 0x43, 0x3c, 0x83, 0x80, 0xd7, 0x51, 0x7c, + 0x62, 0xc3, 0xc7, 0x35, 0xf9, 0x08, 0xdc, 0xee, 0xb4, 0xdf, 0x2c, 0x6f, 0xbb, 0x3b, 0x8d, 0xbd, + 0xff, 0xe7, 0xf1, 0x15, 0xf6, 0x51, 0x6d, 0x61, 0x3d, 0x7e, 0xaf, 0xdb, 0x8f, 0x4d, 0x43, 0x75, + 0x84, 0x75, 0x22, 0xe0, 0x15, 0x12, 0x8a, 0x6b, 0x2d, 0xbb, 0x9f, 0xf0, 0x2f, 0x2d, 0x57, 0x5c, + 0xeb, 0x9e, 0xbe, 0xeb, 0x63, 0x36, 0x4c, 0xe9, 0x2d, 0x42, 0x46, 0x2c, 0x19, 0x36, 0x3d, 0x5d, + 0x5e, 0x8a, 0x6b, 0x2d, 0xeb, 0x6a, 0x59, 0xc5, 0xec, 0xd7, 0x6b, 0x72, 0x1e, 0x56, 0xda, 0x22, + 0x92, 0x8a, 0x45, 0xaa, 0x59, 0xdd, 0x76, 0x76, 0x56, 0x68, 0x86, 0x2d, 0xb1, 0xcf, 0xa1, 0xde, + 0x4b, 0x98, 0xcf, 0xbb, 0x31, 0x8b, 0x74, 0xeb, 0xf9, 0x93, 0x81, 0x65, 0xa5, 0x97, 0xba, 0xf5, + 0x64, 0xcc, 0x22, 0x69, 0x59, 0x19, 0xa0, 0xeb, 0xa4, 0x46, 0x09, 0x97, 0x23, 0x11, 0x9a, 0x0c, + 0xba, 0x34, 0x17, 0x58, 0xc7, 0x1f, 0xc3, 0xda, 0x7e, 0x28, 0xfc, 0xf1, 0x23, 0xae, 0x18, 0x16, + 0x97, 0x80, 0x17, 0x98, 0x96, 0x70, 0x77, 0x3c, 0x8a, 0x6b, 0x6b, 0xda, 0x81, 0x46, 0x7b, 0x1c, + 0x67, 0x86, 0x4d, 0xa8, 0x7d, 0xc5, 0x13, 0x99, 0xb6, 0xef, 0x1a, 0x4d, 0xa1, 0x0e, 0x27, 0x14, + 0x7e, 0xde, 0x1e, 0xab, 0x34, 0xc3, 0xd6, 0xd5, 0x4f, 0x0e, 0x9c, 0xea, 0x2a, 0x91, 0xb0, 0x21, + 0x7f, 0xa2, 0x4b, 0xad, 0xeb, 0xf0, 0xec, 0xe9, 0x75, 0xed, 0xb3, 0x3b, 0xf5, 0x7d, 0xce, 0x4d, + 0x74, 0x2b, 0x34, 0x85, 0xe4, 0x16, 0x40, 0x7b, 0x1c, 0xdf, 0x8b, 0x54, 0x12, 0x70, 0xb9, 0xa0, + 0x9e, 0x39, 0x31, 0x5a, 0x30, 0x24, 0x9f, 0xc0, 0x2a, 0x86, 0x97, 0x6e, 0x74, 0x71, 0xe3, 0xd9, + 0x7c, 0xe3, 0x5c, 0xf0, 0x74, 0xce, 0xd8, 0x72, 0xbd, 0x06, 0x1b, 0xf3, 0x54, 0x6d, 0xbd, 0xfd, + 0xce, 0x40, 0x62, 0x96, 0x5c, 0x6a, 0x91, 0xdd, 0x30, 0x5b, 0x14, 0xdb, 0x8d, 0x25, 0xb1, 0xe5, + 0xee, 0xca, 0x45, 0x77, 0xba, 0xaa, 0xdd, 0xe0, 0x1b, 0xcb, 0xda, 0xa3, 0x06, 0x68, 0xe9, 0x23, + 0x36, 0x0c, 0x7c, 0xec, 0x2a, 0x8f, 0x1a, 0x60, 0x8f, 0xfe, 0x79, 0x61, 0x5e, 0xf7, 0xfe, 0xdd, + 0xb3, 0xb5, 0xf7, 0xc3, 0xfe, 0x8b, 0x76, 0xa4, 0x64, 0xb3, 0x82, 0xd6, 0x29, 0xd4, 0x9a, 0xfd, + 0x70, 0x8c, 0x9a, 0xaa, 0xd1, 0x58, 0xa8, 0x35, 0x54, 0xbc, 0x44, 0x4d, 0xcd, 0x68, 0x2c, 0xb4, + 0x91, 0xfc, 0xb9, 0x30, 0x92, 0x9b, 0xff, 0xa5, 0x48, 0xc8, 0x25, 0x58, 0xeb, 0x46, 0x2c, 0x96, + 0x23, 0xa1, 0x0c, 0x83, 0x15, 0xd4, 0xcf, 0x0b, 0xb3, 0xcb, 0xb5, 0x91, 0x8a, 0x29, 0x67, 0x03, + 0xdd, 0x65, 0xd7, 0x61, 0x25, 0x15, 0x65, 0x03, 0x3a, 0x7f, 0x91, 0x7b, 0xe9, 0x8a, 0x66, 0x56, + 0xd9, 0x9c, 0xdf, 0x9c, 0x77, 0x25, 0xe3, 0x25, 0x69, 0xbb, 0x0d, 0xb5, 0xf9, 0x5b, 0xf5, 0x41, + 0xe1, 0x56, 0x65, 0x4f, 0x96, 0x36, 0x99, 0xe1, 0xb4, 0xac, 0xcd, 0xdf, 0x8e, 0x63, 0x07, 0x4e, + 0x2d, 0x30, 0x23, 0x97, 0xa1, 0xd2, 0x55, 0x2c, 0x59, 0xce, 0xdc, 0x98, 0x90, 0x0f, 0xc1, 0xbd, + 0x17, 0x0d, 0x70, 0x54, 0x9c, 0x64, 0xa9, 0x0d, 0xf4, 0x3c, 0x7b, 0x68, 0xe7, 0xc8, 0x0d, 0xfb, + 0x22, 0xe4, 0x82, 0xa2, 0x76, 0x0f, 0xab, 0x5a, 0xd0, 0xee, 0x69, 0x2d, 0x92, 0xeb, 0xcd, 0x62, + 0x8e, 0xb3, 0xb7, 0x42, 0x73, 0x81, 0x4e, 0xcf, 0x53, 0x3b, 0xcb, 0xaa, 0x66, 0x96, 0x59, 0x68, + 0xa3, 0xfc, 0xd5, 0x81, 0x33, 0x9f, 0x72, 0xd5, 0x1e, 0xb1, 0x68, 0xc8, 0x07, 0xf8, 0xb0, 0x3e, + 0x0c, 0xa4, 0xd2, 0x55, 0xba, 0x04, 0xe5, 0x5e, 0xd7, 0xa6, 0x6e, 0x31, 0xf7, 0x72, 0xaf, 0x5b, + 0x68, 0x4e, 0xd3, 0x85, 0x69, 0x73, 0x6e, 0x43, 0x23, 0xfb, 0x92, 0x0c, 0x64, 0xd3, 0x43, 0x65, + 0x51, 0xa4, 0x87, 0xa9, 0xf9, 0x97, 0x0c, 0xd2, 0x9e, 0xcc, 0xb0, 0x6e, 0xe2, 0x7b, 0x5f, 0xab, + 0x84, 0x21, 0xe9, 0x55, 0x6a, 0x00, 0xb9, 0x0a, 0x1e, 0x46, 0x59, 0xdb, 0x76, 0x76, 0xd6, 0xf7, + 0xce, 0x15, 0xcb, 0x89, 0xf4, 0x35, 0x73, 0x6d, 0x40, 0xd1, 0xcc, 0x46, 0xf8, 0x87, 0x03, 0x67, + 0x17, 0x46, 0x28, 0x63, 0x72, 0x05, 0xaa, 0x8f, 0xf9, 0x4b, 0x2e, 0x97, 0x17, 0xd3, 0xda, 0xbc, + 0x77, 0x0f, 0x4f, 0x0c, 0xd5, 0x5d, 0x1e, 0xaa, 0x77, 0x52, 0xa8, 0x95, 0x62, 0xa8, 0x57, 0xa0, + 0x7a, 0x18, 0x0e, 0x34, 0xb3, 0xea, 0x32, 0x66, 0xc6, 0xc6, 0x46, 0xfa, 0x10, 0xd6, 0xf1, 0x89, + 0xef, 0x44, 0x2f, 0xf0, 0x77, 0x82, 0xe3, 0x7c, 0xc2, 0xd5, 0x48, 0xa4, 0x4f, 0xaa, 0x45, 0xba, + 0x67, 0x62, 0x96, 0xb0, 0x09, 0x57, 0x3c, 0x49, 0xff, 0x39, 0x99, 0xc0, 0x7a, 0x8b, 0x61, 0xbd, + 0x3b, 0x8b, 0xfc, 0xa3, 0x44, 0x28, 0x6e, 0x1e, 0xfd, 0xf7, 0x3f, 0x8a, 0xa7, 0xa1, 0xf2, 0x99, + 0xe8, 0xdb, 0x2f, 0x68, 0x9d, 0x1a, 0x60, 0x27, 0x0d, 0xf7, 0x95, 0xc9, 0x44, 0x9d, 0xa6, 0x10, + 0x7b, 0x91, 0x85, 0xc1, 0xa0, 0xd7, 0xb5, 0xff, 0x86, 0x14, 0x9a, 0x13, 0x2f, 0xdf, 0x86, 0x8d, + 0xf7, 0x0a, 0x49, 0x36, 0x61, 0x15, 0xef, 0xa0, 0x95, 0x6f, 0x96, 0x08, 0x81, 0xf5, 0xb6, 0x08, + 0x43, 0xee, 0xa7, 0x15, 0xdd, 0x74, 0xf6, 0xb7, 0x5f, 0xff, 0xde, 0x2a, 0xbd, 0x7a, 0xdb, 0x72, + 0x5e, 0xbf, 0x6d, 0x39, 0xbf, 0xbd, 0x6d, 0x95, 0xbe, 0x3b, 0x6e, 0x95, 0x7e, 0x38, 0x6e, 0x39, + 0xaf, 0x8f, 0x5b, 0xa5, 0x5f, 0x8e, 0x5b, 0xa5, 0x7e, 0x15, 0xbf, 0xf9, 0x37, 0xff, 0x0e, 0x00, + 0x00, 0xff, 0xff, 0x41, 0x48, 0xf5, 0xe3, 0x44, 0x0c, 0x00, 0x00, } func (m *AccessInfo) Marshal() (dAtA []byte, err error) { @@ -2799,6 +2864,57 @@ func (m *FaultInjectReq) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *SyncProtection) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SyncProtection) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SyncProtection) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.ValidTS != 0 { + i = encodeVarintOperations(dAtA, i, uint64(m.ValidTS)) + i-- + dAtA[i] = 0x20 + } + if len(m.Objects) > 0 { + for iNdEx := len(m.Objects) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Objects[iNdEx]) + copy(dAtA[i:], m.Objects[iNdEx]) + i = encodeVarintOperations(dAtA, i, uint64(len(m.Objects[iNdEx]))) + i-- + dAtA[i] = 0x1a + } + } + if len(m.JobID) > 0 { + i -= len(m.JobID) + copy(dAtA[i:], m.JobID) + i = encodeVarintOperations(dAtA, i, uint64(len(m.JobID))) + i-- + dAtA[i] = 0x12 + } + if len(m.Op) > 0 { + i -= len(m.Op) + copy(dAtA[i:], m.Op) + i = encodeVarintOperations(dAtA, i, uint64(len(m.Op))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + func encodeVarintOperations(dAtA []byte, offset int, v uint64) int { offset -= sovOperations(v) base := offset @@ -3368,6 +3484,32 @@ func (m *FaultInjectReq) ProtoSize() (n int) { return n } +func (m *SyncProtection) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Op) + if l > 0 { + n += 1 + l + sovOperations(uint64(l)) + } + l = len(m.JobID) + if l > 0 { + n += 1 + l + sovOperations(uint64(l)) + } + if len(m.Objects) > 0 { + for _, s := range m.Objects { + l = len(s) + n += 1 + l + sovOperations(uint64(l)) + } + } + if m.ValidTS != 0 { + n += 1 + sovOperations(uint64(m.ValidTS)) + } + return n +} + func sovOperations(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } @@ -7644,6 +7786,171 @@ func (m *FaultInjectReq) Unmarshal(dAtA []byte) error { } return nil } +func (m *SyncProtection) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowOperations + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SyncProtection: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SyncProtection: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Op", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowOperations + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthOperations + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthOperations + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Op = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field JobID", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowOperations + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthOperations + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthOperations + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.JobID = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Objects", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowOperations + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthOperations + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthOperations + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Objects = append(m.Objects, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ValidTS", wireType) + } + m.ValidTS = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowOperations + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ValidTS |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipOperations(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthOperations + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipOperations(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 diff --git a/pkg/vm/engine/tae/db/gc/v3/exec_v1.go b/pkg/vm/engine/tae/db/gc/v3/exec_v1.go index d2a674788af1e..eb2e70e6bf4fb 100644 --- a/pkg/vm/engine/tae/db/gc/v3/exec_v1.go +++ b/pkg/vm/engine/tae/db/gc/v3/exec_v1.go @@ -73,7 +73,7 @@ type CheckpointBasedGCJob struct { ts *types.TS globalCkpLoc objectio.Location globalCkpVer uint32 - checkpointCli checkpoint.Runner // Added to access catalog + checkpointCli checkpoint.Runner // Added to access catalog syncProtection *SyncProtectionManager // Sync protection manager for cross-cluster sync result struct { diff --git a/pkg/vm/engine/tae/db/gc/v3/tool/sync_protection.go b/pkg/vm/engine/tae/db/gc/v3/tool/sync_protection.go index bed9fe9f6cd8d..115208a6b50b9 100644 --- a/pkg/vm/engine/tae/db/gc/v3/tool/sync_protection.go +++ b/pkg/vm/engine/tae/db/gc/v3/tool/sync_protection.go @@ -37,7 +37,7 @@ import ( // SyncProtectionRequest represents a sync protection request type SyncProtectionRequest struct { JobID string `json:"job_id"` - BF string `json:"bf"` // Base64 encoded BloomFilter + BF string `json:"bf"` // Base64 encoded BloomFilter ValidTS int64 `json:"valid_ts"` TestObject string `json:"test_object"` // Test object name (for debugging) } diff --git a/pkg/vm/engine/tae/rpc/handle_debug.go b/pkg/vm/engine/tae/rpc/handle_debug.go index 306298265a311..da572bf06a854 100644 --- a/pkg/vm/engine/tae/rpc/handle_debug.go +++ b/pkg/vm/engine/tae/rpc/handle_debug.go @@ -800,7 +800,7 @@ func (h *Handle) HandleDiskCleaner( if value == "" { return nil, moerr.NewInvalidArgNoCtx(op, "empty value") } - + // Debug: print received value info logutil.Info( "GC-Sync-Protection-Register-Value-Received", @@ -812,7 +812,7 @@ func (h *Handle) HandleDiskCleaner( return value }()), ) - + var req cmd_util.SyncProtection if err = json.Unmarshal([]byte(value), &req); err != nil { logutil.Error( @@ -822,7 +822,7 @@ func (h *Handle) HandleDiskCleaner( ) return nil, moerr.NewInvalidArgNoCtx(op, value) } - + // Debug: print parsed request info logutil.Info( "GC-Sync-Protection-Register-Parsed", @@ -831,7 +831,7 @@ func (h *Handle) HandleDiskCleaner( zap.Int64("valid-ts", req.ValidTS), zap.String("test-object", req.TestObject), ) - + syncMgr := h.db.DiskCleaner.GetCleaner().GetSyncProtectionManager() if err = syncMgr.RegisterSyncProtection(req.JobID, req.BF, req.ValidTS); err != nil { return nil, err From 26f0bb42eb3ad7a99351957d49256757fd4cf227 Mon Sep 17 00:00:00 2001 From: GreatRiver Date: Wed, 4 Feb 2026 14:31:31 +0800 Subject: [PATCH 10/18] U --- pkg/vm/engine/tae/rpc/handle_debug.go | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/pkg/vm/engine/tae/rpc/handle_debug.go b/pkg/vm/engine/tae/rpc/handle_debug.go index da572bf06a854..c21d1d56dbfc4 100644 --- a/pkg/vm/engine/tae/rpc/handle_debug.go +++ b/pkg/vm/engine/tae/rpc/handle_debug.go @@ -801,18 +801,6 @@ func (h *Handle) HandleDiskCleaner( return nil, moerr.NewInvalidArgNoCtx(op, "empty value") } - // Debug: print received value info - logutil.Info( - "GC-Sync-Protection-Register-Value-Received", - zap.Int("value-len", len(value)), - zap.String("value-prefix", func() string { - if len(value) > 200 { - return value[:200] + "..." - } - return value - }()), - ) - var req cmd_util.SyncProtection if err = json.Unmarshal([]byte(value), &req); err != nil { logutil.Error( @@ -823,15 +811,6 @@ func (h *Handle) HandleDiskCleaner( return nil, moerr.NewInvalidArgNoCtx(op, value) } - // Debug: print parsed request info - logutil.Info( - "GC-Sync-Protection-Register-Parsed", - zap.String("job-id", req.JobID), - zap.Int("bf-len", len(req.BF)), - zap.Int64("valid-ts", req.ValidTS), - zap.String("test-object", req.TestObject), - ) - syncMgr := h.db.DiskCleaner.GetCleaner().GetSyncProtectionManager() if err = syncMgr.RegisterSyncProtection(req.JobID, req.BF, req.ValidTS); err != nil { return nil, err From 9cc05fc48e8e5e1cf1baa107825d3a940b841d88 Mon Sep 17 00:00:00 2001 From: GreatRiver Date: Wed, 4 Feb 2026 14:56:50 +0800 Subject: [PATCH 11/18] U --- pkg/vm/engine/cmd_util/operations.pb.go | 273 +++++++++++------------ pkg/vm/engine/cmd_util/operations.proto | 8 +- pkg/vm/engine/tae/db/gc/v3/checkpoint.go | 29 --- 3 files changed, 139 insertions(+), 171 deletions(-) diff --git a/pkg/vm/engine/cmd_util/operations.pb.go b/pkg/vm/engine/cmd_util/operations.pb.go index dea678cd84b9d..ce1d68131944e 100644 --- a/pkg/vm/engine/cmd_util/operations.pb.go +++ b/pkg/vm/engine/cmd_util/operations.pb.go @@ -1376,13 +1376,6 @@ func (m *SyncProtection) XXX_DiscardUnknown() { var xxx_messageInfo_SyncProtection proto.InternalMessageInfo -func (m *SyncProtection) GetOp() string { - if m != nil { - return m.Op - } - return "" -} - func (m *SyncProtection) GetJobID() string { if m != nil { return m.JobID @@ -1390,11 +1383,11 @@ func (m *SyncProtection) GetJobID() string { return "" } -func (m *SyncProtection) GetObjects() []string { +func (m *SyncProtection) GetBF() string { if m != nil { - return m.Objects + return m.BF } - return nil + return "" } func (m *SyncProtection) GetValidTS() int64 { @@ -1404,6 +1397,13 @@ func (m *SyncProtection) GetValidTS() int64 { return 0 } +func (m *SyncProtection) GetTestObject() string { + if m != nil { + return m.TestObject + } + return "" +} + func init() { proto.RegisterEnum("cmd_util.ChangedListType", ChangedListType_name, ChangedListType_value) proto.RegisterType((*AccessInfo)(nil), "cmd_util.AccessInfo") @@ -1435,85 +1435,86 @@ func init() { func init() { proto.RegisterFile("operations.proto", fileDescriptor_1b4a5877375e491e) } var fileDescriptor_1b4a5877375e491e = []byte{ - // 1243 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x56, 0x3b, 0x93, 0x13, 0xc7, - 0x13, 0xd7, 0x6a, 0x57, 0xd2, 0xa9, 0x75, 0xaf, 0xff, 0xc0, 0x1f, 0x04, 0x85, 0x75, 0x57, 0x14, - 0x65, 0x9f, 0x29, 0x38, 0xe0, 0x28, 0x8a, 0x2a, 0x1c, 0x71, 0x3a, 0x70, 0xc9, 0x3c, 0xee, 0x6a, - 0x24, 0x70, 0x88, 0x47, 0xab, 0x41, 0x5a, 0xb4, 0xda, 0x59, 0xef, 0x8c, 0x8c, 0xe5, 0x4f, 0xe0, - 0xd0, 0x91, 0x63, 0x3b, 0xf4, 0x27, 0x31, 0x21, 0xa1, 0x23, 0x6c, 0x73, 0x81, 0x23, 0x57, 0x39, - 0x26, 0x72, 0x4d, 0xcf, 0xec, 0x43, 0x94, 0x4e, 0x89, 0xcb, 0x81, 0xb3, 0xf9, 0x75, 0xf7, 0xf4, - 0xfc, 0xfa, 0x31, 0x3d, 0x03, 0x9b, 0x22, 0xe6, 0x09, 0x53, 0x81, 0x88, 0xe4, 0x6e, 0x9c, 0x08, - 0x25, 0xc8, 0x8a, 0x3f, 0x19, 0x3c, 0x9b, 0xaa, 0x20, 0x3c, 0x7f, 0x75, 0x18, 0xa8, 0xd1, 0xb4, - 0xbf, 0xeb, 0x8b, 0xc9, 0xb5, 0xa1, 0x18, 0x8a, 0x6b, 0x68, 0xd0, 0x9f, 0x3e, 0x47, 0x84, 0x00, - 0x57, 0x66, 0xe3, 0xf9, 0x0d, 0x15, 0x4c, 0xb8, 0x54, 0x6c, 0x12, 0x1b, 0xc1, 0xc5, 0x2f, 0x00, - 0xee, 0xfa, 0x3e, 0x97, 0xb2, 0x13, 0x3d, 0x17, 0xe4, 0x02, 0xd4, 0xef, 0xfa, 0xbe, 0x98, 0x46, - 0xaa, 0x73, 0xd0, 0x74, 0xb6, 0x9d, 0x9d, 0x35, 0x9a, 0x0b, 0xc8, 0x19, 0xa8, 0x3e, 0x91, 0x3c, - 0xe9, 0x1c, 0x34, 0xcb, 0xa8, 0xb2, 0x48, 0xcb, 0xa9, 0x08, 0x79, 0xe7, 0xa0, 0xe9, 0x1a, 0xb9, - 0x41, 0x77, 0xbc, 0xbf, 0x7e, 0xdc, 0x2a, 0x5d, 0xfc, 0xd6, 0x01, 0xb8, 0x1f, 0x4e, 0xe5, 0xa8, - 0xc7, 0xfa, 0x21, 0x27, 0x77, 0x8a, 0x07, 0xe2, 0x19, 0x8d, 0xbd, 0xd3, 0xbb, 0x69, 0x3c, 0xbb, - 0xb9, 0x6e, 0xdf, 0x7b, 0xf5, 0x66, 0xab, 0x44, 0x8b, 0xf4, 0x5a, 0x00, 0x07, 0x4c, 0xb1, 0x3e, - 0x93, 0xdc, 0x92, 0xf0, 0x68, 0x41, 0x42, 0x9a, 0x50, 0xc3, 0x43, 0x2c, 0x13, 0x8f, 0xa6, 0xd0, - 0x52, 0x79, 0x00, 0x8d, 0x83, 0x40, 0x8e, 0xdb, 0x21, 0x67, 0x11, 0x4f, 0xc8, 0x3a, 0x94, 0x0f, - 0x63, 0xa4, 0x50, 0xa7, 0xe5, 0xc3, 0x98, 0x6c, 0x82, 0xfb, 0x80, 0xcf, 0xd0, 0x6f, 0x9d, 0xea, - 0x25, 0x39, 0x0d, 0x95, 0xa7, 0x2c, 0x9c, 0x72, 0x74, 0x57, 0xa7, 0x06, 0x64, 0xce, 0xa0, 0x3d, - 0xe2, 0xfe, 0x38, 0x16, 0x41, 0xa4, 0xc8, 0x6d, 0x58, 0xc3, 0x20, 0x0f, 0xa6, 0xa6, 0x52, 0xe8, - 0xd6, 0xdd, 0xff, 0xdf, 0xbb, 0x37, 0x5b, 0x6b, 0x3a, 0xe7, 0xbb, 0xa9, 0x82, 0xce, 0xdb, 0x59, - 0x67, 0xb7, 0x60, 0xa3, 0x13, 0x29, 0x9e, 0xf8, 0x3c, 0x56, 0x6d, 0x31, 0x99, 0x04, 0x4a, 0xd7, - 0x02, 0xd9, 0x3f, 0x66, 0x13, 0x6e, 0x49, 0xe6, 0x02, 0xbb, 0x6d, 0x0c, 0xf5, 0x4e, 0x24, 0x63, - 0xee, 0xab, 0xde, 0xe3, 0x7f, 0x94, 0xd9, 0x0b, 0x50, 0x3f, 0x4c, 0x9b, 0xcc, 0x26, 0x20, 0x17, - 0xd8, 0xc3, 0xfa, 0xd0, 0xb0, 0x87, 0x51, 0x2e, 0x63, 0x72, 0x0e, 0xdc, 0xde, 0xcc, 0xa4, 0xaf, - 0xb2, 0x5f, 0x7b, 0xf7, 0x66, 0xcb, 0x0d, 0x22, 0x45, 0xb5, 0x4c, 0xd7, 0xe1, 0x11, 0x97, 0x92, - 0x0d, 0xb9, 0xf5, 0x95, 0x42, 0xad, 0x39, 0x62, 0xb3, 0x50, 0xb0, 0x01, 0xa6, 0x74, 0x95, 0xa6, - 0xd0, 0x9e, 0x71, 0x04, 0x8d, 0x36, 0x53, 0x2c, 0x14, 0x43, 0x3c, 0x83, 0x80, 0xd7, 0x51, 0x7c, - 0x62, 0xc3, 0xc7, 0x35, 0xf9, 0x08, 0xdc, 0xee, 0xb4, 0xdf, 0x2c, 0x6f, 0xbb, 0x3b, 0x8d, 0xbd, - 0xff, 0xe7, 0xf1, 0x15, 0xf6, 0x51, 0x6d, 0x61, 0x3d, 0x7e, 0xaf, 0xdb, 0x8f, 0x4d, 0x43, 0x75, - 0x84, 0x75, 0x22, 0xe0, 0x15, 0x12, 0x8a, 0x6b, 0x2d, 0xbb, 0x9f, 0xf0, 0x2f, 0x2d, 0x57, 0x5c, - 0xeb, 0x9e, 0xbe, 0xeb, 0x63, 0x36, 0x4c, 0xe9, 0x2d, 0x42, 0x46, 0x2c, 0x19, 0x36, 0x3d, 0x5d, - 0x5e, 0x8a, 0x6b, 0x2d, 0xeb, 0x6a, 0x59, 0xc5, 0xec, 0xd7, 0x6b, 0x72, 0x1e, 0x56, 0xda, 0x22, - 0x92, 0x8a, 0x45, 0xaa, 0x59, 0xdd, 0x76, 0x76, 0x56, 0x68, 0x86, 0x2d, 0xb1, 0xcf, 0xa1, 0xde, - 0x4b, 0x98, 0xcf, 0xbb, 0x31, 0x8b, 0x74, 0xeb, 0xf9, 0x93, 0x81, 0x65, 0xa5, 0x97, 0xba, 0xf5, - 0x64, 0xcc, 0x22, 0x69, 0x59, 0x19, 0xa0, 0xeb, 0xa4, 0x46, 0x09, 0x97, 0x23, 0x11, 0x9a, 0x0c, - 0xba, 0x34, 0x17, 0x58, 0xc7, 0x1f, 0xc3, 0xda, 0x7e, 0x28, 0xfc, 0xf1, 0x23, 0xae, 0x18, 0x16, - 0x97, 0x80, 0x17, 0x98, 0x96, 0x70, 0x77, 0x3c, 0x8a, 0x6b, 0x6b, 0xda, 0x81, 0x46, 0x7b, 0x1c, - 0x67, 0x86, 0x4d, 0xa8, 0x7d, 0xc5, 0x13, 0x99, 0xb6, 0xef, 0x1a, 0x4d, 0xa1, 0x0e, 0x27, 0x14, - 0x7e, 0xde, 0x1e, 0xab, 0x34, 0xc3, 0xd6, 0xd5, 0x4f, 0x0e, 0x9c, 0xea, 0x2a, 0x91, 0xb0, 0x21, - 0x7f, 0xa2, 0x4b, 0xad, 0xeb, 0xf0, 0xec, 0xe9, 0x75, 0xed, 0xb3, 0x3b, 0xf5, 0x7d, 0xce, 0x4d, - 0x74, 0x2b, 0x34, 0x85, 0xe4, 0x16, 0x40, 0x7b, 0x1c, 0xdf, 0x8b, 0x54, 0x12, 0x70, 0xb9, 0xa0, - 0x9e, 0x39, 0x31, 0x5a, 0x30, 0x24, 0x9f, 0xc0, 0x2a, 0x86, 0x97, 0x6e, 0x74, 0x71, 0xe3, 0xd9, - 0x7c, 0xe3, 0x5c, 0xf0, 0x74, 0xce, 0xd8, 0x72, 0xbd, 0x06, 0x1b, 0xf3, 0x54, 0x6d, 0xbd, 0xfd, - 0xce, 0x40, 0x62, 0x96, 0x5c, 0x6a, 0x91, 0xdd, 0x30, 0x5b, 0x14, 0xdb, 0x8d, 0x25, 0xb1, 0xe5, - 0xee, 0xca, 0x45, 0x77, 0xba, 0xaa, 0xdd, 0xe0, 0x1b, 0xcb, 0xda, 0xa3, 0x06, 0x68, 0xe9, 0x23, - 0x36, 0x0c, 0x7c, 0xec, 0x2a, 0x8f, 0x1a, 0x60, 0x8f, 0xfe, 0x79, 0x61, 0x5e, 0xf7, 0xfe, 0xdd, - 0xb3, 0xb5, 0xf7, 0xc3, 0xfe, 0x8b, 0x76, 0xa4, 0x64, 0xb3, 0x82, 0xd6, 0x29, 0xd4, 0x9a, 0xfd, - 0x70, 0x8c, 0x9a, 0xaa, 0xd1, 0x58, 0xa8, 0x35, 0x54, 0xbc, 0x44, 0x4d, 0xcd, 0x68, 0x2c, 0xb4, - 0x91, 0xfc, 0xb9, 0x30, 0x92, 0x9b, 0xff, 0xa5, 0x48, 0xc8, 0x25, 0x58, 0xeb, 0x46, 0x2c, 0x96, - 0x23, 0xa1, 0x0c, 0x83, 0x15, 0xd4, 0xcf, 0x0b, 0xb3, 0xcb, 0xb5, 0x91, 0x8a, 0x29, 0x67, 0x03, - 0xdd, 0x65, 0xd7, 0x61, 0x25, 0x15, 0x65, 0x03, 0x3a, 0x7f, 0x91, 0x7b, 0xe9, 0x8a, 0x66, 0x56, - 0xd9, 0x9c, 0xdf, 0x9c, 0x77, 0x25, 0xe3, 0x25, 0x69, 0xbb, 0x0d, 0xb5, 0xf9, 0x5b, 0xf5, 0x41, - 0xe1, 0x56, 0x65, 0x4f, 0x96, 0x36, 0x99, 0xe1, 0xb4, 0xac, 0xcd, 0xdf, 0x8e, 0x63, 0x07, 0x4e, - 0x2d, 0x30, 0x23, 0x97, 0xa1, 0xd2, 0x55, 0x2c, 0x59, 0xce, 0xdc, 0x98, 0x90, 0x0f, 0xc1, 0xbd, - 0x17, 0x0d, 0x70, 0x54, 0x9c, 0x64, 0xa9, 0x0d, 0xf4, 0x3c, 0x7b, 0x68, 0xe7, 0xc8, 0x0d, 0xfb, - 0x22, 0xe4, 0x82, 0xa2, 0x76, 0x0f, 0xab, 0x5a, 0xd0, 0xee, 0x69, 0x2d, 0x92, 0xeb, 0xcd, 0x62, - 0x8e, 0xb3, 0xb7, 0x42, 0x73, 0x81, 0x4e, 0xcf, 0x53, 0x3b, 0xcb, 0xaa, 0x66, 0x96, 0x59, 0x68, - 0xa3, 0xfc, 0xd5, 0x81, 0x33, 0x9f, 0x72, 0xd5, 0x1e, 0xb1, 0x68, 0xc8, 0x07, 0xf8, 0xb0, 0x3e, - 0x0c, 0xa4, 0xd2, 0x55, 0xba, 0x04, 0xe5, 0x5e, 0xd7, 0xa6, 0x6e, 0x31, 0xf7, 0x72, 0xaf, 0x5b, - 0x68, 0x4e, 0xd3, 0x85, 0x69, 0x73, 0x6e, 0x43, 0x23, 0xfb, 0x92, 0x0c, 0x64, 0xd3, 0x43, 0x65, - 0x51, 0xa4, 0x87, 0xa9, 0xf9, 0x97, 0x0c, 0xd2, 0x9e, 0xcc, 0xb0, 0x6e, 0xe2, 0x7b, 0x5f, 0xab, - 0x84, 0x21, 0xe9, 0x55, 0x6a, 0x00, 0xb9, 0x0a, 0x1e, 0x46, 0x59, 0xdb, 0x76, 0x76, 0xd6, 0xf7, - 0xce, 0x15, 0xcb, 0x89, 0xf4, 0x35, 0x73, 0x6d, 0x40, 0xd1, 0xcc, 0x46, 0xf8, 0x87, 0x03, 0x67, - 0x17, 0x46, 0x28, 0x63, 0x72, 0x05, 0xaa, 0x8f, 0xf9, 0x4b, 0x2e, 0x97, 0x17, 0xd3, 0xda, 0xbc, - 0x77, 0x0f, 0x4f, 0x0c, 0xd5, 0x5d, 0x1e, 0xaa, 0x77, 0x52, 0xa8, 0x95, 0x62, 0xa8, 0x57, 0xa0, - 0x7a, 0x18, 0x0e, 0x34, 0xb3, 0xea, 0x32, 0x66, 0xc6, 0xc6, 0x46, 0xfa, 0x10, 0xd6, 0xf1, 0x89, - 0xef, 0x44, 0x2f, 0xf0, 0x77, 0x82, 0xe3, 0x7c, 0xc2, 0xd5, 0x48, 0xa4, 0x4f, 0xaa, 0x45, 0xba, - 0x67, 0x62, 0x96, 0xb0, 0x09, 0x57, 0x3c, 0x49, 0xff, 0x39, 0x99, 0xc0, 0x7a, 0x8b, 0x61, 0xbd, - 0x3b, 0x8b, 0xfc, 0xa3, 0x44, 0x28, 0x6e, 0x1e, 0xfd, 0xf7, 0x3f, 0x8a, 0xa7, 0xa1, 0xf2, 0x99, - 0xe8, 0xdb, 0x2f, 0x68, 0x9d, 0x1a, 0x60, 0x27, 0x0d, 0xf7, 0x95, 0xc9, 0x44, 0x9d, 0xa6, 0x10, - 0x7b, 0x91, 0x85, 0xc1, 0xa0, 0xd7, 0xb5, 0xff, 0x86, 0x14, 0x9a, 0x13, 0x2f, 0xdf, 0x86, 0x8d, - 0xf7, 0x0a, 0x49, 0x36, 0x61, 0x15, 0xef, 0xa0, 0x95, 0x6f, 0x96, 0x08, 0x81, 0xf5, 0xb6, 0x08, - 0x43, 0xee, 0xa7, 0x15, 0xdd, 0x74, 0xf6, 0xb7, 0x5f, 0xff, 0xde, 0x2a, 0xbd, 0x7a, 0xdb, 0x72, - 0x5e, 0xbf, 0x6d, 0x39, 0xbf, 0xbd, 0x6d, 0x95, 0xbe, 0x3b, 0x6e, 0x95, 0x7e, 0x38, 0x6e, 0x39, - 0xaf, 0x8f, 0x5b, 0xa5, 0x5f, 0x8e, 0x5b, 0xa5, 0x7e, 0x15, 0xbf, 0xf9, 0x37, 0xff, 0x0e, 0x00, - 0x00, 0xff, 0xff, 0x41, 0x48, 0xf5, 0xe3, 0x44, 0x0c, 0x00, 0x00, + // 1253 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x56, 0xcd, 0x93, 0x13, 0x45, + 0x14, 0xcf, 0x64, 0x26, 0xc9, 0xe6, 0x65, 0xbf, 0x6c, 0x10, 0x02, 0x85, 0xd9, 0x2d, 0x8a, 0xd2, + 0x95, 0x82, 0x05, 0x96, 0xa2, 0xa8, 0xc2, 0x13, 0xc9, 0xb2, 0x56, 0xe4, 0x63, 0xb7, 0x3a, 0x01, + 0x8f, 0xd8, 0x99, 0x34, 0xc9, 0x90, 0xc9, 0xf4, 0x38, 0xdd, 0x01, 0xe3, 0x5f, 0xe0, 0xd1, 0x93, + 0x67, 0x3d, 0xfa, 0x97, 0xc8, 0x91, 0xa3, 0x27, 0x54, 0xf6, 0xe0, 0xc9, 0x2a, 0xcf, 0x9c, 0xac, + 0x7e, 0xdd, 0x93, 0x99, 0x50, 0xd9, 0x5c, 0x2c, 0x0f, 0xde, 0xfa, 0xf7, 0xde, 0xeb, 0xd7, 0xbf, + 0xf7, 0xd1, 0xaf, 0x1b, 0x36, 0x45, 0xcc, 0x13, 0xa6, 0x02, 0x11, 0xc9, 0xdd, 0x38, 0x11, 0x4a, + 0x90, 0x15, 0x7f, 0xdc, 0x7f, 0x3a, 0x51, 0x41, 0x78, 0xfe, 0xea, 0x20, 0x50, 0xc3, 0x49, 0x6f, + 0xd7, 0x17, 0xe3, 0x6b, 0x03, 0x31, 0x10, 0xd7, 0xd0, 0xa0, 0x37, 0x79, 0x86, 0x08, 0x01, 0xae, + 0xcc, 0xc6, 0xf3, 0x1b, 0x2a, 0x18, 0x73, 0xa9, 0xd8, 0x38, 0x36, 0x82, 0x8b, 0x5f, 0x01, 0xdc, + 0xf5, 0x7d, 0x2e, 0x65, 0x3b, 0x7a, 0x26, 0xc8, 0x05, 0xa8, 0xde, 0xf5, 0x7d, 0x31, 0x89, 0x54, + 0x7b, 0xbf, 0xee, 0x6c, 0x3b, 0x3b, 0x6b, 0x34, 0x13, 0x90, 0x33, 0x50, 0x7e, 0x2c, 0x79, 0xd2, + 0xde, 0xaf, 0x17, 0x51, 0x65, 0x91, 0x96, 0x53, 0x11, 0xf2, 0xf6, 0x7e, 0xdd, 0x35, 0x72, 0x83, + 0xee, 0x78, 0x7f, 0xff, 0xb4, 0x55, 0xb8, 0xf8, 0x9d, 0x03, 0x70, 0x10, 0x4e, 0xe4, 0xb0, 0xcb, + 0x7a, 0x21, 0x27, 0x77, 0xf2, 0x07, 0xe2, 0x19, 0xb5, 0xbd, 0xd3, 0xbb, 0x69, 0x3c, 0xbb, 0x99, + 0xae, 0xe9, 0xbd, 0x7a, 0xb3, 0x55, 0xa0, 0x79, 0x7a, 0x0d, 0x80, 0x7d, 0xa6, 0x58, 0x8f, 0x49, + 0x6e, 0x49, 0x78, 0x34, 0x27, 0x21, 0x75, 0xa8, 0xe0, 0x21, 0x96, 0x89, 0x47, 0x53, 0x68, 0xa9, + 0xdc, 0x87, 0xda, 0x7e, 0x20, 0x47, 0xad, 0x90, 0xb3, 0x88, 0x27, 0x64, 0x1d, 0x8a, 0x87, 0x31, + 0x52, 0xa8, 0xd2, 0xe2, 0x61, 0x4c, 0x36, 0xc1, 0xbd, 0xcf, 0xa7, 0xe8, 0xb7, 0x4a, 0xf5, 0x92, + 0x9c, 0x86, 0xd2, 0x13, 0x16, 0x4e, 0x38, 0xba, 0xab, 0x52, 0x03, 0x66, 0xce, 0xa0, 0x35, 0xe4, + 0xfe, 0x28, 0x16, 0x41, 0xa4, 0xc8, 0x6d, 0x58, 0xc3, 0x20, 0xf7, 0x27, 0xa6, 0x52, 0xe8, 0xd6, + 0x6d, 0x7e, 0xf0, 0xee, 0xcd, 0xd6, 0x9a, 0xce, 0xf9, 0x6e, 0xaa, 0xa0, 0xf3, 0x76, 0xd6, 0xd9, + 0x2d, 0xd8, 0x68, 0x47, 0x8a, 0x27, 0x3e, 0x8f, 0x55, 0x4b, 0x8c, 0xc7, 0x81, 0xd2, 0xb5, 0x40, + 0xf6, 0x8f, 0xd8, 0x98, 0x5b, 0x92, 0x99, 0xc0, 0x6e, 0x1b, 0x41, 0xb5, 0x1d, 0xc9, 0x98, 0xfb, + 0xaa, 0xfb, 0xe8, 0x5f, 0x65, 0xf6, 0x02, 0x54, 0x0f, 0xd3, 0x26, 0xb3, 0x09, 0xc8, 0x04, 0xf6, + 0xb0, 0x1e, 0xd4, 0xec, 0x61, 0x94, 0xcb, 0x98, 0x9c, 0x03, 0xb7, 0x3b, 0x35, 0xe9, 0x2b, 0x35, + 0x2b, 0xef, 0xde, 0x6c, 0xb9, 0x41, 0xa4, 0xa8, 0x96, 0xe9, 0x3a, 0x3c, 0xe4, 0x52, 0xb2, 0x01, + 0xb7, 0xbe, 0x52, 0xa8, 0x35, 0x47, 0x6c, 0x1a, 0x0a, 0xd6, 0xc7, 0x94, 0xae, 0xd2, 0x14, 0xda, + 0x33, 0x8e, 0xa0, 0xd6, 0x62, 0x8a, 0x85, 0x62, 0x80, 0x67, 0x10, 0xf0, 0xda, 0x8a, 0x8f, 0x6d, + 0xf8, 0xb8, 0x26, 0x9f, 0x80, 0xdb, 0x99, 0xf4, 0xea, 0xc5, 0x6d, 0x77, 0xa7, 0xb6, 0xf7, 0x61, + 0x16, 0x5f, 0x6e, 0x1f, 0xd5, 0x16, 0xd6, 0xe3, 0x0f, 0xba, 0xfd, 0xd8, 0x24, 0x54, 0x47, 0x58, + 0x27, 0x02, 0x5e, 0x2e, 0xa1, 0xb8, 0xd6, 0xb2, 0x83, 0x84, 0x7f, 0x6d, 0xb9, 0xe2, 0x5a, 0xf7, + 0xf4, 0x5d, 0x1f, 0xb3, 0x61, 0x4a, 0x6f, 0x11, 0x32, 0x62, 0xc9, 0xa0, 0xee, 0xe9, 0xf2, 0x52, + 0x5c, 0x6b, 0x59, 0x47, 0xcb, 0x4a, 0x66, 0xbf, 0x5e, 0x93, 0xf3, 0xb0, 0xd2, 0x12, 0x91, 0x54, + 0x2c, 0x52, 0xf5, 0xf2, 0xb6, 0xb3, 0xb3, 0x42, 0x67, 0xd8, 0x12, 0xfb, 0x12, 0xaa, 0xdd, 0x84, + 0xf9, 0xbc, 0x13, 0xb3, 0x48, 0xb7, 0x9e, 0x3f, 0xee, 0x5b, 0x56, 0x7a, 0xa9, 0x5b, 0x4f, 0xc6, + 0x2c, 0x92, 0x96, 0x95, 0x01, 0xba, 0x4e, 0x6a, 0x98, 0x70, 0x39, 0x14, 0xa1, 0xc9, 0xa0, 0x4b, + 0x33, 0x81, 0x75, 0xfc, 0x29, 0xac, 0x35, 0x43, 0xe1, 0x8f, 0x1e, 0x72, 0xc5, 0xb0, 0xb8, 0x04, + 0xbc, 0xc0, 0xb4, 0x84, 0xbb, 0xe3, 0x51, 0x5c, 0x5b, 0xd3, 0x36, 0xd4, 0x5a, 0xa3, 0x78, 0x66, + 0x58, 0x87, 0xca, 0x0b, 0x9e, 0xc8, 0xb4, 0x7d, 0xd7, 0x68, 0x0a, 0x75, 0x38, 0xa1, 0xf0, 0xb3, + 0xf6, 0x58, 0xa5, 0x33, 0x6c, 0x5d, 0xfd, 0xec, 0xc0, 0xa9, 0x8e, 0x12, 0x09, 0x1b, 0xf0, 0xc7, + 0xba, 0xd4, 0xba, 0x0e, 0x4f, 0x9f, 0x5c, 0xd7, 0x3e, 0x3b, 0x13, 0xdf, 0xe7, 0xdc, 0x44, 0xb7, + 0x42, 0x53, 0x48, 0x6e, 0x01, 0xb4, 0x46, 0xf1, 0xbd, 0x48, 0x25, 0x01, 0x97, 0x0b, 0xea, 0x99, + 0x11, 0xa3, 0x39, 0x43, 0xf2, 0x19, 0xac, 0x62, 0x78, 0xe9, 0x46, 0x17, 0x37, 0x9e, 0xcd, 0x36, + 0xce, 0x05, 0x4f, 0xe7, 0x8c, 0x2d, 0xd7, 0x6b, 0xb0, 0x31, 0x4f, 0xd5, 0xd6, 0xdb, 0x6f, 0xf7, + 0x25, 0x66, 0xc9, 0xa5, 0x16, 0xd9, 0x0d, 0xd3, 0x45, 0xb1, 0xdd, 0x58, 0x12, 0x5b, 0xe6, 0xae, + 0x98, 0x77, 0xa7, 0xab, 0xda, 0x09, 0xbe, 0xb5, 0xac, 0x3d, 0x6a, 0x80, 0x96, 0x3e, 0x64, 0x83, + 0xc0, 0xc7, 0xae, 0xf2, 0xa8, 0x01, 0xf6, 0xe8, 0x5f, 0x16, 0xe6, 0x75, 0xef, 0xbf, 0x3d, 0x5b, + 0x7b, 0x3f, 0xec, 0x3d, 0x6f, 0x45, 0x4a, 0xd6, 0x4b, 0x68, 0x9d, 0x42, 0xad, 0x69, 0x86, 0x23, + 0xd4, 0x94, 0x8d, 0xc6, 0x42, 0xad, 0xa1, 0xe2, 0x25, 0x6a, 0x2a, 0x46, 0x63, 0xa1, 0x8d, 0xe4, + 0xaf, 0x85, 0x91, 0xdc, 0xfc, 0x3f, 0x45, 0x42, 0x2e, 0xc1, 0x5a, 0x27, 0x62, 0xb1, 0x1c, 0x0a, + 0x65, 0x18, 0xac, 0xa0, 0x7e, 0x5e, 0x38, 0xbb, 0x5c, 0x1b, 0xa9, 0x98, 0x72, 0xd6, 0xd7, 0x5d, + 0x76, 0x1d, 0x56, 0x52, 0xd1, 0x6c, 0x40, 0x67, 0x2f, 0x72, 0x37, 0x5d, 0xd1, 0x99, 0xd5, 0x6c, + 0xce, 0x6f, 0xce, 0xbb, 0x92, 0xf1, 0x92, 0xb4, 0xdd, 0x86, 0xca, 0xfc, 0xad, 0xfa, 0x28, 0x77, + 0xab, 0x66, 0x4f, 0x96, 0x36, 0x99, 0xe2, 0xb4, 0xac, 0xcc, 0xdf, 0x8e, 0x63, 0x07, 0x4e, 0x2d, + 0x30, 0x23, 0x97, 0xa1, 0xd4, 0x51, 0x2c, 0x59, 0xce, 0xdc, 0x98, 0x90, 0x8f, 0xc1, 0xbd, 0x17, + 0xf5, 0x71, 0x54, 0x9c, 0x64, 0xa9, 0x0d, 0xf4, 0x3c, 0x7b, 0x60, 0xe7, 0xc8, 0x0d, 0xfb, 0x22, + 0x64, 0x82, 0xbc, 0x76, 0x0f, 0xab, 0x9a, 0xd3, 0xee, 0x69, 0x2d, 0x92, 0xeb, 0x4e, 0x63, 0x8e, + 0xb3, 0xb7, 0x44, 0x33, 0x81, 0x4e, 0xcf, 0x13, 0x3b, 0xcb, 0xca, 0x66, 0x96, 0x59, 0x68, 0xa3, + 0xfc, 0xcd, 0x81, 0x33, 0x9f, 0x73, 0xd5, 0x1a, 0xb2, 0x68, 0xc0, 0xfb, 0xf8, 0xb0, 0x3e, 0x08, + 0xa4, 0xd2, 0x55, 0xba, 0x04, 0xc5, 0x6e, 0xc7, 0xa6, 0x6e, 0x31, 0xf7, 0x62, 0xb7, 0x93, 0x6b, + 0x4e, 0xd3, 0x85, 0x69, 0x73, 0x6e, 0x43, 0x6d, 0xf6, 0x25, 0xe9, 0xcb, 0xba, 0x87, 0xca, 0xbc, + 0x48, 0x0f, 0x53, 0xf3, 0x2f, 0xe9, 0xa7, 0x3d, 0x39, 0xc3, 0xba, 0x89, 0xef, 0x7d, 0xa3, 0x12, + 0x86, 0xa4, 0x57, 0xa9, 0x01, 0xe4, 0x2a, 0x78, 0x18, 0x65, 0x65, 0xdb, 0xd9, 0x59, 0xdf, 0x3b, + 0x97, 0x2f, 0x27, 0xd2, 0xd7, 0xcc, 0xb5, 0x01, 0x45, 0x33, 0x1b, 0xe1, 0x9f, 0x0e, 0x9c, 0x5d, + 0x18, 0xa1, 0x8c, 0xc9, 0x15, 0x28, 0x3f, 0xe2, 0x2f, 0xb9, 0x5c, 0x5e, 0x4c, 0x6b, 0xf3, 0xde, + 0x3d, 0x3c, 0x31, 0x54, 0x77, 0x79, 0xa8, 0xde, 0x49, 0xa1, 0x96, 0xf2, 0xa1, 0x5e, 0x81, 0xf2, + 0x61, 0xd8, 0xd7, 0xcc, 0xca, 0xcb, 0x98, 0x19, 0x1b, 0x1b, 0xe9, 0x03, 0x58, 0xc7, 0x27, 0xbe, + 0x1d, 0x3d, 0xc7, 0xdf, 0x09, 0x8e, 0xf3, 0x31, 0x57, 0x43, 0x91, 0x3e, 0xa9, 0x16, 0xe9, 0x9e, + 0x89, 0x59, 0xc2, 0xc6, 0x5c, 0xf1, 0x24, 0xfd, 0xe7, 0xcc, 0x04, 0xd6, 0xdb, 0x0b, 0x58, 0xef, + 0x4c, 0x23, 0xff, 0x28, 0x11, 0x8a, 0x9b, 0x47, 0xff, 0x34, 0x94, 0xbe, 0x10, 0x3d, 0xfb, 0x25, + 0xae, 0x52, 0x03, 0xf4, 0xf7, 0xb1, 0x79, 0x60, 0x9d, 0x14, 0x9b, 0x07, 0xd8, 0x71, 0x2c, 0x0c, + 0xfa, 0xdd, 0x8e, 0x7d, 0x99, 0x53, 0xa8, 0xff, 0xad, 0x5d, 0x2e, 0xd5, 0x61, 0x4f, 0xd3, 0xc3, + 0x46, 0xae, 0xd2, 0x9c, 0xc4, 0x9c, 0x7b, 0xf9, 0x36, 0x6c, 0xbc, 0x57, 0x4e, 0xb2, 0x09, 0xab, + 0x78, 0x13, 0xad, 0x7c, 0xb3, 0x40, 0x08, 0xac, 0xb7, 0x44, 0x18, 0x72, 0x3f, 0xad, 0xeb, 0xa6, + 0xd3, 0xdc, 0x7e, 0xfd, 0x47, 0xa3, 0xf0, 0xea, 0x6d, 0xc3, 0x79, 0xfd, 0xb6, 0xe1, 0xfc, 0xfe, + 0xb6, 0x51, 0xf8, 0xfe, 0xb8, 0x51, 0xf8, 0xf1, 0xb8, 0xe1, 0xbc, 0x3e, 0x6e, 0x14, 0x7e, 0x3d, + 0x6e, 0x14, 0x7a, 0x65, 0xfc, 0xec, 0xdf, 0xfc, 0x27, 0x00, 0x00, 0xff, 0xff, 0x8f, 0x84, 0xa2, + 0xbb, 0x4a, 0x0c, 0x00, 0x00, } func (m *AccessInfo) Marshal() (dAtA []byte, err error) { @@ -2884,32 +2885,30 @@ func (m *SyncProtection) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if len(m.TestObject) > 0 { + i -= len(m.TestObject) + copy(dAtA[i:], m.TestObject) + i = encodeVarintOperations(dAtA, i, uint64(len(m.TestObject))) + i-- + dAtA[i] = 0x22 + } if m.ValidTS != 0 { i = encodeVarintOperations(dAtA, i, uint64(m.ValidTS)) i-- - dAtA[i] = 0x20 + dAtA[i] = 0x18 } - if len(m.Objects) > 0 { - for iNdEx := len(m.Objects) - 1; iNdEx >= 0; iNdEx-- { - i -= len(m.Objects[iNdEx]) - copy(dAtA[i:], m.Objects[iNdEx]) - i = encodeVarintOperations(dAtA, i, uint64(len(m.Objects[iNdEx]))) - i-- - dAtA[i] = 0x1a - } + if len(m.BF) > 0 { + i -= len(m.BF) + copy(dAtA[i:], m.BF) + i = encodeVarintOperations(dAtA, i, uint64(len(m.BF))) + i-- + dAtA[i] = 0x12 } if len(m.JobID) > 0 { i -= len(m.JobID) copy(dAtA[i:], m.JobID) i = encodeVarintOperations(dAtA, i, uint64(len(m.JobID))) i-- - dAtA[i] = 0x12 - } - if len(m.Op) > 0 { - i -= len(m.Op) - copy(dAtA[i:], m.Op) - i = encodeVarintOperations(dAtA, i, uint64(len(m.Op))) - i-- dAtA[i] = 0xa } return len(dAtA) - i, nil @@ -3490,23 +3489,21 @@ func (m *SyncProtection) ProtoSize() (n int) { } var l int _ = l - l = len(m.Op) + l = len(m.JobID) if l > 0 { n += 1 + l + sovOperations(uint64(l)) } - l = len(m.JobID) + l = len(m.BF) if l > 0 { n += 1 + l + sovOperations(uint64(l)) } - if len(m.Objects) > 0 { - for _, s := range m.Objects { - l = len(s) - n += 1 + l + sovOperations(uint64(l)) - } - } if m.ValidTS != 0 { n += 1 + sovOperations(uint64(m.ValidTS)) } + l = len(m.TestObject) + if l > 0 { + n += 1 + l + sovOperations(uint64(l)) + } return n } @@ -7817,7 +7814,7 @@ func (m *SyncProtection) Unmarshal(dAtA []byte) error { switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Op", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field JobID", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -7845,11 +7842,11 @@ func (m *SyncProtection) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Op = string(dAtA[iNdEx:postIndex]) + m.JobID = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field JobID", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field BF", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -7877,13 +7874,13 @@ func (m *SyncProtection) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.JobID = string(dAtA[iNdEx:postIndex]) + m.BF = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Objects", wireType) + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ValidTS", wireType) } - var stringLen uint64 + m.ValidTS = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowOperations @@ -7893,29 +7890,16 @@ func (m *SyncProtection) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + m.ValidTS |= int64(b&0x7F) << shift if b < 0x80 { break } } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthOperations - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { - return ErrInvalidLengthOperations - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Objects = append(m.Objects, string(dAtA[iNdEx:postIndex])) - iNdEx = postIndex case 4: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ValidTS", wireType) + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field TestObject", wireType) } - m.ValidTS = 0 + var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowOperations @@ -7925,11 +7909,24 @@ func (m *SyncProtection) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.ValidTS |= int64(b&0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthOperations + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthOperations + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.TestObject = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipOperations(dAtA[iNdEx:]) diff --git a/pkg/vm/engine/cmd_util/operations.proto b/pkg/vm/engine/cmd_util/operations.proto index 9d8bc19fd47bc..13027cf8e041b 100644 --- a/pkg/vm/engine/cmd_util/operations.proto +++ b/pkg/vm/engine/cmd_util/operations.proto @@ -204,8 +204,8 @@ message FaultInjectReq { message SyncProtection { option (gogoproto.typedecl) = false; - string Op = 1; // Operation: register_sync_protection, renew_sync_protection, unregister_sync_protection - string JobID = 2; // Sync job ID - repeated string Objects = 3; // Protected object names (for register) - int64 ValidTS = 4; // Valid timestamp in nanoseconds (for register and renew) + string JobID = 1; // Sync job ID + string BF = 2; // Base64 encoded BloomFilter data (for register) + int64 ValidTS = 3; // Valid timestamp in nanoseconds (for register and renew) + string TestObject = 4; // Test object name for debugging (optional) } \ No newline at end of file diff --git a/pkg/vm/engine/tae/db/gc/v3/checkpoint.go b/pkg/vm/engine/tae/db/gc/v3/checkpoint.go index 74f27b8b96534..50d2235f8cd44 100644 --- a/pkg/vm/engine/tae/db/gc/v3/checkpoint.go +++ b/pkg/vm/engine/tae/db/gc/v3/checkpoint.go @@ -1210,35 +1210,6 @@ func (c *checkpointCleaner) tryGCAgainstGCKPLocked( return } - // Filter out files protected by sync protection before deletion - // This ensures cross-cluster sync operations don't lose their referenced objects - originalCount := len(filesToGC) - - // Debug: print first few files to be deleted - if originalCount > 0 { - logutil.Info( - "GC-Files-To-Delete-Before-Filter", - zap.Int("count", originalCount), - zap.Strings("sample-files", func() []string { - if originalCount <= 5 { - return filesToGC - } - return filesToGC[:5] - }()), - ) - } - - filesToGC = c.syncProtection.FilterProtectedFiles(filesToGC) - if originalCount != len(filesToGC) { - logutil.Info( - "GC-Sync-Protection-Filtered", - zap.String("task", c.TaskNameLocked()), - zap.Int("original-count", originalCount), - zap.Int("filtered-count", len(filesToGC)), - zap.Int("protected-count", originalCount-len(filesToGC)), - ) - } - // Delete files after doGCAgainstGlobalCheckpointLocked // TODO:Requires Physical Removal Policy // Note: Data files are GC'ed normally even when backup protection is active. From e39e7ebc47229edfdc56029844657b73c6c61f2d Mon Sep 17 00:00:00 2001 From: GreatRiver Date: Wed, 4 Feb 2026 15:29:37 +0800 Subject: [PATCH 12/18] U --- pkg/sql/plan/function/ctl/cmd_disk_cleaner.go | 11 ------ .../tae/db/gc/v3/tool/sync_protection.go | 35 ++++++++++--------- 2 files changed, 18 insertions(+), 28 deletions(-) diff --git a/pkg/sql/plan/function/ctl/cmd_disk_cleaner.go b/pkg/sql/plan/function/ctl/cmd_disk_cleaner.go index f7693071db408..428c72dd593eb 100644 --- a/pkg/sql/plan/function/ctl/cmd_disk_cleaner.go +++ b/pkg/sql/plan/function/ctl/cmd_disk_cleaner.go @@ -18,11 +18,9 @@ import ( "github.com/fagongzi/util/protoc" "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/container/types" - "github.com/matrixorigin/matrixone/pkg/logutil" "github.com/matrixorigin/matrixone/pkg/pb/api" "github.com/matrixorigin/matrixone/pkg/vm/engine/cmd_util" "github.com/matrixorigin/matrixone/pkg/vm/process" - "go.uber.org/zap" "strings" ) @@ -67,15 +65,6 @@ func IsValidArg(parameter string, proc *process.Process) (*cmd_util.DiskCleaner, value = strings.Join(parameters[1:], ".") } - // Debug: print parameter info - logutil.Info( - "GC-Sync-Protection-CMD-Parse", - zap.String("op", op), - zap.Int("parameter-len", len(parameter)), - zap.Int("parts-count", len(parameters)), - zap.Int("value-len", len(value)), - ) - return &cmd_util.DiskCleaner{ Op: op, Value: value, diff --git a/pkg/vm/engine/tae/db/gc/v3/tool/sync_protection.go b/pkg/vm/engine/tae/db/gc/v3/tool/sync_protection.go index 115208a6b50b9..51d66bb45198d 100644 --- a/pkg/vm/engine/tae/db/gc/v3/tool/sync_protection.go +++ b/pkg/vm/engine/tae/db/gc/v3/tool/sync_protection.go @@ -25,6 +25,7 @@ import ( "strings" "time" + "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/common/mpool" "github.com/matrixorigin/matrixone/pkg/container/types" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/containers" @@ -56,11 +57,11 @@ type SyncProtectionTester struct { func NewSyncProtectionTester(dsn, dataDir string, sampleCount int, verbose bool, waitTime int) (*SyncProtectionTester, error) { db, err := sql.Open("mysql", dsn) if err != nil { - return nil, fmt.Errorf("failed to connect to database: %w", err) + return nil, moerr.NewInternalErrorNoCtxf("failed to connect to database: %v", err) } if err := db.Ping(); err != nil { - return nil, fmt.Errorf("failed to ping database: %w", err) + return nil, moerr.NewInternalErrorNoCtxf("failed to ping database: %v", err) } return &SyncProtectionTester{ @@ -102,7 +103,7 @@ func (t *SyncProtectionTester) ScanObjectFiles() ([]string, error) { }) if err != nil { - return nil, fmt.Errorf("failed to scan directory: %w", err) + return nil, moerr.NewInternalErrorNoCtxf("failed to scan directory: %v", err) } return objects, nil @@ -139,13 +140,13 @@ func (t *SyncProtectionTester) BuildBloomFilter(objects []string) (string, error // Create BloomFilter using index.NewBloomFilter (xorfilter based) bf, err := index.NewBloomFilter(vec) if err != nil { - return "", fmt.Errorf("failed to create BloomFilter: %w", err) + return "", moerr.NewInternalErrorNoCtxf("failed to create BloomFilter: %v", err) } // Marshal BloomFilter data, err := bf.Marshal() if err != nil { - return "", fmt.Errorf("failed to marshal BloomFilter: %w", err) + return "", moerr.NewInternalErrorNoCtxf("failed to marshal BloomFilter: %v", err) } // Base64 encode @@ -163,7 +164,7 @@ func (t *SyncProtectionTester) RegisterProtection(objects []string) error { // Build BloomFilter bfData, err := t.BuildBloomFilter(objects) if err != nil { - return fmt.Errorf("failed to build BloomFilter: %w", err) + return moerr.NewInternalErrorNoCtxf("failed to build BloomFilter: %v", err) } // Send first protected object name for testing @@ -181,7 +182,7 @@ func (t *SyncProtectionTester) RegisterProtection(objects []string) error { jsonData, err := json.Marshal(req) if err != nil { - return fmt.Errorf("failed to marshal request: %w", err) + return moerr.NewInternalErrorNoCtxf("failed to marshal request: %v", err) } query := fmt.Sprintf("SELECT mo_ctl('dn', 'diskcleaner', 'register_sync_protection.%s')", string(jsonData)) @@ -193,7 +194,7 @@ func (t *SyncProtectionTester) RegisterProtection(objects []string) error { var result string err = t.db.QueryRow(query).Scan(&result) if err != nil { - return fmt.Errorf("failed to register protection: %w", err) + return moerr.NewInternalErrorNoCtxf("failed to register protection: %v", err) } if t.verbose { @@ -202,7 +203,7 @@ func (t *SyncProtectionTester) RegisterProtection(objects []string) error { // Check if successful if strings.Contains(strings.ToLower(result), "error") { - return fmt.Errorf("register protection returned error: %s", result) + return moerr.NewInternalErrorNoCtxf("register protection returned error: %s", result) } t.protectedFiles = objects @@ -218,7 +219,7 @@ func (t *SyncProtectionTester) RenewProtection() error { jsonData, err := json.Marshal(req) if err != nil { - return fmt.Errorf("failed to marshal request: %w", err) + return moerr.NewInternalErrorNoCtxf("failed to marshal request: %v", err) } query := fmt.Sprintf("SELECT mo_ctl('dn', 'diskcleaner', 'renew_sync_protection.%s')", string(jsonData)) @@ -230,7 +231,7 @@ func (t *SyncProtectionTester) RenewProtection() error { var result string err = t.db.QueryRow(query).Scan(&result) if err != nil { - return fmt.Errorf("failed to renew protection: %w", err) + return moerr.NewInternalErrorNoCtxf("failed to renew protection: %v", err) } if t.verbose { @@ -248,7 +249,7 @@ func (t *SyncProtectionTester) UnregisterProtection() error { jsonData, err := json.Marshal(req) if err != nil { - return fmt.Errorf("failed to marshal request: %w", err) + return moerr.NewInternalErrorNoCtxf("failed to marshal request: %v", err) } query := fmt.Sprintf("SELECT mo_ctl('dn', 'diskcleaner', 'unregister_sync_protection.%s')", string(jsonData)) @@ -260,7 +261,7 @@ func (t *SyncProtectionTester) UnregisterProtection() error { var result string err = t.db.QueryRow(query).Scan(&result) if err != nil { - return fmt.Errorf("failed to unregister protection: %w", err) + return moerr.NewInternalErrorNoCtxf("failed to unregister protection: %v", err) } if t.verbose { @@ -281,7 +282,7 @@ func (t *SyncProtectionTester) TriggerGC() error { var result string err := t.db.QueryRow(query).Scan(&result) if err != nil { - return fmt.Errorf("failed to trigger GC: %w", err) + return moerr.NewInternalErrorNoCtxf("failed to trigger GC: %v", err) } if t.verbose { @@ -349,7 +350,7 @@ func (t *SyncProtectionTester) RunTest() error { fmt.Printf(" Found %d object files\n", len(objects)) if len(objects) == 0 { - return fmt.Errorf("no object files found, please check data directory: %s", t.dataDir) + return moerr.NewInternalErrorNoCtxf("no object files found, please check data directory: %s", t.dataDir) } // Step 2: Randomly select objects @@ -368,7 +369,7 @@ func (t *SyncProtectionTester) RunTest() error { // Step 3: Build BloomFilter and register protection fmt.Println("[Step 3] Building BloomFilter and registering sync protection...") if err := t.RegisterProtection(selected); err != nil { - return fmt.Errorf("failed to register protection: %w", err) + return moerr.NewInternalErrorNoCtxf("failed to register protection: %v", err) } registered = true fmt.Println(" ✓ Registration successful!") @@ -412,7 +413,7 @@ func (t *SyncProtectionTester) RunTest() error { } } // Validation failed, stop test - return fmt.Errorf("protection mechanism validation failed: %d protected files were deleted", newlyDeleted) + return moerr.NewInternalErrorNoCtxf("protection mechanism validation failed: %d protected files were deleted", newlyDeleted) } else { fmt.Println(" ✓ [SUCCESS] All protected files were not deleted!") } From 331fb4cf9150405792cf051507a2361df7eda84f Mon Sep 17 00:00:00 2001 From: GreatRiver Date: Wed, 4 Feb 2026 15:52:04 +0800 Subject: [PATCH 13/18] U --- pkg/vm/engine/tae/db/gc/v3/sync_protection.go | 32 ++--- .../tae/db/gc/v3/tool/sync_protection.go | 122 +++++++++--------- 2 files changed, 76 insertions(+), 78 deletions(-) diff --git a/pkg/vm/engine/tae/db/gc/v3/sync_protection.go b/pkg/vm/engine/tae/db/gc/v3/sync_protection.go index e5b1f8df8641f..5929ddac466f2 100644 --- a/pkg/vm/engine/tae/db/gc/v3/sync_protection.go +++ b/pkg/vm/engine/tae/db/gc/v3/sync_protection.go @@ -137,29 +137,25 @@ func (m *SyncProtectionManager) RegisterSyncProtection( } // Unmarshal BloomFilter (using index.BloomFilter which is based on xorfilter - deterministic) - // Use recover to handle panic from invalid data + // Validate minimum buffer length before unmarshal to avoid panic + // Minimum size: 8 (Seed) + 4*4 (SegmentLength, SegmentLengthMask, SegmentCount, SegmentCountLength) = 24 bytes + if len(bfBytes) < 24 { + logutil.Error( + "GC-Sync-Protection-Register-Invalid-BF-Size", + zap.String("job-id", jobID), + zap.Int("size", len(bfBytes)), + ) + return moerr.NewSyncProtectionInvalidNoCtx() + } + var bf index.BloomFilter - var unmarshalErr error - func() { - defer func() { - if r := recover(); r != nil { - logutil.Error( - "GC-Sync-Protection-Register-Unmarshal-Panic", - zap.String("job-id", jobID), - zap.Any("panic", r), - ) - unmarshalErr = moerr.NewSyncProtectionInvalidNoCtx() - } - }() - unmarshalErr = bf.Unmarshal(bfBytes) - }() - if unmarshalErr != nil { + if err = bf.Unmarshal(bfBytes); err != nil { logutil.Error( "GC-Sync-Protection-Register-Unmarshal-Error", zap.String("job-id", jobID), - zap.Error(unmarshalErr), + zap.Error(err), ) - return unmarshalErr + return moerr.NewSyncProtectionInvalidNoCtx() } m.protections[jobID] = &SyncProtection{ diff --git a/pkg/vm/engine/tae/db/gc/v3/tool/sync_protection.go b/pkg/vm/engine/tae/db/gc/v3/tool/sync_protection.go index 51d66bb45198d..a34f31b7cd7cd 100644 --- a/pkg/vm/engine/tae/db/gc/v3/tool/sync_protection.go +++ b/pkg/vm/engine/tae/db/gc/v3/tool/sync_protection.go @@ -28,8 +28,10 @@ import ( "github.com/matrixorigin/matrixone/pkg/common/moerr" "github.com/matrixorigin/matrixone/pkg/common/mpool" "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/logutil" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/containers" "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/index" + "go.uber.org/zap" _ "github.com/go-sql-driver/mysql" "github.com/spf13/cobra" @@ -153,7 +155,10 @@ func (t *SyncProtectionTester) BuildBloomFilter(objects []string) (string, error base64Data := base64.StdEncoding.EncodeToString(data) if t.verbose { - fmt.Printf(" BloomFilter: %d objects, %d bytes, base64 len=%d\n", len(objects), len(data), len(base64Data)) + logutil.Info("GC-Tool-BloomFilter-Built", + zap.Int("objects", len(objects)), + zap.Int("bytes", len(data)), + zap.Int("base64-len", len(base64Data))) } return base64Data, nil @@ -188,7 +193,7 @@ func (t *SyncProtectionTester) RegisterProtection(objects []string) error { query := fmt.Sprintf("SELECT mo_ctl('dn', 'diskcleaner', 'register_sync_protection.%s')", string(jsonData)) if t.verbose { - fmt.Printf(" SQL length: %d\n", len(query)) + logutil.Info("GC-Tool-Register-SQL", zap.Int("length", len(query))) } var result string @@ -198,7 +203,7 @@ func (t *SyncProtectionTester) RegisterProtection(objects []string) error { } if t.verbose { - fmt.Printf(" Result: %s\n", result) + logutil.Info("GC-Tool-Register-Result", zap.String("result", result)) } // Check if successful @@ -225,7 +230,7 @@ func (t *SyncProtectionTester) RenewProtection() error { query := fmt.Sprintf("SELECT mo_ctl('dn', 'diskcleaner', 'renew_sync_protection.%s')", string(jsonData)) if t.verbose { - fmt.Printf(" SQL: %s\n", query) + logutil.Info("GC-Tool-Renew-SQL", zap.String("query", query)) } var result string @@ -235,7 +240,7 @@ func (t *SyncProtectionTester) RenewProtection() error { } if t.verbose { - fmt.Printf(" Result: %s\n", result) + logutil.Info("GC-Tool-Renew-Result", zap.String("result", result)) } return nil @@ -255,7 +260,7 @@ func (t *SyncProtectionTester) UnregisterProtection() error { query := fmt.Sprintf("SELECT mo_ctl('dn', 'diskcleaner', 'unregister_sync_protection.%s')", string(jsonData)) if t.verbose { - fmt.Printf(" SQL: %s\n", query) + logutil.Info("GC-Tool-Unregister-SQL", zap.String("query", query)) } var result string @@ -265,7 +270,7 @@ func (t *SyncProtectionTester) UnregisterProtection() error { } if t.verbose { - fmt.Printf(" Result: %s\n", result) + logutil.Info("GC-Tool-Unregister-Result", zap.String("result", result)) } return nil @@ -276,7 +281,7 @@ func (t *SyncProtectionTester) TriggerGC() error { query := "SELECT mo_ctl('dn', 'diskcleaner', 'force_gc')" if t.verbose { - fmt.Printf(" SQL: %s\n", query) + logutil.Info("GC-Tool-TriggerGC-SQL", zap.String("query", query)) } var result string @@ -286,7 +291,7 @@ func (t *SyncProtectionTester) TriggerGC() error { } if t.verbose { - fmt.Printf(" Result: %s\n", result) + logutil.Info("GC-Tool-TriggerGC-Result", zap.String("result", result)) } return nil @@ -318,88 +323,89 @@ func (t *SyncProtectionTester) CheckFilesExist() (existing, deleted []string) { // RunTest runs the test func (t *SyncProtectionTester) RunTest() error { - fmt.Println("========================================") - fmt.Println("Sync Protection Test (BloomFilter)") - fmt.Println("========================================") - fmt.Printf("Job ID: %s\n", t.jobID) - fmt.Printf("Data directory: %s\n", t.dataDir) - fmt.Printf("Sample count: %d\n", t.sampleCount) - fmt.Printf("Wait time: %d seconds\n", t.waitTime) - fmt.Println() + logutil.Info("GC-Tool-Test-Start", + zap.String("job-id", t.jobID), + zap.String("data-dir", t.dataDir), + zap.Int("sample-count", t.sampleCount), + zap.Int("wait-time", t.waitTime)) // Ensure cleanup on exit registered := false defer func() { if registered { - fmt.Println("[Cleanup] Ensuring protection is unregistered...") + logutil.Info("GC-Tool-Cleanup", zap.String("job-id", t.jobID)) if err := t.UnregisterProtection(); err != nil { - // Ignore error if already unregistered - fmt.Printf(" (Already unregistered or error: %v)\n", err) + logutil.Warn("GC-Tool-Cleanup-Error", zap.Error(err)) } else { - fmt.Println(" ✓ Cleanup successful!") + logutil.Info("GC-Tool-Cleanup-Success") } } }() // Step 1: Scan object files - fmt.Println("[Step 1] Scanning object files...") + logutil.Info("GC-Tool-Step1-Scanning") objects, err := t.ScanObjectFiles() if err != nil { return err } - fmt.Printf(" Found %d object files\n", len(objects)) + logutil.Info("GC-Tool-Step1-Found", zap.Int("count", len(objects))) if len(objects) == 0 { return moerr.NewInternalErrorNoCtxf("no object files found, please check data directory: %s", t.dataDir) } // Step 2: Randomly select objects - fmt.Println("[Step 2] Randomly selecting objects...") + logutil.Info("GC-Tool-Step2-Selecting") selected := t.SelectRandomObjects(objects, t.sampleCount) - fmt.Printf(" Selected %d objects:\n", len(selected)) + logutil.Info("GC-Tool-Step2-Selected", zap.Int("count", len(selected))) for i, obj := range selected { if i < 5 { - fmt.Printf(" - %s\n", obj) + logutil.Info("GC-Tool-Selected-Object", zap.String("name", obj)) } else if i == 5 { - fmt.Printf(" - ... (%d more)\n", len(selected)-5) + logutil.Info("GC-Tool-Selected-More", zap.Int("remaining", len(selected)-5)) break } } // Step 3: Build BloomFilter and register protection - fmt.Println("[Step 3] Building BloomFilter and registering sync protection...") + logutil.Info("GC-Tool-Step3-Registering") if err := t.RegisterProtection(selected); err != nil { return moerr.NewInternalErrorNoCtxf("failed to register protection: %v", err) } registered = true - fmt.Println(" ✓ Registration successful!") + logutil.Info("GC-Tool-Step3-Success") // Step 4: Check initial file status - fmt.Println("[Step 4] Checking initial file status...") + logutil.Info("GC-Tool-Step4-CheckingInitial") existingBefore, deletedBefore := t.CheckFilesExist() - fmt.Printf(" Existing: %d, Deleted: %d\n", len(existingBefore), len(deletedBefore)) + logutil.Info("GC-Tool-Step4-Status", + zap.Int("existing", len(existingBefore)), + zap.Int("deleted", len(deletedBefore))) // Step 5: Trigger GC - fmt.Println("[Step 5] Triggering GC...") + logutil.Info("GC-Tool-Step5-TriggeringGC") if err := t.TriggerGC(); err != nil { - fmt.Printf(" ⚠ Warning: Failed to trigger GC: %v\n", err) + logutil.Warn("GC-Tool-Step5-Warning", zap.Error(err)) } else { - fmt.Println(" ✓ GC triggered successfully!") + logutil.Info("GC-Tool-Step5-Success") } // Wait for GC to complete - fmt.Printf("[Step 6] Waiting for GC to complete (%d seconds)...\n", t.waitTime) + logutil.Info("GC-Tool-Step6-Waiting", zap.Int("seconds", t.waitTime)) time.Sleep(time.Duration(t.waitTime) * time.Second) // Step 7: Check file protection status - fmt.Println("[Step 7] Checking file protection status...") + logutil.Info("GC-Tool-Step7-CheckingProtection") existingAfter, deletedAfter := t.CheckFilesExist() - fmt.Printf(" Existing: %d, Deleted: %d\n", len(existingAfter), len(deletedAfter)) + logutil.Info("GC-Tool-Step7-Status", + zap.Int("existing", len(existingAfter)), + zap.Int("deleted", len(deletedAfter))) // Compare results newlyDeleted := len(deletedAfter) - len(deletedBefore) if newlyDeleted > 0 { - fmt.Printf(" ✗ [FAILED] %d protected files were deleted!\n", newlyDeleted) + logutil.Error("GC-Tool-Step7-FAILED", + zap.Int("deleted-count", newlyDeleted)) for _, f := range deletedAfter { found := false for _, bf := range deletedBefore { @@ -409,54 +415,50 @@ func (t *SyncProtectionTester) RunTest() error { } } if !found { - fmt.Printf(" - Deleted: %s\n", f) + logutil.Error("GC-Tool-Deleted-File", zap.String("name", f)) } } - // Validation failed, stop test return moerr.NewInternalErrorNoCtxf("protection mechanism validation failed: %d protected files were deleted", newlyDeleted) - } else { - fmt.Println(" ✓ [SUCCESS] All protected files were not deleted!") } + logutil.Info("GC-Tool-Step7-SUCCESS") // Step 8: Test renewal - fmt.Println("[Step 8] Testing renewal...") + logutil.Info("GC-Tool-Step8-Renewing") if err := t.RenewProtection(); err != nil { - fmt.Printf(" ⚠ Warning: Renewal failed: %v\n", err) + logutil.Warn("GC-Tool-Step8-Warning", zap.Error(err)) } else { - fmt.Println(" ✓ Renewal successful!") + logutil.Info("GC-Tool-Step8-Success") } // Step 9: Unregister protection - fmt.Println("[Step 9] Unregistering protection (soft delete)...") + logutil.Info("GC-Tool-Step9-Unregistering") if err := t.UnregisterProtection(); err != nil { - fmt.Printf(" ⚠ Warning: Unregister failed: %v\n", err) + logutil.Warn("GC-Tool-Step9-Warning", zap.Error(err)) } else { - registered = false // Mark as unregistered so defer won't try again - fmt.Println(" ✓ Unregister successful!") + registered = false + logutil.Info("GC-Tool-Step9-Success") } // Step 10: Trigger GC again - fmt.Println("[Step 10] Triggering GC again...") + logutil.Info("GC-Tool-Step10-TriggeringGC") if err := t.TriggerGC(); err != nil { - fmt.Printf(" ⚠ Warning: Failed to trigger GC: %v\n", err) + logutil.Warn("GC-Tool-Step10-Warning", zap.Error(err)) } else { - fmt.Println(" ✓ GC triggered successfully!") + logutil.Info("GC-Tool-Step10-Success") } // Wait for GC to complete - fmt.Printf("[Step 11] Waiting for GC to complete (%d seconds)...\n", t.waitTime) + logutil.Info("GC-Tool-Step11-Waiting", zap.Int("seconds", t.waitTime)) time.Sleep(time.Duration(t.waitTime) * time.Second) // Step 12: Final check - fmt.Println("[Step 12] Final check...") + logutil.Info("GC-Tool-Step12-FinalCheck") existingFinal, deletedFinal := t.CheckFilesExist() - fmt.Printf(" Existing: %d, Deleted: %d\n", len(existingFinal), len(deletedFinal)) - - fmt.Println() - fmt.Println("========================================") - fmt.Println("Test completed!") - fmt.Println("========================================") + logutil.Info("GC-Tool-Step12-Status", + zap.Int("existing", len(existingFinal)), + zap.Int("deleted", len(deletedFinal))) + logutil.Info("GC-Tool-Test-Completed") return nil } From dd94dbe337ccfed94decf8974ce515a013e8253c Mon Sep 17 00:00:00 2001 From: GreatRiver Date: Wed, 4 Feb 2026 16:16:03 +0800 Subject: [PATCH 14/18] U --- pkg/vm/engine/tae/db/gc/v3/checkpoint.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/vm/engine/tae/db/gc/v3/checkpoint.go b/pkg/vm/engine/tae/db/gc/v3/checkpoint.go index 50d2235f8cd44..c47d0110dc855 100644 --- a/pkg/vm/engine/tae/db/gc/v3/checkpoint.go +++ b/pkg/vm/engine/tae/db/gc/v3/checkpoint.go @@ -1235,14 +1235,6 @@ func (c *checkpointCleaner) tryGCAgainstGCKPLocked( v2.GCSnapshotDeleteDurationHistogram.Observe(time.Since(deleteStart).Seconds()) } - // Cleanup soft-deleted sync protections when checkpoint watermark > validTS - // This ensures protections are only removed after the checkpoint has recorded the commit - gcWaterMarkEntry := c.GetGCWaterMark() - if gcWaterMarkEntry != nil { - checkpointWatermark := gcWaterMarkEntry.GetEnd().ToTimestamp().PhysicalTime - c.syncProtection.CleanupSoftDeleted(checkpointWatermark) - } - if c.GetGCWaterMark() == nil { return nil } @@ -1856,6 +1848,14 @@ func (c *checkpointCleaner) tryScanLocked( v2.GCErrorIOErrorCounter.Inc() return } + + // Cleanup soft-deleted sync protections when checkpoint watermark > validTS + // This ensures protections are only removed after the checkpoint has recorded the commit + scanWaterMarkEntry := c.GetScanWaterMark() + if scanWaterMarkEntry != nil { + checkpointWatermark := scanWaterMarkEntry.GetEnd().ToTimestamp().PhysicalTime + c.syncProtection.CleanupSoftDeleted(checkpointWatermark) + } return } From 1e01577275c06f89e9cf13c60b2e978287cc40e1 Mon Sep 17 00:00:00 2001 From: GreatRiver Date: Wed, 4 Feb 2026 16:50:00 +0800 Subject: [PATCH 15/18] U --- .../tae/db/gc/v3/tool/sync_protection_test.go | 273 ++++++++++++++++++ 1 file changed, 273 insertions(+) create mode 100644 pkg/vm/engine/tae/db/gc/v3/tool/sync_protection_test.go diff --git a/pkg/vm/engine/tae/db/gc/v3/tool/sync_protection_test.go b/pkg/vm/engine/tae/db/gc/v3/tool/sync_protection_test.go new file mode 100644 index 0000000000000..9de2d9036f492 --- /dev/null +++ b/pkg/vm/engine/tae/db/gc/v3/tool/sync_protection_test.go @@ -0,0 +1,273 @@ +// Copyright 2021 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/base64" + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSelectRandomObjects(t *testing.T) { + tester := &SyncProtectionTester{} + + // Test with count > len(objects) + objects := []string{"obj1", "obj2", "obj3"} + result := tester.SelectRandomObjects(objects, 10) + assert.Equal(t, objects, result) + + // Test with count < len(objects) + result = tester.SelectRandomObjects(objects, 2) + assert.Len(t, result, 2) + + // Test with count == len(objects) + result = tester.SelectRandomObjects(objects, 3) + assert.Len(t, result, 3) + + // Test with empty objects + result = tester.SelectRandomObjects([]string{}, 5) + assert.Empty(t, result) + + // Test with count == 0 + result = tester.SelectRandomObjects(objects, 0) + assert.Empty(t, result) +} + +func TestScanObjectFiles(t *testing.T) { + // Create temp directory + tmpDir, err := os.MkdirTemp("", "gc-tool-test-*") + require.NoError(t, err) + defer os.RemoveAll(tmpDir) + + tester := &SyncProtectionTester{ + dataDir: tmpDir, + } + + // Test empty directory + objects, err := tester.ScanObjectFiles() + require.NoError(t, err) + assert.Empty(t, objects) + + // Create valid object files (42 chars, UUID format with underscore) + validFiles := []string{ + "019c226d-9e98-7ecc-9662-712ff0edcbfb_00000", + "019c226d-9e98-7ecc-9662-712ff0edcbfc_00001", + } + for _, f := range validFiles { + err := os.WriteFile(filepath.Join(tmpDir, f), []byte("test"), 0644) + require.NoError(t, err) + } + + // Create invalid files (should be ignored) + invalidFiles := []string{ + "short.txt", + "no-underscore-in-this-file-name-at-all-xx", + "not-enough-dashes_00000000000000000000000", + } + for _, f := range invalidFiles { + err := os.WriteFile(filepath.Join(tmpDir, f), []byte("test"), 0644) + require.NoError(t, err) + } + + // Scan should only find valid files + objects, err = tester.ScanObjectFiles() + require.NoError(t, err) + assert.Len(t, objects, 2) + assert.Contains(t, objects, validFiles[0]) + assert.Contains(t, objects, validFiles[1]) +} + +func TestScanObjectFiles_Subdirectory(t *testing.T) { + // Create temp directory with subdirectory + tmpDir, err := os.MkdirTemp("", "gc-tool-test-*") + require.NoError(t, err) + defer os.RemoveAll(tmpDir) + + subDir := filepath.Join(tmpDir, "subdir") + err = os.MkdirAll(subDir, 0755) + require.NoError(t, err) + + tester := &SyncProtectionTester{ + dataDir: tmpDir, + } + + // Create valid object file in subdirectory + validFile := "019c226d-9e98-7ecc-9662-712ff0edcbfb_00000" + err = os.WriteFile(filepath.Join(subDir, validFile), []byte("test"), 0644) + require.NoError(t, err) + + // Scan should find file in subdirectory + objects, err := tester.ScanObjectFiles() + require.NoError(t, err) + assert.Len(t, objects, 1) + assert.Contains(t, objects, validFile) +} + +func TestBuildBloomFilter(t *testing.T) { + tester := &SyncProtectionTester{ + verbose: false, + } + + objects := []string{"obj1", "obj2", "obj3"} + bfData, err := tester.BuildBloomFilter(objects) + require.NoError(t, err) + assert.NotEmpty(t, bfData) + + // Verify it's valid base64 + decoded, err := base64.StdEncoding.DecodeString(bfData) + require.NoError(t, err) + assert.NotEmpty(t, decoded) +} + +func TestBuildBloomFilter_Empty(t *testing.T) { + tester := &SyncProtectionTester{ + verbose: false, + } + + // Empty objects should still work (creates empty BF) + bfData, err := tester.BuildBloomFilter([]string{}) + // Note: index.NewBloomFilter may return error for empty input + if err != nil { + // Expected behavior for empty input + return + } + assert.NotEmpty(t, bfData) +} + +func TestBuildBloomFilter_LargeInput(t *testing.T) { + tester := &SyncProtectionTester{ + verbose: false, + } + + // Create many objects + objects := make([]string, 1000) + for i := 0; i < 1000; i++ { + objects[i] = "019c226d-9e98-7ecc-9662-712ff0edcbfb_" + string(rune('0'+i%10)) + string(rune('0'+i/10%10)) + string(rune('0'+i/100%10)) + } + + bfData, err := tester.BuildBloomFilter(objects) + require.NoError(t, err) + assert.NotEmpty(t, bfData) +} + +func TestSyncProtectionRequest_JSON(t *testing.T) { + req := SyncProtectionRequest{ + JobID: "test-job-123", + BF: "base64encodeddata", + ValidTS: 1234567890, + TestObject: "test-obj", + } + + // Marshal + data, err := json.Marshal(req) + require.NoError(t, err) + + // Unmarshal + var decoded SyncProtectionRequest + err = json.Unmarshal(data, &decoded) + require.NoError(t, err) + + assert.Equal(t, req.JobID, decoded.JobID) + assert.Equal(t, req.BF, decoded.BF) + assert.Equal(t, req.ValidTS, decoded.ValidTS) + assert.Equal(t, req.TestObject, decoded.TestObject) +} + +func TestSyncProtectionRequest_JSON_Partial(t *testing.T) { + // Test with only required fields (for renew/unregister) + req := SyncProtectionRequest{ + JobID: "test-job-123", + ValidTS: 1234567890, + } + + data, err := json.Marshal(req) + require.NoError(t, err) + + var decoded SyncProtectionRequest + err = json.Unmarshal(data, &decoded) + require.NoError(t, err) + + assert.Equal(t, req.JobID, decoded.JobID) + assert.Equal(t, req.ValidTS, decoded.ValidTS) + assert.Empty(t, decoded.BF) + assert.Empty(t, decoded.TestObject) +} + +func TestCheckFilesExist(t *testing.T) { + // Create temp directory + tmpDir, err := os.MkdirTemp("", "gc-tool-test-*") + require.NoError(t, err) + defer os.RemoveAll(tmpDir) + + tester := &SyncProtectionTester{ + dataDir: tmpDir, + protectedFiles: []string{"exists.txt", "not-exists.txt"}, + } + + // Create one file + err = os.WriteFile(filepath.Join(tmpDir, "exists.txt"), []byte("test"), 0644) + require.NoError(t, err) + + existing, deleted := tester.CheckFilesExist() + assert.Contains(t, existing, "exists.txt") + assert.Contains(t, deleted, "not-exists.txt") +} + +func TestCheckFilesExist_Empty(t *testing.T) { + tester := &SyncProtectionTester{ + dataDir: "/tmp", + protectedFiles: []string{}, + } + + existing, deleted := tester.CheckFilesExist() + assert.Empty(t, existing) + assert.Empty(t, deleted) +} + +func TestPrepareSyncProtectionCommand(t *testing.T) { + cmd := PrepareSyncProtectionCommand() + + assert.Equal(t, "sync-protection", cmd.Use) + assert.NotEmpty(t, cmd.Short) + assert.NotEmpty(t, cmd.Long) + + // Check flags exist + assert.NotNil(t, cmd.Flags().Lookup("dsn")) + assert.NotNil(t, cmd.Flags().Lookup("data-dir")) + assert.NotNil(t, cmd.Flags().Lookup("sample")) + assert.NotNil(t, cmd.Flags().Lookup("verbose")) + assert.NotNil(t, cmd.Flags().Lookup("wait")) + + // Check default values + dsn, _ := cmd.Flags().GetString("dsn") + assert.Equal(t, "root:111@tcp(127.0.0.1:6001)/", dsn) + + dataDir, _ := cmd.Flags().GetString("data-dir") + assert.Equal(t, "./mo-data/shared", dataDir) + + sample, _ := cmd.Flags().GetInt("sample") + assert.Equal(t, 10, sample) + + verbose, _ := cmd.Flags().GetBool("verbose") + assert.False(t, verbose) + + wait, _ := cmd.Flags().GetInt("wait") + assert.Equal(t, 30, wait) +} From 3adf478669384bd7bf1b1ebcb200d9900338d564 Mon Sep 17 00:00:00 2001 From: GreatRiver Date: Wed, 4 Feb 2026 17:51:49 +0800 Subject: [PATCH 16/18] U --- .../v3/tool/main.go => cmd/mo-tool/gc/gc.go | 21 +- .../mo-tool/gc}/sync_protection.go | 6 +- cmd/mo-tool/main.go | 2 + .../tae/db/gc/v3/tool/sync_protection_test.go | 273 ------------------ 4 files changed, 14 insertions(+), 288 deletions(-) rename pkg/vm/engine/tae/db/gc/v3/tool/main.go => cmd/mo-tool/gc/gc.go (69%) rename {pkg/vm/engine/tae/db/gc/v3/tool => cmd/mo-tool/gc}/sync_protection.go (99%) delete mode 100644 pkg/vm/engine/tae/db/gc/v3/tool/sync_protection_test.go diff --git a/pkg/vm/engine/tae/db/gc/v3/tool/main.go b/cmd/mo-tool/gc/gc.go similarity index 69% rename from pkg/vm/engine/tae/db/gc/v3/tool/main.go rename to cmd/mo-tool/gc/gc.go index 7a58204406370..f2559c81ed895 100644 --- a/pkg/vm/engine/tae/db/gc/v3/tool/main.go +++ b/cmd/mo-tool/gc/gc.go @@ -12,24 +12,21 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package gc import ( - "os" - "github.com/spf13/cobra" ) -func main() { - var rootCmd = &cobra.Command{ - Use: "gc-tool", - Short: "GC testing tool", - Long: "GC testing tool for MatrixOne garbage collection", +// PrepareCommand prepares the gc command +func PrepareCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "gc", + Short: "GC related tools", + Long: "GC related tools for testing and debugging", } - rootCmd.AddCommand(PrepareSyncProtectionCommand()) + cmd.AddCommand(PrepareSyncProtectionCommand()) - if err := rootCmd.Execute(); err != nil { - os.Exit(1) - } + return cmd } diff --git a/pkg/vm/engine/tae/db/gc/v3/tool/sync_protection.go b/cmd/mo-tool/gc/sync_protection.go similarity index 99% rename from pkg/vm/engine/tae/db/gc/v3/tool/sync_protection.go rename to cmd/mo-tool/gc/sync_protection.go index a34f31b7cd7cd..bd8661be27090 100644 --- a/pkg/vm/engine/tae/db/gc/v3/tool/sync_protection.go +++ b/cmd/mo-tool/gc/sync_protection.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package gc import ( "database/sql" @@ -56,7 +56,7 @@ type SyncProtectionTester struct { waitTime int } -func NewSyncProtectionTester(dsn, dataDir string, sampleCount int, verbose bool, waitTime int) (*SyncProtectionTester, error) { +func newSyncProtectionTester(dsn, dataDir string, sampleCount int, verbose bool, waitTime int) (*SyncProtectionTester, error) { db, err := sql.Open("mysql", dsn) if err != nil { return nil, moerr.NewInternalErrorNoCtxf("failed to connect to database: %v", err) @@ -484,7 +484,7 @@ This command will: 4. Trigger GC and verify protected files are not deleted 5. Test renewal and unregister functionality`, RunE: func(cmd *cobra.Command, args []string) error { - tester, err := NewSyncProtectionTester(dsn, dataDir, sampleCount, verbose, waitTime) + tester, err := newSyncProtectionTester(dsn, dataDir, sampleCount, verbose, waitTime) if err != nil { return err } diff --git a/cmd/mo-tool/main.go b/cmd/mo-tool/main.go index 8df6e75f29d51..72aaadf669141 100644 --- a/cmd/mo-tool/main.go +++ b/cmd/mo-tool/main.go @@ -22,6 +22,7 @@ import ( inspect "github.com/matrixorigin/matrixone/cmd/mo-inspect" object "github.com/matrixorigin/matrixone/cmd/mo-object-tool" ckp "github.com/matrixorigin/matrixone/cmd/mo-object-tool/ckp" + gc "github.com/matrixorigin/matrixone/cmd/mo-tool/gc" "github.com/spf13/cobra" ) @@ -37,6 +38,7 @@ func main() { rootCmd.AddCommand(dashboard.PrepareCommand()) rootCmd.AddCommand(object.PrepareCommand()) rootCmd.AddCommand(ckp.PrepareCommand()) + rootCmd.AddCommand(gc.PrepareCommand()) if err := rootCmd.Execute(); err != nil { os.Exit(1) diff --git a/pkg/vm/engine/tae/db/gc/v3/tool/sync_protection_test.go b/pkg/vm/engine/tae/db/gc/v3/tool/sync_protection_test.go deleted file mode 100644 index 9de2d9036f492..0000000000000 --- a/pkg/vm/engine/tae/db/gc/v3/tool/sync_protection_test.go +++ /dev/null @@ -1,273 +0,0 @@ -// Copyright 2021 Matrix Origin -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "encoding/base64" - "encoding/json" - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestSelectRandomObjects(t *testing.T) { - tester := &SyncProtectionTester{} - - // Test with count > len(objects) - objects := []string{"obj1", "obj2", "obj3"} - result := tester.SelectRandomObjects(objects, 10) - assert.Equal(t, objects, result) - - // Test with count < len(objects) - result = tester.SelectRandomObjects(objects, 2) - assert.Len(t, result, 2) - - // Test with count == len(objects) - result = tester.SelectRandomObjects(objects, 3) - assert.Len(t, result, 3) - - // Test with empty objects - result = tester.SelectRandomObjects([]string{}, 5) - assert.Empty(t, result) - - // Test with count == 0 - result = tester.SelectRandomObjects(objects, 0) - assert.Empty(t, result) -} - -func TestScanObjectFiles(t *testing.T) { - // Create temp directory - tmpDir, err := os.MkdirTemp("", "gc-tool-test-*") - require.NoError(t, err) - defer os.RemoveAll(tmpDir) - - tester := &SyncProtectionTester{ - dataDir: tmpDir, - } - - // Test empty directory - objects, err := tester.ScanObjectFiles() - require.NoError(t, err) - assert.Empty(t, objects) - - // Create valid object files (42 chars, UUID format with underscore) - validFiles := []string{ - "019c226d-9e98-7ecc-9662-712ff0edcbfb_00000", - "019c226d-9e98-7ecc-9662-712ff0edcbfc_00001", - } - for _, f := range validFiles { - err := os.WriteFile(filepath.Join(tmpDir, f), []byte("test"), 0644) - require.NoError(t, err) - } - - // Create invalid files (should be ignored) - invalidFiles := []string{ - "short.txt", - "no-underscore-in-this-file-name-at-all-xx", - "not-enough-dashes_00000000000000000000000", - } - for _, f := range invalidFiles { - err := os.WriteFile(filepath.Join(tmpDir, f), []byte("test"), 0644) - require.NoError(t, err) - } - - // Scan should only find valid files - objects, err = tester.ScanObjectFiles() - require.NoError(t, err) - assert.Len(t, objects, 2) - assert.Contains(t, objects, validFiles[0]) - assert.Contains(t, objects, validFiles[1]) -} - -func TestScanObjectFiles_Subdirectory(t *testing.T) { - // Create temp directory with subdirectory - tmpDir, err := os.MkdirTemp("", "gc-tool-test-*") - require.NoError(t, err) - defer os.RemoveAll(tmpDir) - - subDir := filepath.Join(tmpDir, "subdir") - err = os.MkdirAll(subDir, 0755) - require.NoError(t, err) - - tester := &SyncProtectionTester{ - dataDir: tmpDir, - } - - // Create valid object file in subdirectory - validFile := "019c226d-9e98-7ecc-9662-712ff0edcbfb_00000" - err = os.WriteFile(filepath.Join(subDir, validFile), []byte("test"), 0644) - require.NoError(t, err) - - // Scan should find file in subdirectory - objects, err := tester.ScanObjectFiles() - require.NoError(t, err) - assert.Len(t, objects, 1) - assert.Contains(t, objects, validFile) -} - -func TestBuildBloomFilter(t *testing.T) { - tester := &SyncProtectionTester{ - verbose: false, - } - - objects := []string{"obj1", "obj2", "obj3"} - bfData, err := tester.BuildBloomFilter(objects) - require.NoError(t, err) - assert.NotEmpty(t, bfData) - - // Verify it's valid base64 - decoded, err := base64.StdEncoding.DecodeString(bfData) - require.NoError(t, err) - assert.NotEmpty(t, decoded) -} - -func TestBuildBloomFilter_Empty(t *testing.T) { - tester := &SyncProtectionTester{ - verbose: false, - } - - // Empty objects should still work (creates empty BF) - bfData, err := tester.BuildBloomFilter([]string{}) - // Note: index.NewBloomFilter may return error for empty input - if err != nil { - // Expected behavior for empty input - return - } - assert.NotEmpty(t, bfData) -} - -func TestBuildBloomFilter_LargeInput(t *testing.T) { - tester := &SyncProtectionTester{ - verbose: false, - } - - // Create many objects - objects := make([]string, 1000) - for i := 0; i < 1000; i++ { - objects[i] = "019c226d-9e98-7ecc-9662-712ff0edcbfb_" + string(rune('0'+i%10)) + string(rune('0'+i/10%10)) + string(rune('0'+i/100%10)) - } - - bfData, err := tester.BuildBloomFilter(objects) - require.NoError(t, err) - assert.NotEmpty(t, bfData) -} - -func TestSyncProtectionRequest_JSON(t *testing.T) { - req := SyncProtectionRequest{ - JobID: "test-job-123", - BF: "base64encodeddata", - ValidTS: 1234567890, - TestObject: "test-obj", - } - - // Marshal - data, err := json.Marshal(req) - require.NoError(t, err) - - // Unmarshal - var decoded SyncProtectionRequest - err = json.Unmarshal(data, &decoded) - require.NoError(t, err) - - assert.Equal(t, req.JobID, decoded.JobID) - assert.Equal(t, req.BF, decoded.BF) - assert.Equal(t, req.ValidTS, decoded.ValidTS) - assert.Equal(t, req.TestObject, decoded.TestObject) -} - -func TestSyncProtectionRequest_JSON_Partial(t *testing.T) { - // Test with only required fields (for renew/unregister) - req := SyncProtectionRequest{ - JobID: "test-job-123", - ValidTS: 1234567890, - } - - data, err := json.Marshal(req) - require.NoError(t, err) - - var decoded SyncProtectionRequest - err = json.Unmarshal(data, &decoded) - require.NoError(t, err) - - assert.Equal(t, req.JobID, decoded.JobID) - assert.Equal(t, req.ValidTS, decoded.ValidTS) - assert.Empty(t, decoded.BF) - assert.Empty(t, decoded.TestObject) -} - -func TestCheckFilesExist(t *testing.T) { - // Create temp directory - tmpDir, err := os.MkdirTemp("", "gc-tool-test-*") - require.NoError(t, err) - defer os.RemoveAll(tmpDir) - - tester := &SyncProtectionTester{ - dataDir: tmpDir, - protectedFiles: []string{"exists.txt", "not-exists.txt"}, - } - - // Create one file - err = os.WriteFile(filepath.Join(tmpDir, "exists.txt"), []byte("test"), 0644) - require.NoError(t, err) - - existing, deleted := tester.CheckFilesExist() - assert.Contains(t, existing, "exists.txt") - assert.Contains(t, deleted, "not-exists.txt") -} - -func TestCheckFilesExist_Empty(t *testing.T) { - tester := &SyncProtectionTester{ - dataDir: "/tmp", - protectedFiles: []string{}, - } - - existing, deleted := tester.CheckFilesExist() - assert.Empty(t, existing) - assert.Empty(t, deleted) -} - -func TestPrepareSyncProtectionCommand(t *testing.T) { - cmd := PrepareSyncProtectionCommand() - - assert.Equal(t, "sync-protection", cmd.Use) - assert.NotEmpty(t, cmd.Short) - assert.NotEmpty(t, cmd.Long) - - // Check flags exist - assert.NotNil(t, cmd.Flags().Lookup("dsn")) - assert.NotNil(t, cmd.Flags().Lookup("data-dir")) - assert.NotNil(t, cmd.Flags().Lookup("sample")) - assert.NotNil(t, cmd.Flags().Lookup("verbose")) - assert.NotNil(t, cmd.Flags().Lookup("wait")) - - // Check default values - dsn, _ := cmd.Flags().GetString("dsn") - assert.Equal(t, "root:111@tcp(127.0.0.1:6001)/", dsn) - - dataDir, _ := cmd.Flags().GetString("data-dir") - assert.Equal(t, "./mo-data/shared", dataDir) - - sample, _ := cmd.Flags().GetInt("sample") - assert.Equal(t, 10, sample) - - verbose, _ := cmd.Flags().GetBool("verbose") - assert.False(t, verbose) - - wait, _ := cmd.Flags().GetInt("wait") - assert.Equal(t, 30, wait) -} From 3d26b254e97257358a2446b1419c0326c0b5d4e7 Mon Sep 17 00:00:00 2001 From: GreatRiver Date: Wed, 4 Feb 2026 18:55:53 +0800 Subject: [PATCH 17/18] U --- cmd/mo-tool/gc/sync_protection.go | 23 +- cmd/mo-tool/gc/sync_protection_test.go | 393 ++++++++++++++++++ pkg/vm/engine/tae/db/gc/v3/sync_protection.go | 4 +- 3 files changed, 417 insertions(+), 3 deletions(-) create mode 100644 cmd/mo-tool/gc/sync_protection_test.go diff --git a/cmd/mo-tool/gc/sync_protection.go b/cmd/mo-tool/gc/sync_protection.go index bd8661be27090..d8f02fbeb2466 100644 --- a/cmd/mo-tool/gc/sync_protection.go +++ b/cmd/mo-tool/gc/sync_protection.go @@ -37,6 +37,25 @@ import ( "github.com/spf13/cobra" ) +// DBQuerier is an interface for database query operations +type DBQuerier interface { + QueryRow(query string, args ...any) *sql.Row + Close() error +} + +// dbWrapper wraps *sql.DB to implement DBQuerier +type dbWrapper struct { + db *sql.DB +} + +func (w *dbWrapper) QueryRow(query string, args ...any) *sql.Row { + return w.db.QueryRow(query, args...) +} + +func (w *dbWrapper) Close() error { + return w.db.Close() +} + // SyncProtectionRequest represents a sync protection request type SyncProtectionRequest struct { JobID string `json:"job_id"` @@ -47,7 +66,7 @@ type SyncProtectionRequest struct { // SyncProtectionTester tests the sync protection mechanism type SyncProtectionTester struct { - db *sql.DB + db DBQuerier dataDir string jobID string protectedFiles []string @@ -67,7 +86,7 @@ func newSyncProtectionTester(dsn, dataDir string, sampleCount int, verbose bool, } return &SyncProtectionTester{ - db: db, + db: &dbWrapper{db: db}, dataDir: dataDir, jobID: fmt.Sprintf("sync-test-%d", time.Now().UnixNano()), sampleCount: sampleCount, diff --git a/cmd/mo-tool/gc/sync_protection_test.go b/cmd/mo-tool/gc/sync_protection_test.go new file mode 100644 index 0000000000000..4831ffeaedb8e --- /dev/null +++ b/cmd/mo-tool/gc/sync_protection_test.go @@ -0,0 +1,393 @@ +// Copyright 2021 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gc + +import ( + "database/sql" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// mockRow implements a mock for sql.Row +type mockRow struct { + result string + err error +} + +func (r *mockRow) Scan(dest ...any) error { + if r.err != nil { + return r.err + } + if len(dest) > 0 { + if s, ok := dest[0].(*string); ok { + *s = r.result + } + } + return nil +} + +// mockDB implements DBQuerier for testing +type mockDB struct { + queryResult string + queryErr error + closed bool +} + +func (m *mockDB) QueryRow(query string, args ...any) *sql.Row { + // We can't return a real sql.Row, so we'll use a workaround + // by storing the result and checking it in tests + return nil +} + +func (m *mockDB) Close() error { + m.closed = true + return nil +} + +// mockDBWithResult is a more complete mock that can return results +type mockDBWithResult struct { + results map[string]string + errors map[string]error + closed bool +} + +func newMockDB() *mockDBWithResult { + return &mockDBWithResult{ + results: make(map[string]string), + errors: make(map[string]error), + } +} + +func (m *mockDBWithResult) QueryRow(query string, args ...any) *sql.Row { + return nil +} + +func (m *mockDBWithResult) Close() error { + m.closed = true + return nil +} + +func TestSyncProtectionRequest(t *testing.T) { + req := SyncProtectionRequest{ + JobID: "test-job-1", + BF: "base64data", + ValidTS: 1234567890, + TestObject: "test-object", + } + + assert.Equal(t, "test-job-1", req.JobID) + assert.Equal(t, "base64data", req.BF) + assert.Equal(t, int64(1234567890), req.ValidTS) + assert.Equal(t, "test-object", req.TestObject) +} + +func TestSyncProtectionTester_SelectRandomObjects(t *testing.T) { + tester := &SyncProtectionTester{ + sampleCount: 5, + } + + // Test with fewer objects than count + objects := []string{"obj1", "obj2", "obj3"} + selected := tester.SelectRandomObjects(objects, 5) + assert.Equal(t, 3, len(selected)) + + // Test with more objects than count + objects = []string{"obj1", "obj2", "obj3", "obj4", "obj5", "obj6", "obj7", "obj8", "obj9", "obj10"} + selected = tester.SelectRandomObjects(objects, 5) + assert.Equal(t, 5, len(selected)) + + // Verify original slice is not modified + assert.Equal(t, 10, len(objects)) + + // Test with equal count + objects = []string{"obj1", "obj2", "obj3"} + selected = tester.SelectRandomObjects(objects, 3) + assert.Equal(t, 3, len(selected)) +} + +func TestSyncProtectionTester_ScanObjectFiles(t *testing.T) { + // Create temp directory + tmpDir, err := os.MkdirTemp("", "gc-test-*") + require.NoError(t, err) + defer os.RemoveAll(tmpDir) + + tester := &SyncProtectionTester{ + dataDir: tmpDir, + } + + // Test empty directory + objects, err := tester.ScanObjectFiles() + require.NoError(t, err) + assert.Equal(t, 0, len(objects)) + + // Create some test files with valid object name pattern + // Format: 019c226d-9e98-7ecc-9662-712ff0edcbfb_00000 (42 characters) + validNames := []string{ + "019c226d-9e98-7ecc-9662-712ff0edcbfb_00000", + "019c226d-9e98-7ecc-9662-712ff0edcbfb_00001", + } + for _, name := range validNames { + f, err := os.Create(filepath.Join(tmpDir, name)) + require.NoError(t, err) + f.Close() + } + + // Create invalid files (should be ignored) + invalidNames := []string{ + "invalid.txt", + "short", + "no-underscore-but-has-dashes-here", + } + for _, name := range invalidNames { + f, err := os.Create(filepath.Join(tmpDir, name)) + require.NoError(t, err) + f.Close() + } + + // Scan again + objects, err = tester.ScanObjectFiles() + require.NoError(t, err) + assert.Equal(t, 2, len(objects)) +} + +func TestSyncProtectionTester_ScanObjectFiles_WithSubdirectory(t *testing.T) { + // Create temp directory with subdirectory + tmpDir, err := os.MkdirTemp("", "gc-test-*") + require.NoError(t, err) + defer os.RemoveAll(tmpDir) + + subDir := filepath.Join(tmpDir, "subdir") + err = os.Mkdir(subDir, 0755) + require.NoError(t, err) + + // Create file in subdirectory + validName := "019c226d-9e98-7ecc-9662-712ff0edcbfb_00000" + f, err := os.Create(filepath.Join(subDir, validName)) + require.NoError(t, err) + f.Close() + + tester := &SyncProtectionTester{ + dataDir: tmpDir, + } + + objects, err := tester.ScanObjectFiles() + require.NoError(t, err) + assert.Equal(t, 1, len(objects)) +} + +func TestSyncProtectionTester_ScanObjectFiles_NonExistentDir(t *testing.T) { + tester := &SyncProtectionTester{ + dataDir: "/non/existent/path/that/does/not/exist", + } + + _, err := tester.ScanObjectFiles() + assert.Error(t, err) +} + +func TestSyncProtectionTester_BuildBloomFilter(t *testing.T) { + tester := &SyncProtectionTester{ + verbose: false, + } + + objects := []string{ + "019c226d-9e98-7ecc-9662-712ff0edcbfb_00000", + "019c226d-9e98-7ecc-9662-712ff0edcbfb_00001", + "019c226d-9e98-7ecc-9662-712ff0edcbfb_00002", + } + + bfData, err := tester.BuildBloomFilter(objects) + require.NoError(t, err) + assert.NotEmpty(t, bfData) + + // Verify it's valid base64 + assert.NotContains(t, bfData, " ") +} + +func TestSyncProtectionTester_BuildBloomFilter_SingleObject(t *testing.T) { + tester := &SyncProtectionTester{ + verbose: false, + } + + objects := []string{"single-object"} + bfData, err := tester.BuildBloomFilter(objects) + require.NoError(t, err) + assert.NotEmpty(t, bfData) +} + +func TestSyncProtectionTester_BuildBloomFilter_Verbose(t *testing.T) { + tester := &SyncProtectionTester{ + verbose: true, + } + + objects := []string{"obj1", "obj2"} + bfData, err := tester.BuildBloomFilter(objects) + require.NoError(t, err) + assert.NotEmpty(t, bfData) +} + +func TestSyncProtectionTester_CheckFilesExist(t *testing.T) { + // Create temp directory + tmpDir, err := os.MkdirTemp("", "gc-test-*") + require.NoError(t, err) + defer os.RemoveAll(tmpDir) + + // Create some files + existingFile := "019c226d-9e98-7ecc-9662-712ff0edcbfb_00000" + f, err := os.Create(filepath.Join(tmpDir, existingFile)) + require.NoError(t, err) + f.Close() + + tester := &SyncProtectionTester{ + dataDir: tmpDir, + protectedFiles: []string{existingFile, "non-existent-file"}, + } + + existing, deleted := tester.CheckFilesExist() + assert.Equal(t, 1, len(existing)) + assert.Equal(t, 1, len(deleted)) + assert.Contains(t, existing, existingFile) + assert.Contains(t, deleted, "non-existent-file") +} + +func TestSyncProtectionTester_CheckFilesExist_AllExist(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "gc-test-*") + require.NoError(t, err) + defer os.RemoveAll(tmpDir) + + files := []string{"file1", "file2", "file3"} + for _, name := range files { + f, err := os.Create(filepath.Join(tmpDir, name)) + require.NoError(t, err) + f.Close() + } + + tester := &SyncProtectionTester{ + dataDir: tmpDir, + protectedFiles: files, + } + + existing, deleted := tester.CheckFilesExist() + assert.Equal(t, 3, len(existing)) + assert.Equal(t, 0, len(deleted)) +} + +func TestSyncProtectionTester_CheckFilesExist_NoneExist(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "gc-test-*") + require.NoError(t, err) + defer os.RemoveAll(tmpDir) + + tester := &SyncProtectionTester{ + dataDir: tmpDir, + protectedFiles: []string{"non-existent-1", "non-existent-2"}, + } + + existing, deleted := tester.CheckFilesExist() + assert.Equal(t, 0, len(existing)) + assert.Equal(t, 2, len(deleted)) +} + +func TestSyncProtectionTester_CheckFilesExist_EmptyList(t *testing.T) { + tmpDir, err := os.MkdirTemp("", "gc-test-*") + require.NoError(t, err) + defer os.RemoveAll(tmpDir) + + tester := &SyncProtectionTester{ + dataDir: tmpDir, + protectedFiles: []string{}, + } + + existing, deleted := tester.CheckFilesExist() + assert.Equal(t, 0, len(existing)) + assert.Equal(t, 0, len(deleted)) +} + +func TestSyncProtectionTester_Close(t *testing.T) { + // Test with nil db + tester := &SyncProtectionTester{ + db: nil, + } + tester.Close() // Should not panic + + // Test with mock db + mockDB := newMockDB() + tester = &SyncProtectionTester{ + db: mockDB, + } + tester.Close() + assert.True(t, mockDB.closed) +} + +func TestPrepareSyncProtectionCommand(t *testing.T) { + cmd := PrepareSyncProtectionCommand() + + assert.Equal(t, "sync-protection", cmd.Use) + assert.NotEmpty(t, cmd.Short) + assert.NotEmpty(t, cmd.Long) + + // Check flags exist + flags := cmd.Flags() + assert.NotNil(t, flags.Lookup("dsn")) + assert.NotNil(t, flags.Lookup("data-dir")) + assert.NotNil(t, flags.Lookup("sample")) + assert.NotNil(t, flags.Lookup("verbose")) + assert.NotNil(t, flags.Lookup("wait")) + + // Check default values + dsnFlag := flags.Lookup("dsn") + assert.Equal(t, "root:111@tcp(127.0.0.1:6001)/", dsnFlag.DefValue) + + dataDirFlag := flags.Lookup("data-dir") + assert.Equal(t, "./mo-data/shared", dataDirFlag.DefValue) + + sampleFlag := flags.Lookup("sample") + assert.Equal(t, "10", sampleFlag.DefValue) + + verboseFlag := flags.Lookup("verbose") + assert.Equal(t, "false", verboseFlag.DefValue) + + waitFlag := flags.Lookup("wait") + assert.Equal(t, "30", waitFlag.DefValue) +} + +func TestPrepareCommand(t *testing.T) { + cmd := PrepareCommand() + + assert.Equal(t, "gc", cmd.Use) + assert.NotEmpty(t, cmd.Short) + + // Check subcommands + subCmds := cmd.Commands() + assert.GreaterOrEqual(t, len(subCmds), 1) + + // Find sync-protection subcommand + var syncProtectionCmd *bool + for _, sub := range subCmds { + if sub.Use == "sync-protection" { + b := true + syncProtectionCmd = &b + break + } + } + assert.NotNil(t, syncProtectionCmd) +} + +func TestDBWrapper(t *testing.T) { + // Test that dbWrapper implements DBQuerier + var _ DBQuerier = &dbWrapper{} +} diff --git a/pkg/vm/engine/tae/db/gc/v3/sync_protection.go b/pkg/vm/engine/tae/db/gc/v3/sync_protection.go index 5929ddac466f2..cde64cd143d15 100644 --- a/pkg/vm/engine/tae/db/gc/v3/sync_protection.go +++ b/pkg/vm/engine/tae/db/gc/v3/sync_protection.go @@ -32,7 +32,9 @@ const ( DefaultSyncProtectionTTL = 20 * time.Minute // DefaultMaxSyncProtections is the default maximum number of sync protections - DefaultMaxSyncProtections = 100 + // Set to a large value to support many concurrent sync jobs + // Jobs may take ~1.5 hours to be cleaned up after completion + DefaultMaxSyncProtections = 1000000 ) // SyncProtection represents a single sync protection entry From 549b9c0b4ec6f119297e44670c56342cfd6177de Mon Sep 17 00:00:00 2001 From: GreatRiver Date: Thu, 5 Feb 2026 11:01:14 +0800 Subject: [PATCH 18/18] U --- cmd/mo-tool/gc/gc.go | 32 -- cmd/mo-tool/gc/sync_protection.go | 523 ------------------------- cmd/mo-tool/gc/sync_protection_test.go | 393 ------------------- cmd/mo-tool/main.go | 2 - 4 files changed, 950 deletions(-) delete mode 100644 cmd/mo-tool/gc/gc.go delete mode 100644 cmd/mo-tool/gc/sync_protection.go delete mode 100644 cmd/mo-tool/gc/sync_protection_test.go diff --git a/cmd/mo-tool/gc/gc.go b/cmd/mo-tool/gc/gc.go deleted file mode 100644 index f2559c81ed895..0000000000000 --- a/cmd/mo-tool/gc/gc.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2021 Matrix Origin -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package gc - -import ( - "github.com/spf13/cobra" -) - -// PrepareCommand prepares the gc command -func PrepareCommand() *cobra.Command { - cmd := &cobra.Command{ - Use: "gc", - Short: "GC related tools", - Long: "GC related tools for testing and debugging", - } - - cmd.AddCommand(PrepareSyncProtectionCommand()) - - return cmd -} diff --git a/cmd/mo-tool/gc/sync_protection.go b/cmd/mo-tool/gc/sync_protection.go deleted file mode 100644 index d8f02fbeb2466..0000000000000 --- a/cmd/mo-tool/gc/sync_protection.go +++ /dev/null @@ -1,523 +0,0 @@ -// Copyright 2021 Matrix Origin -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package gc - -import ( - "database/sql" - "encoding/base64" - "encoding/json" - "fmt" - "math/rand" - "os" - "path/filepath" - "strings" - "time" - - "github.com/matrixorigin/matrixone/pkg/common/moerr" - "github.com/matrixorigin/matrixone/pkg/common/mpool" - "github.com/matrixorigin/matrixone/pkg/container/types" - "github.com/matrixorigin/matrixone/pkg/logutil" - "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/containers" - "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/index" - "go.uber.org/zap" - - _ "github.com/go-sql-driver/mysql" - "github.com/spf13/cobra" -) - -// DBQuerier is an interface for database query operations -type DBQuerier interface { - QueryRow(query string, args ...any) *sql.Row - Close() error -} - -// dbWrapper wraps *sql.DB to implement DBQuerier -type dbWrapper struct { - db *sql.DB -} - -func (w *dbWrapper) QueryRow(query string, args ...any) *sql.Row { - return w.db.QueryRow(query, args...) -} - -func (w *dbWrapper) Close() error { - return w.db.Close() -} - -// SyncProtectionRequest represents a sync protection request -type SyncProtectionRequest struct { - JobID string `json:"job_id"` - BF string `json:"bf"` // Base64 encoded BloomFilter - ValidTS int64 `json:"valid_ts"` - TestObject string `json:"test_object"` // Test object name (for debugging) -} - -// SyncProtectionTester tests the sync protection mechanism -type SyncProtectionTester struct { - db DBQuerier - dataDir string - jobID string - protectedFiles []string - sampleCount int - verbose bool - waitTime int -} - -func newSyncProtectionTester(dsn, dataDir string, sampleCount int, verbose bool, waitTime int) (*SyncProtectionTester, error) { - db, err := sql.Open("mysql", dsn) - if err != nil { - return nil, moerr.NewInternalErrorNoCtxf("failed to connect to database: %v", err) - } - - if err := db.Ping(); err != nil { - return nil, moerr.NewInternalErrorNoCtxf("failed to ping database: %v", err) - } - - return &SyncProtectionTester{ - db: &dbWrapper{db: db}, - dataDir: dataDir, - jobID: fmt.Sprintf("sync-test-%d", time.Now().UnixNano()), - sampleCount: sampleCount, - verbose: verbose, - waitTime: waitTime, - }, nil -} - -func (t *SyncProtectionTester) Close() { - if t.db != nil { - t.db.Close() - } -} - -// ScanObjectFiles scans the directory for object files -func (t *SyncProtectionTester) ScanObjectFiles() ([]string, error) { - var objects []string - - err := filepath.Walk(t.dataDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - return nil - } - - // Match object file name pattern - name := info.Name() - // MatrixOne object files are typically UUID format with underscore - // Format: 019c226d-9e98-7ecc-9662-712ff0edcbfb_00000 (42 characters) - if len(name) == 42 && strings.Contains(name, "_") && strings.Count(name, "-") == 4 { - objects = append(objects, name) - } - return nil - }) - - if err != nil { - return nil, moerr.NewInternalErrorNoCtxf("failed to scan directory: %v", err) - } - - return objects, nil -} - -// SelectRandomObjects randomly selects objects -func (t *SyncProtectionTester) SelectRandomObjects(objects []string, count int) []string { - if len(objects) <= count { - return objects - } - - // Copy slice to avoid modifying original data - copied := make([]string, len(objects)) - copy(copied, objects) - - // Shuffle randomly - rand.Shuffle(len(copied), func(i, j int) { - copied[i], copied[j] = copied[j], copied[i] - }) - - return copied[:count] -} - -// BuildBloomFilter builds a BloomFilter using xorfilter (deterministic hash) -func (t *SyncProtectionTester) BuildBloomFilter(objects []string) (string, error) { - // Create a containers.Vector with all object names - vec := containers.MakeVector(types.T_varchar.ToType(), mpool.MustNewZero()) - defer vec.Close() - - for _, obj := range objects { - vec.Append([]byte(obj), false) - } - - // Create BloomFilter using index.NewBloomFilter (xorfilter based) - bf, err := index.NewBloomFilter(vec) - if err != nil { - return "", moerr.NewInternalErrorNoCtxf("failed to create BloomFilter: %v", err) - } - - // Marshal BloomFilter - data, err := bf.Marshal() - if err != nil { - return "", moerr.NewInternalErrorNoCtxf("failed to marshal BloomFilter: %v", err) - } - - // Base64 encode - base64Data := base64.StdEncoding.EncodeToString(data) - - if t.verbose { - logutil.Info("GC-Tool-BloomFilter-Built", - zap.Int("objects", len(objects)), - zap.Int("bytes", len(data)), - zap.Int("base64-len", len(base64Data))) - } - - return base64Data, nil -} - -// RegisterProtection registers protection -func (t *SyncProtectionTester) RegisterProtection(objects []string) error { - // Build BloomFilter - bfData, err := t.BuildBloomFilter(objects) - if err != nil { - return moerr.NewInternalErrorNoCtxf("failed to build BloomFilter: %v", err) - } - - // Send first protected object name for testing - testObject := "" - if len(objects) > 0 { - testObject = objects[0] - } - - req := SyncProtectionRequest{ - JobID: t.jobID, - BF: bfData, - ValidTS: time.Now().UnixNano(), - TestObject: testObject, - } - - jsonData, err := json.Marshal(req) - if err != nil { - return moerr.NewInternalErrorNoCtxf("failed to marshal request: %v", err) - } - - query := fmt.Sprintf("SELECT mo_ctl('dn', 'diskcleaner', 'register_sync_protection.%s')", string(jsonData)) - - if t.verbose { - logutil.Info("GC-Tool-Register-SQL", zap.Int("length", len(query))) - } - - var result string - err = t.db.QueryRow(query).Scan(&result) - if err != nil { - return moerr.NewInternalErrorNoCtxf("failed to register protection: %v", err) - } - - if t.verbose { - logutil.Info("GC-Tool-Register-Result", zap.String("result", result)) - } - - // Check if successful - if strings.Contains(strings.ToLower(result), "error") { - return moerr.NewInternalErrorNoCtxf("register protection returned error: %s", result) - } - - t.protectedFiles = objects - return nil -} - -// RenewProtection renews protection -func (t *SyncProtectionTester) RenewProtection() error { - req := SyncProtectionRequest{ - JobID: t.jobID, - ValidTS: time.Now().UnixNano(), - } - - jsonData, err := json.Marshal(req) - if err != nil { - return moerr.NewInternalErrorNoCtxf("failed to marshal request: %v", err) - } - - query := fmt.Sprintf("SELECT mo_ctl('dn', 'diskcleaner', 'renew_sync_protection.%s')", string(jsonData)) - - if t.verbose { - logutil.Info("GC-Tool-Renew-SQL", zap.String("query", query)) - } - - var result string - err = t.db.QueryRow(query).Scan(&result) - if err != nil { - return moerr.NewInternalErrorNoCtxf("failed to renew protection: %v", err) - } - - if t.verbose { - logutil.Info("GC-Tool-Renew-Result", zap.String("result", result)) - } - - return nil -} - -// UnregisterProtection unregisters protection -func (t *SyncProtectionTester) UnregisterProtection() error { - req := SyncProtectionRequest{ - JobID: t.jobID, - } - - jsonData, err := json.Marshal(req) - if err != nil { - return moerr.NewInternalErrorNoCtxf("failed to marshal request: %v", err) - } - - query := fmt.Sprintf("SELECT mo_ctl('dn', 'diskcleaner', 'unregister_sync_protection.%s')", string(jsonData)) - - if t.verbose { - logutil.Info("GC-Tool-Unregister-SQL", zap.String("query", query)) - } - - var result string - err = t.db.QueryRow(query).Scan(&result) - if err != nil { - return moerr.NewInternalErrorNoCtxf("failed to unregister protection: %v", err) - } - - if t.verbose { - logutil.Info("GC-Tool-Unregister-Result", zap.String("result", result)) - } - - return nil -} - -// TriggerGC triggers GC -func (t *SyncProtectionTester) TriggerGC() error { - query := "SELECT mo_ctl('dn', 'diskcleaner', 'force_gc')" - - if t.verbose { - logutil.Info("GC-Tool-TriggerGC-SQL", zap.String("query", query)) - } - - var result string - err := t.db.QueryRow(query).Scan(&result) - if err != nil { - return moerr.NewInternalErrorNoCtxf("failed to trigger GC: %v", err) - } - - if t.verbose { - logutil.Info("GC-Tool-TriggerGC-Result", zap.String("result", result)) - } - - return nil -} - -// CheckFilesExist checks if files exist -func (t *SyncProtectionTester) CheckFilesExist() (existing, deleted []string) { - for _, file := range t.protectedFiles { - // Search for file in data directory - found := false - filepath.Walk(t.dataDir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return nil - } - if info.Name() == file { - found = true - return filepath.SkipAll - } - return nil - }) - if found { - existing = append(existing, file) - } else { - deleted = append(deleted, file) - } - } - return -} - -// RunTest runs the test -func (t *SyncProtectionTester) RunTest() error { - logutil.Info("GC-Tool-Test-Start", - zap.String("job-id", t.jobID), - zap.String("data-dir", t.dataDir), - zap.Int("sample-count", t.sampleCount), - zap.Int("wait-time", t.waitTime)) - - // Ensure cleanup on exit - registered := false - defer func() { - if registered { - logutil.Info("GC-Tool-Cleanup", zap.String("job-id", t.jobID)) - if err := t.UnregisterProtection(); err != nil { - logutil.Warn("GC-Tool-Cleanup-Error", zap.Error(err)) - } else { - logutil.Info("GC-Tool-Cleanup-Success") - } - } - }() - - // Step 1: Scan object files - logutil.Info("GC-Tool-Step1-Scanning") - objects, err := t.ScanObjectFiles() - if err != nil { - return err - } - logutil.Info("GC-Tool-Step1-Found", zap.Int("count", len(objects))) - - if len(objects) == 0 { - return moerr.NewInternalErrorNoCtxf("no object files found, please check data directory: %s", t.dataDir) - } - - // Step 2: Randomly select objects - logutil.Info("GC-Tool-Step2-Selecting") - selected := t.SelectRandomObjects(objects, t.sampleCount) - logutil.Info("GC-Tool-Step2-Selected", zap.Int("count", len(selected))) - for i, obj := range selected { - if i < 5 { - logutil.Info("GC-Tool-Selected-Object", zap.String("name", obj)) - } else if i == 5 { - logutil.Info("GC-Tool-Selected-More", zap.Int("remaining", len(selected)-5)) - break - } - } - - // Step 3: Build BloomFilter and register protection - logutil.Info("GC-Tool-Step3-Registering") - if err := t.RegisterProtection(selected); err != nil { - return moerr.NewInternalErrorNoCtxf("failed to register protection: %v", err) - } - registered = true - logutil.Info("GC-Tool-Step3-Success") - - // Step 4: Check initial file status - logutil.Info("GC-Tool-Step4-CheckingInitial") - existingBefore, deletedBefore := t.CheckFilesExist() - logutil.Info("GC-Tool-Step4-Status", - zap.Int("existing", len(existingBefore)), - zap.Int("deleted", len(deletedBefore))) - - // Step 5: Trigger GC - logutil.Info("GC-Tool-Step5-TriggeringGC") - if err := t.TriggerGC(); err != nil { - logutil.Warn("GC-Tool-Step5-Warning", zap.Error(err)) - } else { - logutil.Info("GC-Tool-Step5-Success") - } - - // Wait for GC to complete - logutil.Info("GC-Tool-Step6-Waiting", zap.Int("seconds", t.waitTime)) - time.Sleep(time.Duration(t.waitTime) * time.Second) - - // Step 7: Check file protection status - logutil.Info("GC-Tool-Step7-CheckingProtection") - existingAfter, deletedAfter := t.CheckFilesExist() - logutil.Info("GC-Tool-Step7-Status", - zap.Int("existing", len(existingAfter)), - zap.Int("deleted", len(deletedAfter))) - - // Compare results - newlyDeleted := len(deletedAfter) - len(deletedBefore) - if newlyDeleted > 0 { - logutil.Error("GC-Tool-Step7-FAILED", - zap.Int("deleted-count", newlyDeleted)) - for _, f := range deletedAfter { - found := false - for _, bf := range deletedBefore { - if f == bf { - found = true - break - } - } - if !found { - logutil.Error("GC-Tool-Deleted-File", zap.String("name", f)) - } - } - return moerr.NewInternalErrorNoCtxf("protection mechanism validation failed: %d protected files were deleted", newlyDeleted) - } - logutil.Info("GC-Tool-Step7-SUCCESS") - - // Step 8: Test renewal - logutil.Info("GC-Tool-Step8-Renewing") - if err := t.RenewProtection(); err != nil { - logutil.Warn("GC-Tool-Step8-Warning", zap.Error(err)) - } else { - logutil.Info("GC-Tool-Step8-Success") - } - - // Step 9: Unregister protection - logutil.Info("GC-Tool-Step9-Unregistering") - if err := t.UnregisterProtection(); err != nil { - logutil.Warn("GC-Tool-Step9-Warning", zap.Error(err)) - } else { - registered = false - logutil.Info("GC-Tool-Step9-Success") - } - - // Step 10: Trigger GC again - logutil.Info("GC-Tool-Step10-TriggeringGC") - if err := t.TriggerGC(); err != nil { - logutil.Warn("GC-Tool-Step10-Warning", zap.Error(err)) - } else { - logutil.Info("GC-Tool-Step10-Success") - } - - // Wait for GC to complete - logutil.Info("GC-Tool-Step11-Waiting", zap.Int("seconds", t.waitTime)) - time.Sleep(time.Duration(t.waitTime) * time.Second) - - // Step 12: Final check - logutil.Info("GC-Tool-Step12-FinalCheck") - existingFinal, deletedFinal := t.CheckFilesExist() - logutil.Info("GC-Tool-Step12-Status", - zap.Int("existing", len(existingFinal)), - zap.Int("deleted", len(deletedFinal))) - - logutil.Info("GC-Tool-Test-Completed") - return nil -} - -// PrepareSyncProtectionCommand prepares the sync protection test command -func PrepareSyncProtectionCommand() *cobra.Command { - var ( - dsn string - dataDir string - sampleCount int - verbose bool - waitTime int - ) - - cmd := &cobra.Command{ - Use: "sync-protection", - Short: "Test sync protection mechanism", - Long: `Test cross-cluster sync protection mechanism. - -This command will: -1. Scan the specified directory for object files -2. Randomly select some objects to build a BloomFilter -3. Register BloomFilter protection -4. Trigger GC and verify protected files are not deleted -5. Test renewal and unregister functionality`, - RunE: func(cmd *cobra.Command, args []string) error { - tester, err := newSyncProtectionTester(dsn, dataDir, sampleCount, verbose, waitTime) - if err != nil { - return err - } - defer tester.Close() - - return tester.RunTest() - }, - } - - cmd.Flags().StringVar(&dsn, "dsn", "root:111@tcp(127.0.0.1:6001)/", "Database connection string") - cmd.Flags().StringVar(&dataDir, "data-dir", "./mo-data/shared", "Data directory path") - cmd.Flags().IntVar(&sampleCount, "sample", 10, "Number of objects to sample") - cmd.Flags().BoolVar(&verbose, "verbose", false, "Show verbose output") - cmd.Flags().IntVar(&waitTime, "wait", 30, "Time to wait for GC to complete (seconds)") - - return cmd -} diff --git a/cmd/mo-tool/gc/sync_protection_test.go b/cmd/mo-tool/gc/sync_protection_test.go deleted file mode 100644 index 4831ffeaedb8e..0000000000000 --- a/cmd/mo-tool/gc/sync_protection_test.go +++ /dev/null @@ -1,393 +0,0 @@ -// Copyright 2021 Matrix Origin -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package gc - -import ( - "database/sql" - "os" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// mockRow implements a mock for sql.Row -type mockRow struct { - result string - err error -} - -func (r *mockRow) Scan(dest ...any) error { - if r.err != nil { - return r.err - } - if len(dest) > 0 { - if s, ok := dest[0].(*string); ok { - *s = r.result - } - } - return nil -} - -// mockDB implements DBQuerier for testing -type mockDB struct { - queryResult string - queryErr error - closed bool -} - -func (m *mockDB) QueryRow(query string, args ...any) *sql.Row { - // We can't return a real sql.Row, so we'll use a workaround - // by storing the result and checking it in tests - return nil -} - -func (m *mockDB) Close() error { - m.closed = true - return nil -} - -// mockDBWithResult is a more complete mock that can return results -type mockDBWithResult struct { - results map[string]string - errors map[string]error - closed bool -} - -func newMockDB() *mockDBWithResult { - return &mockDBWithResult{ - results: make(map[string]string), - errors: make(map[string]error), - } -} - -func (m *mockDBWithResult) QueryRow(query string, args ...any) *sql.Row { - return nil -} - -func (m *mockDBWithResult) Close() error { - m.closed = true - return nil -} - -func TestSyncProtectionRequest(t *testing.T) { - req := SyncProtectionRequest{ - JobID: "test-job-1", - BF: "base64data", - ValidTS: 1234567890, - TestObject: "test-object", - } - - assert.Equal(t, "test-job-1", req.JobID) - assert.Equal(t, "base64data", req.BF) - assert.Equal(t, int64(1234567890), req.ValidTS) - assert.Equal(t, "test-object", req.TestObject) -} - -func TestSyncProtectionTester_SelectRandomObjects(t *testing.T) { - tester := &SyncProtectionTester{ - sampleCount: 5, - } - - // Test with fewer objects than count - objects := []string{"obj1", "obj2", "obj3"} - selected := tester.SelectRandomObjects(objects, 5) - assert.Equal(t, 3, len(selected)) - - // Test with more objects than count - objects = []string{"obj1", "obj2", "obj3", "obj4", "obj5", "obj6", "obj7", "obj8", "obj9", "obj10"} - selected = tester.SelectRandomObjects(objects, 5) - assert.Equal(t, 5, len(selected)) - - // Verify original slice is not modified - assert.Equal(t, 10, len(objects)) - - // Test with equal count - objects = []string{"obj1", "obj2", "obj3"} - selected = tester.SelectRandomObjects(objects, 3) - assert.Equal(t, 3, len(selected)) -} - -func TestSyncProtectionTester_ScanObjectFiles(t *testing.T) { - // Create temp directory - tmpDir, err := os.MkdirTemp("", "gc-test-*") - require.NoError(t, err) - defer os.RemoveAll(tmpDir) - - tester := &SyncProtectionTester{ - dataDir: tmpDir, - } - - // Test empty directory - objects, err := tester.ScanObjectFiles() - require.NoError(t, err) - assert.Equal(t, 0, len(objects)) - - // Create some test files with valid object name pattern - // Format: 019c226d-9e98-7ecc-9662-712ff0edcbfb_00000 (42 characters) - validNames := []string{ - "019c226d-9e98-7ecc-9662-712ff0edcbfb_00000", - "019c226d-9e98-7ecc-9662-712ff0edcbfb_00001", - } - for _, name := range validNames { - f, err := os.Create(filepath.Join(tmpDir, name)) - require.NoError(t, err) - f.Close() - } - - // Create invalid files (should be ignored) - invalidNames := []string{ - "invalid.txt", - "short", - "no-underscore-but-has-dashes-here", - } - for _, name := range invalidNames { - f, err := os.Create(filepath.Join(tmpDir, name)) - require.NoError(t, err) - f.Close() - } - - // Scan again - objects, err = tester.ScanObjectFiles() - require.NoError(t, err) - assert.Equal(t, 2, len(objects)) -} - -func TestSyncProtectionTester_ScanObjectFiles_WithSubdirectory(t *testing.T) { - // Create temp directory with subdirectory - tmpDir, err := os.MkdirTemp("", "gc-test-*") - require.NoError(t, err) - defer os.RemoveAll(tmpDir) - - subDir := filepath.Join(tmpDir, "subdir") - err = os.Mkdir(subDir, 0755) - require.NoError(t, err) - - // Create file in subdirectory - validName := "019c226d-9e98-7ecc-9662-712ff0edcbfb_00000" - f, err := os.Create(filepath.Join(subDir, validName)) - require.NoError(t, err) - f.Close() - - tester := &SyncProtectionTester{ - dataDir: tmpDir, - } - - objects, err := tester.ScanObjectFiles() - require.NoError(t, err) - assert.Equal(t, 1, len(objects)) -} - -func TestSyncProtectionTester_ScanObjectFiles_NonExistentDir(t *testing.T) { - tester := &SyncProtectionTester{ - dataDir: "/non/existent/path/that/does/not/exist", - } - - _, err := tester.ScanObjectFiles() - assert.Error(t, err) -} - -func TestSyncProtectionTester_BuildBloomFilter(t *testing.T) { - tester := &SyncProtectionTester{ - verbose: false, - } - - objects := []string{ - "019c226d-9e98-7ecc-9662-712ff0edcbfb_00000", - "019c226d-9e98-7ecc-9662-712ff0edcbfb_00001", - "019c226d-9e98-7ecc-9662-712ff0edcbfb_00002", - } - - bfData, err := tester.BuildBloomFilter(objects) - require.NoError(t, err) - assert.NotEmpty(t, bfData) - - // Verify it's valid base64 - assert.NotContains(t, bfData, " ") -} - -func TestSyncProtectionTester_BuildBloomFilter_SingleObject(t *testing.T) { - tester := &SyncProtectionTester{ - verbose: false, - } - - objects := []string{"single-object"} - bfData, err := tester.BuildBloomFilter(objects) - require.NoError(t, err) - assert.NotEmpty(t, bfData) -} - -func TestSyncProtectionTester_BuildBloomFilter_Verbose(t *testing.T) { - tester := &SyncProtectionTester{ - verbose: true, - } - - objects := []string{"obj1", "obj2"} - bfData, err := tester.BuildBloomFilter(objects) - require.NoError(t, err) - assert.NotEmpty(t, bfData) -} - -func TestSyncProtectionTester_CheckFilesExist(t *testing.T) { - // Create temp directory - tmpDir, err := os.MkdirTemp("", "gc-test-*") - require.NoError(t, err) - defer os.RemoveAll(tmpDir) - - // Create some files - existingFile := "019c226d-9e98-7ecc-9662-712ff0edcbfb_00000" - f, err := os.Create(filepath.Join(tmpDir, existingFile)) - require.NoError(t, err) - f.Close() - - tester := &SyncProtectionTester{ - dataDir: tmpDir, - protectedFiles: []string{existingFile, "non-existent-file"}, - } - - existing, deleted := tester.CheckFilesExist() - assert.Equal(t, 1, len(existing)) - assert.Equal(t, 1, len(deleted)) - assert.Contains(t, existing, existingFile) - assert.Contains(t, deleted, "non-existent-file") -} - -func TestSyncProtectionTester_CheckFilesExist_AllExist(t *testing.T) { - tmpDir, err := os.MkdirTemp("", "gc-test-*") - require.NoError(t, err) - defer os.RemoveAll(tmpDir) - - files := []string{"file1", "file2", "file3"} - for _, name := range files { - f, err := os.Create(filepath.Join(tmpDir, name)) - require.NoError(t, err) - f.Close() - } - - tester := &SyncProtectionTester{ - dataDir: tmpDir, - protectedFiles: files, - } - - existing, deleted := tester.CheckFilesExist() - assert.Equal(t, 3, len(existing)) - assert.Equal(t, 0, len(deleted)) -} - -func TestSyncProtectionTester_CheckFilesExist_NoneExist(t *testing.T) { - tmpDir, err := os.MkdirTemp("", "gc-test-*") - require.NoError(t, err) - defer os.RemoveAll(tmpDir) - - tester := &SyncProtectionTester{ - dataDir: tmpDir, - protectedFiles: []string{"non-existent-1", "non-existent-2"}, - } - - existing, deleted := tester.CheckFilesExist() - assert.Equal(t, 0, len(existing)) - assert.Equal(t, 2, len(deleted)) -} - -func TestSyncProtectionTester_CheckFilesExist_EmptyList(t *testing.T) { - tmpDir, err := os.MkdirTemp("", "gc-test-*") - require.NoError(t, err) - defer os.RemoveAll(tmpDir) - - tester := &SyncProtectionTester{ - dataDir: tmpDir, - protectedFiles: []string{}, - } - - existing, deleted := tester.CheckFilesExist() - assert.Equal(t, 0, len(existing)) - assert.Equal(t, 0, len(deleted)) -} - -func TestSyncProtectionTester_Close(t *testing.T) { - // Test with nil db - tester := &SyncProtectionTester{ - db: nil, - } - tester.Close() // Should not panic - - // Test with mock db - mockDB := newMockDB() - tester = &SyncProtectionTester{ - db: mockDB, - } - tester.Close() - assert.True(t, mockDB.closed) -} - -func TestPrepareSyncProtectionCommand(t *testing.T) { - cmd := PrepareSyncProtectionCommand() - - assert.Equal(t, "sync-protection", cmd.Use) - assert.NotEmpty(t, cmd.Short) - assert.NotEmpty(t, cmd.Long) - - // Check flags exist - flags := cmd.Flags() - assert.NotNil(t, flags.Lookup("dsn")) - assert.NotNil(t, flags.Lookup("data-dir")) - assert.NotNil(t, flags.Lookup("sample")) - assert.NotNil(t, flags.Lookup("verbose")) - assert.NotNil(t, flags.Lookup("wait")) - - // Check default values - dsnFlag := flags.Lookup("dsn") - assert.Equal(t, "root:111@tcp(127.0.0.1:6001)/", dsnFlag.DefValue) - - dataDirFlag := flags.Lookup("data-dir") - assert.Equal(t, "./mo-data/shared", dataDirFlag.DefValue) - - sampleFlag := flags.Lookup("sample") - assert.Equal(t, "10", sampleFlag.DefValue) - - verboseFlag := flags.Lookup("verbose") - assert.Equal(t, "false", verboseFlag.DefValue) - - waitFlag := flags.Lookup("wait") - assert.Equal(t, "30", waitFlag.DefValue) -} - -func TestPrepareCommand(t *testing.T) { - cmd := PrepareCommand() - - assert.Equal(t, "gc", cmd.Use) - assert.NotEmpty(t, cmd.Short) - - // Check subcommands - subCmds := cmd.Commands() - assert.GreaterOrEqual(t, len(subCmds), 1) - - // Find sync-protection subcommand - var syncProtectionCmd *bool - for _, sub := range subCmds { - if sub.Use == "sync-protection" { - b := true - syncProtectionCmd = &b - break - } - } - assert.NotNil(t, syncProtectionCmd) -} - -func TestDBWrapper(t *testing.T) { - // Test that dbWrapper implements DBQuerier - var _ DBQuerier = &dbWrapper{} -} diff --git a/cmd/mo-tool/main.go b/cmd/mo-tool/main.go index 72aaadf669141..8df6e75f29d51 100644 --- a/cmd/mo-tool/main.go +++ b/cmd/mo-tool/main.go @@ -22,7 +22,6 @@ import ( inspect "github.com/matrixorigin/matrixone/cmd/mo-inspect" object "github.com/matrixorigin/matrixone/cmd/mo-object-tool" ckp "github.com/matrixorigin/matrixone/cmd/mo-object-tool/ckp" - gc "github.com/matrixorigin/matrixone/cmd/mo-tool/gc" "github.com/spf13/cobra" ) @@ -38,7 +37,6 @@ func main() { rootCmd.AddCommand(dashboard.PrepareCommand()) rootCmd.AddCommand(object.PrepareCommand()) rootCmd.AddCommand(ckp.PrepareCommand()) - rootCmd.AddCommand(gc.PrepareCommand()) if err := rootCmd.Execute(); err != nil { os.Exit(1)