diff --git a/pkg/common/moerr/error.go b/pkg/common/moerr/error.go index 014a64d488b0e..40b146576edce 100644 --- a/pkg/common/moerr/error.go +++ b/pkg/common/moerr/error.go @@ -251,6 +251,14 @@ const ( // ErrSchedulerClosed scheduler has been closed, cannot schedule new jobs ErrSchedulerClosed uint16 = 20641 + // GC sync protection errors + ErrGCIsRunning uint16 = 20642 + ErrSyncProtectionNotFound uint16 = 20643 + ErrSyncProtectionExists uint16 = 20644 + ErrSyncProtectionMaxCount uint16 = 20645 + ErrSyncProtectionSoftDelete uint16 = 20646 + ErrSyncProtectionInvalid uint16 = 20647 + // Group 7: lock service // ErrDeadLockDetected lockservice has detected a deadlock and should abort the transaction if it receives this error ErrDeadLockDetected uint16 = 20701 @@ -505,6 +513,14 @@ var errorMsgRefer = map[uint16]moErrorMsgItem{ ErrOfflineTxnWrite: {ER_UNKNOWN_ERROR, []string{MySQLDefaultSqlState}, "write offline txn: %s"}, ErrSchedulerClosed: {ER_UNKNOWN_ERROR, []string{MySQLDefaultSqlState}, "scheduler closed"}, + // GC sync protection errors + ErrGCIsRunning: {ER_UNKNOWN_ERROR, []string{MySQLDefaultSqlState}, "GC is running, please retry later"}, + ErrSyncProtectionNotFound: {ER_UNKNOWN_ERROR, []string{MySQLDefaultSqlState}, "sync protection not found: %s"}, + ErrSyncProtectionExists: {ER_UNKNOWN_ERROR, []string{MySQLDefaultSqlState}, "sync protection already exists: %s"}, + ErrSyncProtectionMaxCount: {ER_UNKNOWN_ERROR, []string{MySQLDefaultSqlState}, "sync protection max count reached: %d"}, + ErrSyncProtectionSoftDelete: {ER_UNKNOWN_ERROR, []string{MySQLDefaultSqlState}, "sync protection is soft deleted: %s"}, + ErrSyncProtectionInvalid: {ER_UNKNOWN_ERROR, []string{MySQLDefaultSqlState}, "invalid sync protection request"}, + // Group 7: lock service ErrDeadLockDetected: {ER_UNKNOWN_ERROR, []string{MySQLDefaultSqlState}, "deadlock detected"}, ErrLockTableBindChanged: {ER_UNKNOWN_ERROR, []string{MySQLDefaultSqlState}, "lock table bind changed"}, diff --git a/pkg/common/moerr/error_no_ctx.go b/pkg/common/moerr/error_no_ctx.go index 6f2cd474ad6b9..64f5414dcc327 100644 --- a/pkg/common/moerr/error_no_ctx.go +++ b/pkg/common/moerr/error_no_ctx.go @@ -499,3 +499,28 @@ func NewCantCompileForPrepareNoCtx() *Error { func NewSchedulerClosedNoCtx() *Error { return newError(Context(), ErrSchedulerClosed) } + +// GC sync protection errors +func NewGCIsRunningNoCtx() *Error { + return newError(Context(), ErrGCIsRunning) +} + +func NewSyncProtectionNotFoundNoCtx(jobID string) *Error { + return newError(Context(), ErrSyncProtectionNotFound, jobID) +} + +func NewSyncProtectionExistsNoCtx(jobID string) *Error { + return newError(Context(), ErrSyncProtectionExists, jobID) +} + +func NewSyncProtectionMaxCountNoCtx(maxCount int) *Error { + return newError(Context(), ErrSyncProtectionMaxCount, maxCount) +} + +func NewSyncProtectionSoftDeleteNoCtx(jobID string) *Error { + return newError(Context(), ErrSyncProtectionSoftDelete, jobID) +} + +func NewSyncProtectionInvalidNoCtx() *Error { + return newError(Context(), ErrSyncProtectionInvalid) +} diff --git a/pkg/sql/plan/function/ctl/cmd_disk_cleaner.go b/pkg/sql/plan/function/ctl/cmd_disk_cleaner.go index 24c65f17cfbd4..428c72dd593eb 100644 --- a/pkg/sql/plan/function/ctl/cmd_disk_cleaner.go +++ b/pkg/sql/plan/function/ctl/cmd_disk_cleaner.go @@ -26,13 +26,16 @@ import ( func IsValidArg(parameter string, proc *process.Process) (*cmd_util.DiskCleaner, error) { parameters := strings.Split(parameter, ".") - if len(parameters) > 3 || len(parameters) < 1 { + if len(parameters) < 1 { return nil, moerr.NewInternalError(proc.Ctx, "handleDiskCleaner: invalid argument!") } op := parameters[0] switch op { case cmd_util.AddChecker, cmd_util.RemoveChecker: - break + // These operations need key validation, check parameter count later + if len(parameters) > 3 { + return nil, moerr.NewInternalError(proc.Ctx, "handleDiskCleaner: invalid argument!") + } case cmd_util.StopGC, cmd_util.StartGC: return &cmd_util.DiskCleaner{ Op: op, @@ -52,6 +55,20 @@ func IsValidArg(parameter string, proc *process.Process) (*cmd_util.DiskCleaner, Op: op, Key: cmd_util.GCVerify, }, nil + case cmd_util.RegisterSyncProtection, cmd_util.RenewSyncProtection, cmd_util.UnregisterSyncProtection: + // Sync protection operations expect JSON value in the second parameter + // Format: register_sync_protection.{"job_id":"xxx","objects":["obj1"],"valid_ts":123} + // Note: JSON may contain dots, so we join all remaining parts + value := "" + if len(parameters) > 1 { + // Join remaining parts as JSON value (in case JSON contains dots) + value = strings.Join(parameters[1:], ".") + } + + return &cmd_util.DiskCleaner{ + Op: op, + Value: value, + }, nil default: return nil, moerr.NewInternalError(proc.Ctx, "handleDiskCleaner: invalid operation!") } diff --git a/pkg/vm/engine/cmd_util/operations.go b/pkg/vm/engine/cmd_util/operations.go index f6345f368a9a7..0710135490033 100644 --- a/pkg/vm/engine/cmd_util/operations.go +++ b/pkg/vm/engine/cmd_util/operations.go @@ -424,3 +424,11 @@ func (f *FaultInjectReq) MarshalBinary() ([]byte, error) { func (f *FaultInjectReq) UnmarshalBinary(data []byte) error { return f.Unmarshal(data) } + +// SyncProtection is the request for sync protection operations +type SyncProtection struct { + JobID string `json:"job_id"` // Sync job ID + BF string `json:"bf"` // Base64 encoded BloomFilter data (for register) + ValidTS int64 `json:"valid_ts"` // Valid timestamp in nanoseconds (for register and renew) + TestObject string `json:"test_object"` // Test object name for debugging (optional) +} diff --git a/pkg/vm/engine/cmd_util/operations.pb.go b/pkg/vm/engine/cmd_util/operations.pb.go index acfcab731f067..ce1d68131944e 100644 --- a/pkg/vm/engine/cmd_util/operations.pb.go +++ b/pkg/vm/engine/cmd_util/operations.pb.go @@ -1343,6 +1343,67 @@ func (m *FaultInjectReq) GetParameter() string { return "" } +func (m *SyncProtection) Reset() { *m = SyncProtection{} } +func (m *SyncProtection) String() string { return proto.CompactTextString(m) } +func (*SyncProtection) ProtoMessage() {} +func (*SyncProtection) Descriptor() ([]byte, []int) { + return fileDescriptor_1b4a5877375e491e, []int{23} +} +func (m *SyncProtection) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *SyncProtection) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_SyncProtection.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *SyncProtection) XXX_Merge(src proto.Message) { + xxx_messageInfo_SyncProtection.Merge(m, src) +} +func (m *SyncProtection) XXX_Size() int { + return m.ProtoSize() +} +func (m *SyncProtection) XXX_DiscardUnknown() { + xxx_messageInfo_SyncProtection.DiscardUnknown(m) +} + +var xxx_messageInfo_SyncProtection proto.InternalMessageInfo + +func (m *SyncProtection) GetJobID() string { + if m != nil { + return m.JobID + } + return "" +} + +func (m *SyncProtection) GetBF() string { + if m != nil { + return m.BF + } + return "" +} + +func (m *SyncProtection) GetValidTS() int64 { + if m != nil { + return m.ValidTS + } + return 0 +} + +func (m *SyncProtection) GetTestObject() string { + if m != nil { + return m.TestObject + } + return "" +} + func init() { proto.RegisterEnum("cmd_util.ChangedListType", ChangedListType_name, ChangedListType_value) proto.RegisterType((*AccessInfo)(nil), "cmd_util.AccessInfo") @@ -1368,87 +1429,92 @@ func init() { proto.RegisterType((*GetChangedTableListReq)(nil), "cmd_util.GetChangedTableListReq") proto.RegisterType((*GetChangedTableListResp)(nil), "cmd_util.GetChangedTableListResp") proto.RegisterType((*FaultInjectReq)(nil), "cmd_util.FaultInjectReq") + proto.RegisterType((*SyncProtection)(nil), "cmd_util.SyncProtection") } func init() { proto.RegisterFile("operations.proto", fileDescriptor_1b4a5877375e491e) } var fileDescriptor_1b4a5877375e491e = []byte{ - // 1195 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x56, 0xbd, 0xaf, 0x13, 0x47, - 0x10, 0xf7, 0xf9, 0xce, 0xf6, 0xf3, 0xf8, 0x7d, 0x38, 0x0b, 0x01, 0x83, 0x88, 0x9f, 0x85, 0x50, - 0xf2, 0x82, 0xe0, 0x01, 0x46, 0x08, 0x89, 0x54, 0xd8, 0x86, 0xc8, 0xe2, 0xe3, 0xa1, 0xb5, 0x21, - 0x25, 0x59, 0x9f, 0x17, 0xfb, 0xf0, 0xf9, 0xee, 0x72, 0xbb, 0x0e, 0x71, 0xfe, 0x82, 0x94, 0xa9, - 0x52, 0x27, 0x65, 0xfe, 0x92, 0x50, 0x52, 0xa6, 0x22, 0x09, 0xaf, 0x48, 0x15, 0x29, 0x35, 0x55, - 0xb4, 0xb3, 0x7b, 0x1f, 0x46, 0xc6, 0x4d, 0x94, 0x22, 0xdd, 0xfe, 0x66, 0x66, 0x67, 0x7f, 0xbf, - 0x99, 0xb9, 0xdd, 0x83, 0x7a, 0x18, 0xf1, 0x98, 0x49, 0x2f, 0x0c, 0xc4, 0x61, 0x14, 0x87, 0x32, - 0x24, 0x5b, 0xee, 0x7c, 0xfc, 0x74, 0x21, 0x3d, 0xff, 0xec, 0xe5, 0x89, 0x27, 0xa7, 0x8b, 0xd1, - 0xa1, 0x1b, 0xce, 0xaf, 0x4c, 0xc2, 0x49, 0x78, 0x05, 0x03, 0x46, 0x8b, 0x67, 0x88, 0x10, 0xe0, - 0x4a, 0x6f, 0x3c, 0xbb, 0x27, 0xbd, 0x39, 0x17, 0x92, 0xcd, 0x23, 0x6d, 0x38, 0xff, 0x25, 0xc0, - 0x6d, 0xd7, 0xe5, 0x42, 0xf4, 0x83, 0x67, 0x21, 0x39, 0x07, 0xd5, 0xdb, 0xae, 0x1b, 0x2e, 0x02, - 0xd9, 0xef, 0x35, 0xac, 0x96, 0x75, 0xb0, 0x43, 0x33, 0x03, 0x39, 0x05, 0xe5, 0xc7, 0x82, 0xc7, - 0xfd, 0x5e, 0xa3, 0x88, 0x2e, 0x83, 0x94, 0x9d, 0x86, 0x3e, 0xef, 0xf7, 0x1a, 0xb6, 0xb6, 0x6b, - 0x74, 0xcb, 0xf9, 0xfb, 0xa7, 0xfd, 0xc2, 0xf9, 0xef, 0x2c, 0x80, 0xbb, 0xfe, 0x42, 0x4c, 0x87, - 0x6c, 0xe4, 0x73, 0x72, 0x2b, 0x7f, 0x20, 0x9e, 0x51, 0x6b, 0x9f, 0x3c, 0x4c, 0xf4, 0x1c, 0x66, - 0xbe, 0x8e, 0xf3, 0xf2, 0xf5, 0x7e, 0x81, 0xe6, 0xe9, 0x35, 0x01, 0x7a, 0x4c, 0xb2, 0x11, 0x13, - 0xdc, 0x90, 0x70, 0x68, 0xce, 0x42, 0x1a, 0x50, 0xc1, 0x43, 0x0c, 0x13, 0x87, 0x26, 0xd0, 0x50, - 0xb9, 0x07, 0xb5, 0x9e, 0x27, 0x66, 0x5d, 0x9f, 0xb3, 0x80, 0xc7, 0x64, 0x17, 0x8a, 0x47, 0x11, - 0x52, 0xa8, 0xd2, 0xe2, 0x51, 0x44, 0xea, 0x60, 0xdf, 0xe3, 0x4b, 0xcc, 0x5b, 0xa5, 0x6a, 0x49, - 0x4e, 0x42, 0xe9, 0x09, 0xf3, 0x17, 0x1c, 0xd3, 0x55, 0xa9, 0x06, 0x69, 0x32, 0xe8, 0x4e, 0xb9, - 0x3b, 0x8b, 0x42, 0x2f, 0x90, 0xe4, 0x26, 0xec, 0xa0, 0xc8, 0xde, 0x42, 0x77, 0x0a, 0xd3, 0xda, - 0x9d, 0x0f, 0xde, 0xbe, 0xde, 0xdf, 0x51, 0x35, 0x3f, 0x4c, 0x1c, 0x74, 0x35, 0xce, 0x24, 0xbb, - 0x01, 0x7b, 0xfd, 0x40, 0xf2, 0xd8, 0xe5, 0x91, 0xec, 0x86, 0xf3, 0xb9, 0x27, 0x55, 0x2f, 0x90, - 0xfd, 0x43, 0x36, 0xe7, 0x86, 0x64, 0x66, 0x30, 0xdb, 0x66, 0x50, 0xed, 0x07, 0x22, 0xe2, 0xae, - 0x1c, 0x3e, 0xfc, 0x57, 0x95, 0x3d, 0x07, 0xd5, 0xa3, 0x64, 0xc8, 0x4c, 0x01, 0x32, 0x83, 0x39, - 0x6c, 0x04, 0x35, 0x73, 0x18, 0xe5, 0x22, 0x22, 0x67, 0xc0, 0x1e, 0x2e, 0x75, 0xf9, 0x4a, 0x9d, - 0xca, 0xdb, 0xd7, 0xfb, 0xb6, 0x17, 0x48, 0xaa, 0x6c, 0xaa, 0x0f, 0x0f, 0xb8, 0x10, 0x6c, 0xc2, - 0x4d, 0xae, 0x04, 0x2a, 0xcf, 0x23, 0xb6, 0xf4, 0x43, 0x36, 0xc6, 0x92, 0x6e, 0xd3, 0x04, 0x9a, - 0x33, 0x1e, 0x41, 0xad, 0xcb, 0x24, 0xf3, 0xc3, 0x09, 0x9e, 0x41, 0xc0, 0xe9, 0x4b, 0x3e, 0x37, - 0xf2, 0x71, 0x4d, 0x3e, 0x01, 0x7b, 0xb0, 0x18, 0x35, 0x8a, 0x2d, 0xfb, 0xa0, 0xd6, 0xfe, 0x30, - 0xd3, 0x97, 0xdb, 0x47, 0x55, 0x84, 0xc9, 0xf8, 0x83, 0x1a, 0x3f, 0xb6, 0xf0, 0xe5, 0x23, 0xec, - 0x13, 0x01, 0x27, 0x57, 0x50, 0x5c, 0x2b, 0xdb, 0xdd, 0x98, 0x7f, 0x65, 0xb8, 0xe2, 0x5a, 0xcd, - 0xf4, 0x6d, 0x17, 0xab, 0xa1, 0x5b, 0x6f, 0x10, 0x32, 0x62, 0xf1, 0xa4, 0xe1, 0xa8, 0xf6, 0x52, - 0x5c, 0x2b, 0xdb, 0x40, 0xd9, 0x4a, 0x7a, 0xbf, 0x5a, 0x93, 0xb3, 0xb0, 0xd5, 0x0d, 0x03, 0x21, - 0x59, 0x20, 0x1b, 0xe5, 0x96, 0x75, 0xb0, 0x45, 0x53, 0x6c, 0x88, 0x7d, 0x01, 0xd5, 0x61, 0xcc, - 0x5c, 0x3e, 0x88, 0x58, 0xa0, 0x46, 0xcf, 0x9d, 0x8f, 0x0d, 0x2b, 0xb5, 0x54, 0xa3, 0x27, 0x22, - 0x16, 0x08, 0xc3, 0x4a, 0x03, 0xd5, 0x27, 0x39, 0x8d, 0xb9, 0x98, 0x86, 0xbe, 0xae, 0xa0, 0x4d, - 0x33, 0x83, 0x49, 0xfc, 0x29, 0xec, 0x74, 0xfc, 0xd0, 0x9d, 0x3d, 0xe0, 0x92, 0x61, 0x73, 0x09, - 0x38, 0x9e, 0x1e, 0x09, 0xfb, 0xc0, 0xa1, 0xb8, 0x36, 0xa1, 0x7d, 0xa8, 0x75, 0x67, 0x51, 0x1a, - 0xd8, 0x80, 0xca, 0xd7, 0x3c, 0x16, 0xc9, 0xf8, 0xee, 0xd0, 0x04, 0x2a, 0x39, 0x7e, 0xe8, 0x66, - 0xe3, 0xb1, 0x4d, 0x53, 0x6c, 0x52, 0xfd, 0x6c, 0xc1, 0x89, 0x81, 0x0c, 0x63, 0x36, 0xe1, 0x8f, - 0x55, 0xab, 0x55, 0x1f, 0x9e, 0x3e, 0xb9, 0xaa, 0x72, 0x0e, 0x16, 0xae, 0xcb, 0xb9, 0x56, 0xb7, - 0x45, 0x13, 0x48, 0x6e, 0x00, 0x74, 0x67, 0xd1, 0x9d, 0x40, 0xc6, 0x1e, 0x17, 0x6b, 0xfa, 0x99, - 0x11, 0xa3, 0xb9, 0x40, 0xf2, 0x19, 0x6c, 0xa3, 0xbc, 0x64, 0xa3, 0x8d, 0x1b, 0x4f, 0x67, 0x1b, - 0x57, 0xc4, 0xd3, 0x95, 0x60, 0xc3, 0xf5, 0x0a, 0xec, 0xad, 0x52, 0x35, 0xfd, 0x76, 0xfb, 0x63, - 0x81, 0x55, 0xb2, 0xa9, 0x41, 0x66, 0xc3, 0x72, 0x9d, 0xb6, 0x6b, 0x1b, 0xb4, 0x65, 0xe9, 0x8a, - 0xf9, 0x74, 0xaa, 0xab, 0x03, 0xef, 0x5b, 0xc3, 0xda, 0xa1, 0x1a, 0x28, 0xeb, 0x03, 0x36, 0xf1, - 0x5c, 0x9c, 0x2a, 0x87, 0x6a, 0x60, 0x8e, 0xfe, 0x65, 0x6d, 0x5d, 0xdb, 0xff, 0xed, 0xd9, 0x2a, - 0xfb, 0xd1, 0xe8, 0x79, 0x37, 0x90, 0xa2, 0x51, 0xc2, 0xe8, 0x04, 0x2a, 0x4f, 0xc7, 0x9f, 0xa1, - 0xa7, 0xac, 0x3d, 0x06, 0x2a, 0x0f, 0x0d, 0x5f, 0xa0, 0xa7, 0xa2, 0x3d, 0x06, 0x1a, 0x25, 0x7f, - 0xad, 0x55, 0x72, 0xfd, 0xff, 0xa4, 0x84, 0x5c, 0x80, 0x9d, 0x41, 0xc0, 0x22, 0x31, 0x0d, 0xa5, - 0x66, 0xb0, 0x85, 0xfe, 0x55, 0x63, 0xfa, 0x71, 0xed, 0x25, 0x66, 0xca, 0xd9, 0x58, 0x4d, 0xd9, - 0x55, 0xd8, 0x4a, 0x4c, 0xe9, 0x05, 0x9d, 0xbd, 0xc8, 0xc3, 0x64, 0x45, 0xd3, 0xa8, 0xf4, 0x9e, - 0xaf, 0xaf, 0xa6, 0x12, 0xd1, 0x86, 0xb2, 0xdd, 0x84, 0xca, 0xea, 0x57, 0xf5, 0x51, 0xee, 0xab, - 0x4a, 0x9f, 0x2c, 0x15, 0xb2, 0xc4, 0xdb, 0xb2, 0xb2, 0xfa, 0x75, 0x1c, 0x5b, 0x70, 0x62, 0x4d, - 0x18, 0xb9, 0x08, 0xa5, 0x81, 0x64, 0xf1, 0x66, 0xe6, 0x3a, 0x84, 0x7c, 0x0c, 0xf6, 0x9d, 0x60, - 0x8c, 0x57, 0xc5, 0xfb, 0x22, 0x55, 0x80, 0xba, 0xcf, 0xee, 0x9b, 0x7b, 0xe4, 0x9a, 0x79, 0x11, - 0x32, 0x43, 0xde, 0xdb, 0xc6, 0xae, 0xe6, 0xbc, 0x6d, 0xe5, 0x45, 0x72, 0xc3, 0x65, 0xc4, 0xf1, - 0xee, 0x2d, 0xd1, 0xcc, 0xa0, 0xca, 0xf3, 0xc4, 0xdc, 0x65, 0x65, 0x7d, 0x97, 0x19, 0x68, 0x54, - 0xfe, 0x66, 0xc1, 0xa9, 0xcf, 0xb9, 0xec, 0x4e, 0x59, 0x30, 0xe1, 0x63, 0x7c, 0x58, 0xef, 0x7b, - 0x42, 0xaa, 0x2e, 0x5d, 0x80, 0xe2, 0x70, 0x60, 0x4a, 0xb7, 0x9e, 0x7b, 0x71, 0x38, 0xc8, 0x0d, - 0xa7, 0x9e, 0xc2, 0x64, 0x38, 0x5b, 0x50, 0x4b, 0x7f, 0x49, 0xc6, 0xa2, 0xe1, 0xa0, 0x33, 0x6f, - 0x52, 0x97, 0xa9, 0xfe, 0x2f, 0x19, 0x27, 0x33, 0x99, 0x62, 0x35, 0xc4, 0x77, 0xbe, 0x91, 0x31, - 0x43, 0xd2, 0xdb, 0x54, 0x03, 0x72, 0x19, 0x1c, 0x54, 0x59, 0x69, 0x59, 0x07, 0xbb, 0xed, 0x33, - 0xf9, 0x76, 0x22, 0x7d, 0xc5, 0x5c, 0x05, 0x50, 0x0c, 0x33, 0x0a, 0xff, 0xb4, 0xe0, 0xf4, 0x5a, - 0x85, 0x22, 0x22, 0x97, 0xa0, 0xfc, 0x90, 0xbf, 0xe0, 0x62, 0x73, 0x33, 0x4d, 0xcc, 0x3b, 0xdf, - 0xe1, 0x7b, 0xa5, 0xda, 0x9b, 0xa5, 0x3a, 0xef, 0x93, 0x5a, 0xca, 0x4b, 0xbd, 0x04, 0xe5, 0x23, - 0x7f, 0xac, 0x98, 0x95, 0x37, 0x31, 0xd3, 0x31, 0x46, 0xe9, 0x7d, 0xd8, 0xc5, 0x27, 0xbe, 0x1f, - 0x3c, 0xc7, 0xbf, 0x13, 0xbc, 0xce, 0xe7, 0x5c, 0x4e, 0xc3, 0xe4, 0x49, 0x35, 0x48, 0xcd, 0x4c, - 0xc4, 0x62, 0x36, 0xe7, 0x92, 0xc7, 0xc9, 0x7f, 0x4e, 0x6a, 0xd0, 0xd9, 0x2e, 0xde, 0x84, 0xbd, - 0x77, 0xca, 0x4a, 0xea, 0xb0, 0x8d, 0x5f, 0x84, 0xb1, 0xd7, 0x0b, 0x84, 0xc0, 0x6e, 0x37, 0xf4, - 0x7d, 0xee, 0x26, 0xf5, 0xad, 0x5b, 0x9d, 0xd6, 0xab, 0x3f, 0x9a, 0x85, 0x97, 0x6f, 0x9a, 0xd6, - 0xab, 0x37, 0x4d, 0xeb, 0xf7, 0x37, 0xcd, 0xc2, 0xf7, 0xc7, 0xcd, 0xc2, 0x8f, 0xc7, 0x4d, 0xeb, - 0xd5, 0x71, 0xb3, 0xf0, 0xeb, 0x71, 0xb3, 0x30, 0x2a, 0xe3, 0x4f, 0xf7, 0xf5, 0x7f, 0x02, 0x00, - 0x00, 0xff, 0xff, 0xce, 0x43, 0x55, 0xaa, 0xd2, 0x0b, 0x00, 0x00, + // 1253 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x56, 0xcd, 0x93, 0x13, 0x45, + 0x14, 0xcf, 0x64, 0x26, 0xc9, 0xe6, 0x65, 0xbf, 0x6c, 0x10, 0x02, 0x85, 0xd9, 0x2d, 0x8a, 0xd2, + 0x95, 0x82, 0x05, 0x96, 0xa2, 0xa8, 0xc2, 0x13, 0xc9, 0xb2, 0x56, 0xe4, 0x63, 0xb7, 0x3a, 0x01, + 0x8f, 0xd8, 0x99, 0x34, 0xc9, 0x90, 0xc9, 0xf4, 0x38, 0xdd, 0x01, 0xe3, 0x5f, 0xe0, 0xd1, 0x93, + 0x67, 0x3d, 0xfa, 0x97, 0xc8, 0x91, 0xa3, 0x27, 0x54, 0xf6, 0xe0, 0xc9, 0x2a, 0xcf, 0x9c, 0xac, + 0x7e, 0xdd, 0x93, 0x99, 0x50, 0xd9, 0x5c, 0x2c, 0x0f, 0xde, 0xfa, 0xf7, 0xde, 0xeb, 0xd7, 0xbf, + 0xf7, 0xd1, 0xaf, 0x1b, 0x36, 0x45, 0xcc, 0x13, 0xa6, 0x02, 0x11, 0xc9, 0xdd, 0x38, 0x11, 0x4a, + 0x90, 0x15, 0x7f, 0xdc, 0x7f, 0x3a, 0x51, 0x41, 0x78, 0xfe, 0xea, 0x20, 0x50, 0xc3, 0x49, 0x6f, + 0xd7, 0x17, 0xe3, 0x6b, 0x03, 0x31, 0x10, 0xd7, 0xd0, 0xa0, 0x37, 0x79, 0x86, 0x08, 0x01, 0xae, + 0xcc, 0xc6, 0xf3, 0x1b, 0x2a, 0x18, 0x73, 0xa9, 0xd8, 0x38, 0x36, 0x82, 0x8b, 0x5f, 0x01, 0xdc, + 0xf5, 0x7d, 0x2e, 0x65, 0x3b, 0x7a, 0x26, 0xc8, 0x05, 0xa8, 0xde, 0xf5, 0x7d, 0x31, 0x89, 0x54, + 0x7b, 0xbf, 0xee, 0x6c, 0x3b, 0x3b, 0x6b, 0x34, 0x13, 0x90, 0x33, 0x50, 0x7e, 0x2c, 0x79, 0xd2, + 0xde, 0xaf, 0x17, 0x51, 0x65, 0x91, 0x96, 0x53, 0x11, 0xf2, 0xf6, 0x7e, 0xdd, 0x35, 0x72, 0x83, + 0xee, 0x78, 0x7f, 0xff, 0xb4, 0x55, 0xb8, 0xf8, 0x9d, 0x03, 0x70, 0x10, 0x4e, 0xe4, 0xb0, 0xcb, + 0x7a, 0x21, 0x27, 0x77, 0xf2, 0x07, 0xe2, 0x19, 0xb5, 0xbd, 0xd3, 0xbb, 0x69, 0x3c, 0xbb, 0x99, + 0xae, 0xe9, 0xbd, 0x7a, 0xb3, 0x55, 0xa0, 0x79, 0x7a, 0x0d, 0x80, 0x7d, 0xa6, 0x58, 0x8f, 0x49, + 0x6e, 0x49, 0x78, 0x34, 0x27, 0x21, 0x75, 0xa8, 0xe0, 0x21, 0x96, 0x89, 0x47, 0x53, 0x68, 0xa9, + 0xdc, 0x87, 0xda, 0x7e, 0x20, 0x47, 0xad, 0x90, 0xb3, 0x88, 0x27, 0x64, 0x1d, 0x8a, 0x87, 0x31, + 0x52, 0xa8, 0xd2, 0xe2, 0x61, 0x4c, 0x36, 0xc1, 0xbd, 0xcf, 0xa7, 0xe8, 0xb7, 0x4a, 0xf5, 0x92, + 0x9c, 0x86, 0xd2, 0x13, 0x16, 0x4e, 0x38, 0xba, 0xab, 0x52, 0x03, 0x66, 0xce, 0xa0, 0x35, 0xe4, + 0xfe, 0x28, 0x16, 0x41, 0xa4, 0xc8, 0x6d, 0x58, 0xc3, 0x20, 0xf7, 0x27, 0xa6, 0x52, 0xe8, 0xd6, + 0x6d, 0x7e, 0xf0, 0xee, 0xcd, 0xd6, 0x9a, 0xce, 0xf9, 0x6e, 0xaa, 0xa0, 0xf3, 0x76, 0xd6, 0xd9, + 0x2d, 0xd8, 0x68, 0x47, 0x8a, 0x27, 0x3e, 0x8f, 0x55, 0x4b, 0x8c, 0xc7, 0x81, 0xd2, 0xb5, 0x40, + 0xf6, 0x8f, 0xd8, 0x98, 0x5b, 0x92, 0x99, 0xc0, 0x6e, 0x1b, 0x41, 0xb5, 0x1d, 0xc9, 0x98, 0xfb, + 0xaa, 0xfb, 0xe8, 0x5f, 0x65, 0xf6, 0x02, 0x54, 0x0f, 0xd3, 0x26, 0xb3, 0x09, 0xc8, 0x04, 0xf6, + 0xb0, 0x1e, 0xd4, 0xec, 0x61, 0x94, 0xcb, 0x98, 0x9c, 0x03, 0xb7, 0x3b, 0x35, 0xe9, 0x2b, 0x35, + 0x2b, 0xef, 0xde, 0x6c, 0xb9, 0x41, 0xa4, 0xa8, 0x96, 0xe9, 0x3a, 0x3c, 0xe4, 0x52, 0xb2, 0x01, + 0xb7, 0xbe, 0x52, 0xa8, 0x35, 0x47, 0x6c, 0x1a, 0x0a, 0xd6, 0xc7, 0x94, 0xae, 0xd2, 0x14, 0xda, + 0x33, 0x8e, 0xa0, 0xd6, 0x62, 0x8a, 0x85, 0x62, 0x80, 0x67, 0x10, 0xf0, 0xda, 0x8a, 0x8f, 0x6d, + 0xf8, 0xb8, 0x26, 0x9f, 0x80, 0xdb, 0x99, 0xf4, 0xea, 0xc5, 0x6d, 0x77, 0xa7, 0xb6, 0xf7, 0x61, + 0x16, 0x5f, 0x6e, 0x1f, 0xd5, 0x16, 0xd6, 0xe3, 0x0f, 0xba, 0xfd, 0xd8, 0x24, 0x54, 0x47, 0x58, + 0x27, 0x02, 0x5e, 0x2e, 0xa1, 0xb8, 0xd6, 0xb2, 0x83, 0x84, 0x7f, 0x6d, 0xb9, 0xe2, 0x5a, 0xf7, + 0xf4, 0x5d, 0x1f, 0xb3, 0x61, 0x4a, 0x6f, 0x11, 0x32, 0x62, 0xc9, 0xa0, 0xee, 0xe9, 0xf2, 0x52, + 0x5c, 0x6b, 0x59, 0x47, 0xcb, 0x4a, 0x66, 0xbf, 0x5e, 0x93, 0xf3, 0xb0, 0xd2, 0x12, 0x91, 0x54, + 0x2c, 0x52, 0xf5, 0xf2, 0xb6, 0xb3, 0xb3, 0x42, 0x67, 0xd8, 0x12, 0xfb, 0x12, 0xaa, 0xdd, 0x84, + 0xf9, 0xbc, 0x13, 0xb3, 0x48, 0xb7, 0x9e, 0x3f, 0xee, 0x5b, 0x56, 0x7a, 0xa9, 0x5b, 0x4f, 0xc6, + 0x2c, 0x92, 0x96, 0x95, 0x01, 0xba, 0x4e, 0x6a, 0x98, 0x70, 0x39, 0x14, 0xa1, 0xc9, 0xa0, 0x4b, + 0x33, 0x81, 0x75, 0xfc, 0x29, 0xac, 0x35, 0x43, 0xe1, 0x8f, 0x1e, 0x72, 0xc5, 0xb0, 0xb8, 0x04, + 0xbc, 0xc0, 0xb4, 0x84, 0xbb, 0xe3, 0x51, 0x5c, 0x5b, 0xd3, 0x36, 0xd4, 0x5a, 0xa3, 0x78, 0x66, + 0x58, 0x87, 0xca, 0x0b, 0x9e, 0xc8, 0xb4, 0x7d, 0xd7, 0x68, 0x0a, 0x75, 0x38, 0xa1, 0xf0, 0xb3, + 0xf6, 0x58, 0xa5, 0x33, 0x6c, 0x5d, 0xfd, 0xec, 0xc0, 0xa9, 0x8e, 0x12, 0x09, 0x1b, 0xf0, 0xc7, + 0xba, 0xd4, 0xba, 0x0e, 0x4f, 0x9f, 0x5c, 0xd7, 0x3e, 0x3b, 0x13, 0xdf, 0xe7, 0xdc, 0x44, 0xb7, + 0x42, 0x53, 0x48, 0x6e, 0x01, 0xb4, 0x46, 0xf1, 0xbd, 0x48, 0x25, 0x01, 0x97, 0x0b, 0xea, 0x99, + 0x11, 0xa3, 0x39, 0x43, 0xf2, 0x19, 0xac, 0x62, 0x78, 0xe9, 0x46, 0x17, 0x37, 0x9e, 0xcd, 0x36, + 0xce, 0x05, 0x4f, 0xe7, 0x8c, 0x2d, 0xd7, 0x6b, 0xb0, 0x31, 0x4f, 0xd5, 0xd6, 0xdb, 0x6f, 0xf7, + 0x25, 0x66, 0xc9, 0xa5, 0x16, 0xd9, 0x0d, 0xd3, 0x45, 0xb1, 0xdd, 0x58, 0x12, 0x5b, 0xe6, 0xae, + 0x98, 0x77, 0xa7, 0xab, 0xda, 0x09, 0xbe, 0xb5, 0xac, 0x3d, 0x6a, 0x80, 0x96, 0x3e, 0x64, 0x83, + 0xc0, 0xc7, 0xae, 0xf2, 0xa8, 0x01, 0xf6, 0xe8, 0x5f, 0x16, 0xe6, 0x75, 0xef, 0xbf, 0x3d, 0x5b, + 0x7b, 0x3f, 0xec, 0x3d, 0x6f, 0x45, 0x4a, 0xd6, 0x4b, 0x68, 0x9d, 0x42, 0xad, 0x69, 0x86, 0x23, + 0xd4, 0x94, 0x8d, 0xc6, 0x42, 0xad, 0xa1, 0xe2, 0x25, 0x6a, 0x2a, 0x46, 0x63, 0xa1, 0x8d, 0xe4, + 0xaf, 0x85, 0x91, 0xdc, 0xfc, 0x3f, 0x45, 0x42, 0x2e, 0xc1, 0x5a, 0x27, 0x62, 0xb1, 0x1c, 0x0a, + 0x65, 0x18, 0xac, 0xa0, 0x7e, 0x5e, 0x38, 0xbb, 0x5c, 0x1b, 0xa9, 0x98, 0x72, 0xd6, 0xd7, 0x5d, + 0x76, 0x1d, 0x56, 0x52, 0xd1, 0x6c, 0x40, 0x67, 0x2f, 0x72, 0x37, 0x5d, 0xd1, 0x99, 0xd5, 0x6c, + 0xce, 0x6f, 0xce, 0xbb, 0x92, 0xf1, 0x92, 0xb4, 0xdd, 0x86, 0xca, 0xfc, 0xad, 0xfa, 0x28, 0x77, + 0xab, 0x66, 0x4f, 0x96, 0x36, 0x99, 0xe2, 0xb4, 0xac, 0xcc, 0xdf, 0x8e, 0x63, 0x07, 0x4e, 0x2d, + 0x30, 0x23, 0x97, 0xa1, 0xd4, 0x51, 0x2c, 0x59, 0xce, 0xdc, 0x98, 0x90, 0x8f, 0xc1, 0xbd, 0x17, + 0xf5, 0x71, 0x54, 0x9c, 0x64, 0xa9, 0x0d, 0xf4, 0x3c, 0x7b, 0x60, 0xe7, 0xc8, 0x0d, 0xfb, 0x22, + 0x64, 0x82, 0xbc, 0x76, 0x0f, 0xab, 0x9a, 0xd3, 0xee, 0x69, 0x2d, 0x92, 0xeb, 0x4e, 0x63, 0x8e, + 0xb3, 0xb7, 0x44, 0x33, 0x81, 0x4e, 0xcf, 0x13, 0x3b, 0xcb, 0xca, 0x66, 0x96, 0x59, 0x68, 0xa3, + 0xfc, 0xcd, 0x81, 0x33, 0x9f, 0x73, 0xd5, 0x1a, 0xb2, 0x68, 0xc0, 0xfb, 0xf8, 0xb0, 0x3e, 0x08, + 0xa4, 0xd2, 0x55, 0xba, 0x04, 0xc5, 0x6e, 0xc7, 0xa6, 0x6e, 0x31, 0xf7, 0x62, 0xb7, 0x93, 0x6b, + 0x4e, 0xd3, 0x85, 0x69, 0x73, 0x6e, 0x43, 0x6d, 0xf6, 0x25, 0xe9, 0xcb, 0xba, 0x87, 0xca, 0xbc, + 0x48, 0x0f, 0x53, 0xf3, 0x2f, 0xe9, 0xa7, 0x3d, 0x39, 0xc3, 0xba, 0x89, 0xef, 0x7d, 0xa3, 0x12, + 0x86, 0xa4, 0x57, 0xa9, 0x01, 0xe4, 0x2a, 0x78, 0x18, 0x65, 0x65, 0xdb, 0xd9, 0x59, 0xdf, 0x3b, + 0x97, 0x2f, 0x27, 0xd2, 0xd7, 0xcc, 0xb5, 0x01, 0x45, 0x33, 0x1b, 0xe1, 0x9f, 0x0e, 0x9c, 0x5d, + 0x18, 0xa1, 0x8c, 0xc9, 0x15, 0x28, 0x3f, 0xe2, 0x2f, 0xb9, 0x5c, 0x5e, 0x4c, 0x6b, 0xf3, 0xde, + 0x3d, 0x3c, 0x31, 0x54, 0x77, 0x79, 0xa8, 0xde, 0x49, 0xa1, 0x96, 0xf2, 0xa1, 0x5e, 0x81, 0xf2, + 0x61, 0xd8, 0xd7, 0xcc, 0xca, 0xcb, 0x98, 0x19, 0x1b, 0x1b, 0xe9, 0x03, 0x58, 0xc7, 0x27, 0xbe, + 0x1d, 0x3d, 0xc7, 0xdf, 0x09, 0x8e, 0xf3, 0x31, 0x57, 0x43, 0x91, 0x3e, 0xa9, 0x16, 0xe9, 0x9e, + 0x89, 0x59, 0xc2, 0xc6, 0x5c, 0xf1, 0x24, 0xfd, 0xe7, 0xcc, 0x04, 0xd6, 0xdb, 0x0b, 0x58, 0xef, + 0x4c, 0x23, 0xff, 0x28, 0x11, 0x8a, 0x9b, 0x47, 0xff, 0x34, 0x94, 0xbe, 0x10, 0x3d, 0xfb, 0x25, + 0xae, 0x52, 0x03, 0xf4, 0xf7, 0xb1, 0x79, 0x60, 0x9d, 0x14, 0x9b, 0x07, 0xd8, 0x71, 0x2c, 0x0c, + 0xfa, 0xdd, 0x8e, 0x7d, 0x99, 0x53, 0xa8, 0xff, 0xad, 0x5d, 0x2e, 0xd5, 0x61, 0x4f, 0xd3, 0xc3, + 0x46, 0xae, 0xd2, 0x9c, 0xc4, 0x9c, 0x7b, 0xf9, 0x36, 0x6c, 0xbc, 0x57, 0x4e, 0xb2, 0x09, 0xab, + 0x78, 0x13, 0xad, 0x7c, 0xb3, 0x40, 0x08, 0xac, 0xb7, 0x44, 0x18, 0x72, 0x3f, 0xad, 0xeb, 0xa6, + 0xd3, 0xdc, 0x7e, 0xfd, 0x47, 0xa3, 0xf0, 0xea, 0x6d, 0xc3, 0x79, 0xfd, 0xb6, 0xe1, 0xfc, 0xfe, + 0xb6, 0x51, 0xf8, 0xfe, 0xb8, 0x51, 0xf8, 0xf1, 0xb8, 0xe1, 0xbc, 0x3e, 0x6e, 0x14, 0x7e, 0x3d, + 0x6e, 0x14, 0x7a, 0x65, 0xfc, 0xec, 0xdf, 0xfc, 0x27, 0x00, 0x00, 0xff, 0xff, 0x8f, 0x84, 0xa2, + 0xbb, 0x4a, 0x0c, 0x00, 0x00, } func (m *AccessInfo) Marshal() (dAtA []byte, err error) { @@ -2799,6 +2865,55 @@ func (m *FaultInjectReq) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *SyncProtection) Marshal() (dAtA []byte, err error) { + size := m.ProtoSize() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SyncProtection) MarshalTo(dAtA []byte) (int, error) { + size := m.ProtoSize() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *SyncProtection) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.TestObject) > 0 { + i -= len(m.TestObject) + copy(dAtA[i:], m.TestObject) + i = encodeVarintOperations(dAtA, i, uint64(len(m.TestObject))) + i-- + dAtA[i] = 0x22 + } + if m.ValidTS != 0 { + i = encodeVarintOperations(dAtA, i, uint64(m.ValidTS)) + i-- + dAtA[i] = 0x18 + } + if len(m.BF) > 0 { + i -= len(m.BF) + copy(dAtA[i:], m.BF) + i = encodeVarintOperations(dAtA, i, uint64(len(m.BF))) + i-- + dAtA[i] = 0x12 + } + if len(m.JobID) > 0 { + i -= len(m.JobID) + copy(dAtA[i:], m.JobID) + i = encodeVarintOperations(dAtA, i, uint64(len(m.JobID))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + func encodeVarintOperations(dAtA []byte, offset int, v uint64) int { offset -= sovOperations(v) base := offset @@ -3368,6 +3483,30 @@ func (m *FaultInjectReq) ProtoSize() (n int) { return n } +func (m *SyncProtection) ProtoSize() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.JobID) + if l > 0 { + n += 1 + l + sovOperations(uint64(l)) + } + l = len(m.BF) + if l > 0 { + n += 1 + l + sovOperations(uint64(l)) + } + if m.ValidTS != 0 { + n += 1 + sovOperations(uint64(m.ValidTS)) + } + l = len(m.TestObject) + if l > 0 { + n += 1 + l + sovOperations(uint64(l)) + } + return n +} + func sovOperations(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } @@ -7644,6 +7783,171 @@ func (m *FaultInjectReq) Unmarshal(dAtA []byte) error { } return nil } +func (m *SyncProtection) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowOperations + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SyncProtection: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SyncProtection: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field JobID", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowOperations + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthOperations + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthOperations + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.JobID = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field BF", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowOperations + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthOperations + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthOperations + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.BF = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ValidTS", wireType) + } + m.ValidTS = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowOperations + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ValidTS |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field TestObject", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowOperations + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthOperations + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthOperations + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.TestObject = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipOperations(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthOperations + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipOperations(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 diff --git a/pkg/vm/engine/cmd_util/operations.proto b/pkg/vm/engine/cmd_util/operations.proto index f85001fcef50a..13027cf8e041b 100644 --- a/pkg/vm/engine/cmd_util/operations.proto +++ b/pkg/vm/engine/cmd_util/operations.proto @@ -200,4 +200,12 @@ message FaultInjectReq { option (gogoproto.typedecl) = false; string method = 1; string parameter = 2; +} + +message SyncProtection { + option (gogoproto.typedecl) = false; + string JobID = 1; // Sync job ID + string BF = 2; // Base64 encoded BloomFilter data (for register) + int64 ValidTS = 3; // Valid timestamp in nanoseconds (for register and renew) + string TestObject = 4; // Test object name for debugging (optional) } \ No newline at end of file diff --git a/pkg/vm/engine/cmd_util/type.go b/pkg/vm/engine/cmd_util/type.go index d638abba3229c..555b164098f59 100644 --- a/pkg/vm/engine/cmd_util/type.go +++ b/pkg/vm/engine/cmd_util/type.go @@ -33,4 +33,9 @@ const ( GCDetails = "details" GCVerify = "verify" + + // Sync protection operations for cross-cluster sync + RegisterSyncProtection = "register_sync_protection" + RenewSyncProtection = "renew_sync_protection" + UnregisterSyncProtection = "unregister_sync_protection" ) diff --git a/pkg/vm/engine/tae/db/gc/v3/checkpoint.go b/pkg/vm/engine/tae/db/gc/v3/checkpoint.go index b163088cb2377..c47d0110dc855 100644 --- a/pkg/vm/engine/tae/db/gc/v3/checkpoint.go +++ b/pkg/vm/engine/tae/db/gc/v3/checkpoint.go @@ -116,6 +116,9 @@ type checkpointCleaner struct { isActive bool } + // syncProtection is the sync protection manager for cross-cluster sync + syncProtection *SyncProtectionManager + mutation struct { sync.Mutex taskState struct { @@ -220,9 +223,15 @@ func NewCheckpointCleaner( cleaner.mutation.metaFiles = make(map[string]ioutil.TSRangeFile) cleaner.mutation.snapshotMeta = logtail.NewSnapshotMeta() cleaner.backupProtection.isActive = false + cleaner.syncProtection = NewSyncProtectionManager() return cleaner } +// GetSyncProtectionManager returns the sync protection manager +func (c *checkpointCleaner) GetSyncProtectionManager() *SyncProtectionManager { + return c.syncProtection +} + func (c *checkpointCleaner) Stop() { c.mutation.Lock() defer c.mutation.Unlock() @@ -1200,6 +1209,7 @@ func (c *checkpointCleaner) tryGCAgainstGCKPLocked( extraErrMsg = "doGCAgainstGlobalCheckpointLocked failed" return } + // Delete files after doGCAgainstGlobalCheckpointLocked // TODO:Requires Physical Removal Policy // Note: Data files are GC'ed normally even when backup protection is active. @@ -1224,6 +1234,7 @@ func (c *checkpointCleaner) tryGCAgainstGCKPLocked( v2.GCCheckpointDeleteDurationHistogram.Observe(time.Since(deleteStart).Seconds()) v2.GCSnapshotDeleteDurationHistogram.Observe(time.Since(deleteStart).Seconds()) } + if c.GetGCWaterMark() == nil { return nil } @@ -1308,6 +1319,7 @@ func (c *checkpointCleaner) doGCAgainstGlobalCheckpointLocked( c.mutation.snapshotMeta, iscp, c.checkpointCli, + c.syncProtection, memoryBuffer, c.config.canGCCacheSize, c.config.estimateRows, @@ -1483,6 +1495,7 @@ func (c *checkpointCleaner) DoCheck(ctx context.Context) error { c.mutation.snapshotMeta, iscp, c.checkpointCli, + c.syncProtection, buffer, c.config.canGCCacheSize, c.config.estimateRows, @@ -1507,6 +1520,7 @@ func (c *checkpointCleaner) DoCheck(ctx context.Context) error { c.mutation.snapshotMeta, iscp, c.checkpointCli, + c.syncProtection, buffer, c.config.canGCCacheSize, c.config.estimateRows, @@ -1624,6 +1638,14 @@ func (c *checkpointCleaner) Process( c.StartMutationTask("gc-process") defer c.StopMutationTask() + // Set GC running state for sync protection + // This prevents new sync protections from being registered during GC + c.syncProtection.SetGCRunning(true) + defer c.syncProtection.SetGCRunning(false) + + // Cleanup expired sync protections (TTL exceeded, handles crashed sync jobs) + c.syncProtection.CleanupExpired() + // Check backup protection state and create a snapshot at the start of GC // This snapshot will be used throughout the GC process to ensure consistency c.backupProtection.Lock() @@ -1826,6 +1848,14 @@ func (c *checkpointCleaner) tryScanLocked( v2.GCErrorIOErrorCounter.Inc() return } + + // Cleanup soft-deleted sync protections when checkpoint watermark > validTS + // This ensures protections are only removed after the checkpoint has recorded the commit + scanWaterMarkEntry := c.GetScanWaterMark() + if scanWaterMarkEntry != nil { + checkpointWatermark := scanWaterMarkEntry.GetEnd().ToTimestamp().PhysicalTime + c.syncProtection.CleanupSoftDeleted(checkpointWatermark) + } return } diff --git a/pkg/vm/engine/tae/db/gc/v3/exec_v1.go b/pkg/vm/engine/tae/db/gc/v3/exec_v1.go index 99fd054969ecc..eb2e70e6bf4fb 100644 --- a/pkg/vm/engine/tae/db/gc/v3/exec_v1.go +++ b/pkg/vm/engine/tae/db/gc/v3/exec_v1.go @@ -65,15 +65,16 @@ type CheckpointBasedGCJob struct { coarseProbility float64 canGCCacheSize int } - sourcer engine.BaseReader - snapshotMeta *logtail.SnapshotMeta - snapshots *logtail.SnapshotInfo - iscpTables map[uint64]types.TS - pitr *logtail.PitrInfo - ts *types.TS - globalCkpLoc objectio.Location - globalCkpVer uint32 - checkpointCli checkpoint.Runner // Added to access catalog + sourcer engine.BaseReader + snapshotMeta *logtail.SnapshotMeta + snapshots *logtail.SnapshotInfo + iscpTables map[uint64]types.TS + pitr *logtail.PitrInfo + ts *types.TS + globalCkpLoc objectio.Location + globalCkpVer uint32 + checkpointCli checkpoint.Runner // Added to access catalog + syncProtection *SyncProtectionManager // Sync protection manager for cross-cluster sync result struct { vecToGC *vector.Vector @@ -91,6 +92,7 @@ func NewCheckpointBasedGCJob( iscpTables map[uint64]types.TS, snapshotMeta *logtail.SnapshotMeta, checkpointCli checkpoint.Runner, + syncProtection *SyncProtectionManager, buffer *containers.OneSchemaBatchBuffer, isOwner bool, mp *mpool.MPool, @@ -99,15 +101,16 @@ func NewCheckpointBasedGCJob( opts ...GCJobExecutorOption, ) *CheckpointBasedGCJob { e := &CheckpointBasedGCJob{ - sourcer: sourcer, - snapshotMeta: snapshotMeta, - snapshots: snapshots, - pitr: pitr, - ts: ts, - globalCkpLoc: globalCkpLoc, - globalCkpVer: gckpVersion, - iscpTables: iscpTables, - checkpointCli: checkpointCli, + sourcer: sourcer, + snapshotMeta: snapshotMeta, + snapshots: snapshots, + pitr: pitr, + ts: ts, + globalCkpLoc: globalCkpLoc, + globalCkpVer: gckpVersion, + iscpTables: iscpTables, + checkpointCli: checkpointCli, + syncProtection: syncProtection, } for _, opt := range opts { opt(e) @@ -164,6 +167,7 @@ func (e *CheckpointBasedGCJob) Execute(ctx context.Context) error { e.globalCkpVer, e.ts, &transObjects, + e.syncProtection, e.mp, e.fs, ) @@ -235,6 +239,7 @@ func MakeBloomfilterCoarseFilter( ckpVersion uint32, ts *types.TS, transObjects *map[string]map[uint64]*ObjectEntry, + syncProtection *SyncProtectionManager, mp *mpool.MPool, fs fileservice.FileService, ) ( @@ -291,10 +296,19 @@ func MakeBloomfilterCoarseFilter( if !createTS.LT(ts) || !dropTS.LT(ts) { return } - bm.Add(uint64(i)) + + // Check if the object is protected by sync protection + // If protected, skip marking it for GC so it stays in filesNotGC buf := bat.Vecs[0].GetRawBytesAt(i) stats := (objectio.ObjectStats)(buf) name := stats.ObjectName().UnsafeString() + + if syncProtection != nil && syncProtection.IsProtected(name) { + // Protected file: don't mark for GC, it will stay in filesNotGC + return + } + + bm.Add(uint64(i)) tid := tableIDs[i] if (*transObjects)[name] == nil || (*transObjects)[name][tableIDs[i]] == nil { diff --git a/pkg/vm/engine/tae/db/gc/v3/mock_cleaner.go b/pkg/vm/engine/tae/db/gc/v3/mock_cleaner.go index b5a5147d1f60e..48759fdc25cee 100644 --- a/pkg/vm/engine/tae/db/gc/v3/mock_cleaner.go +++ b/pkg/vm/engine/tae/db/gc/v3/mock_cleaner.go @@ -171,3 +171,7 @@ func (c *MockCleaner) RemoveBackupProtection() { func (c *MockCleaner) GetBackupProtection() (protectedTS types.TS, lastUpdateTime time.Time, isActive bool) { return types.TS{}, time.Time{}, false } + +func (c *MockCleaner) GetSyncProtectionManager() *SyncProtectionManager { + return nil +} diff --git a/pkg/vm/engine/tae/db/gc/v3/sync_protection.go b/pkg/vm/engine/tae/db/gc/v3/sync_protection.go new file mode 100644 index 0000000000000..cde64cd143d15 --- /dev/null +++ b/pkg/vm/engine/tae/db/gc/v3/sync_protection.go @@ -0,0 +1,374 @@ +// Copyright 2021 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gc + +import ( + "encoding/base64" + "sync" + "sync/atomic" + "time" + + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/logutil" + "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/index" + "go.uber.org/zap" +) + +const ( + // DefaultSyncProtectionTTL is the default TTL for sync protection + // If a protection is not renewed within this duration, it will be force cleaned + DefaultSyncProtectionTTL = 20 * time.Minute + + // DefaultMaxSyncProtections is the default maximum number of sync protections + // Set to a large value to support many concurrent sync jobs + // Jobs may take ~1.5 hours to be cleaned up after completion + DefaultMaxSyncProtections = 1000000 +) + +// SyncProtection represents a single sync protection entry +type SyncProtection struct { + JobID string // Sync job ID + BF index.BloomFilter // BloomFilter for protected objects (using xorfilter, deterministic) + ValidTS int64 // Valid timestamp (nanoseconds), needs to be renewed + SoftDelete bool // Whether soft deleted + CreateTime time.Time // Creation time for logging +} + +// SyncProtectionManager manages sync protection entries +type SyncProtectionManager struct { + sync.RWMutex + protections map[string]*SyncProtection // jobID -> protection + gcRunning atomic.Bool // Whether GC is running + ttl time.Duration // TTL for non-soft-deleted protections + maxCount int // Maximum number of protections +} + +// NewSyncProtectionManager creates a new SyncProtectionManager +func NewSyncProtectionManager() *SyncProtectionManager { + return &SyncProtectionManager{ + protections: make(map[string]*SyncProtection), + ttl: DefaultSyncProtectionTTL, + maxCount: DefaultMaxSyncProtections, + } +} + +// SetGCRunning sets the GC running state +func (m *SyncProtectionManager) SetGCRunning(running bool) { + m.gcRunning.Store(running) + logutil.Debug( + "GC-Sync-Protection-GC-State-Changed", + zap.Bool("running", running), + ) +} + +// IsGCRunning returns whether GC is running +func (m *SyncProtectionManager) IsGCRunning() bool { + return m.gcRunning.Load() +} + +// RegisterSyncProtection registers a new sync protection with BloomFilter +// bfData is base64 encoded BloomFilter bytes (using index.BloomFilter/xorfilter format) +// Returns error if GC is running or job already exists +func (m *SyncProtectionManager) RegisterSyncProtection( + jobID string, + bfData string, + validTS int64, +) error { + m.Lock() + defer m.Unlock() + + // Check if GC is running + if m.gcRunning.Load() { + logutil.Warn( + "GC-Sync-Protection-Register-Rejected-GC-Running", + zap.String("job-id", jobID), + ) + return moerr.NewGCIsRunningNoCtx() + } + + // Check if job already exists + if _, ok := m.protections[jobID]; ok { + logutil.Warn( + "GC-Sync-Protection-Register-Already-Exists", + zap.String("job-id", jobID), + ) + return moerr.NewSyncProtectionExistsNoCtx(jobID) + } + + // Check max count + if len(m.protections) >= m.maxCount { + logutil.Warn( + "GC-Sync-Protection-Register-Max-Count-Reached", + zap.String("job-id", jobID), + zap.Int("current-count", len(m.protections)), + zap.Int("max-count", m.maxCount), + ) + return moerr.NewSyncProtectionMaxCountNoCtx(m.maxCount) + } + + // Check if BF data is empty + if bfData == "" { + logutil.Error( + "GC-Sync-Protection-Register-Empty-BF", + zap.String("job-id", jobID), + ) + return moerr.NewSyncProtectionInvalidNoCtx() + } + + // Decode base64 BloomFilter data + bfBytes, err := base64.StdEncoding.DecodeString(bfData) + if err != nil { + logutil.Error( + "GC-Sync-Protection-Register-Decode-Error", + zap.String("job-id", jobID), + zap.Error(err), + ) + return moerr.NewSyncProtectionInvalidNoCtx() + } + + // Unmarshal BloomFilter (using index.BloomFilter which is based on xorfilter - deterministic) + // Validate minimum buffer length before unmarshal to avoid panic + // Minimum size: 8 (Seed) + 4*4 (SegmentLength, SegmentLengthMask, SegmentCount, SegmentCountLength) = 24 bytes + if len(bfBytes) < 24 { + logutil.Error( + "GC-Sync-Protection-Register-Invalid-BF-Size", + zap.String("job-id", jobID), + zap.Int("size", len(bfBytes)), + ) + return moerr.NewSyncProtectionInvalidNoCtx() + } + + var bf index.BloomFilter + if err = bf.Unmarshal(bfBytes); err != nil { + logutil.Error( + "GC-Sync-Protection-Register-Unmarshal-Error", + zap.String("job-id", jobID), + zap.Error(err), + ) + return moerr.NewSyncProtectionInvalidNoCtx() + } + + m.protections[jobID] = &SyncProtection{ + JobID: jobID, + BF: bf, + ValidTS: validTS, + SoftDelete: false, + CreateTime: time.Now(), + } + + logutil.Info( + "GC-Sync-Protection-Registered", + zap.String("job-id", jobID), + zap.Int64("valid-ts", validTS), + zap.Int("bf-size", len(bfBytes)), + zap.Int("total-protections", len(m.protections)), + ) + return nil +} + +// RenewSyncProtection renews the valid timestamp of a sync protection +func (m *SyncProtectionManager) RenewSyncProtection(jobID string, validTS int64) error { + m.Lock() + defer m.Unlock() + + p, ok := m.protections[jobID] + if !ok { + logutil.Warn( + "GC-Sync-Protection-Renew-Not-Found", + zap.String("job-id", jobID), + ) + return moerr.NewSyncProtectionNotFoundNoCtx(jobID) + } + + if p.SoftDelete { + logutil.Warn( + "GC-Sync-Protection-Renew-Already-Soft-Deleted", + zap.String("job-id", jobID), + ) + return moerr.NewSyncProtectionSoftDeleteNoCtx(jobID) + } + + oldValidTS := p.ValidTS + p.ValidTS = validTS + + logutil.Debug( + "GC-Sync-Protection-Renewed", + zap.String("job-id", jobID), + zap.Int64("old-valid-ts", oldValidTS), + zap.Int64("new-valid-ts", validTS), + ) + return nil +} + +// UnregisterSyncProtection soft deletes a sync protection +// Returns error if job not found (sync job needs to handle rollback) +func (m *SyncProtectionManager) UnregisterSyncProtection(jobID string) error { + m.Lock() + defer m.Unlock() + + p, ok := m.protections[jobID] + if !ok { + logutil.Warn( + "GC-Sync-Protection-Unregister-Not-Found", + zap.String("job-id", jobID), + ) + return moerr.NewSyncProtectionNotFoundNoCtx(jobID) + } + + p.SoftDelete = true + + logutil.Info( + "GC-Sync-Protection-Soft-Deleted", + zap.String("job-id", jobID), + zap.Int64("valid-ts", p.ValidTS), + ) + return nil +} + +// CleanupSoftDeleted cleans up soft-deleted protections when checkpoint watermark > validTS +// This should be called during GC when processing checkpoints +func (m *SyncProtectionManager) CleanupSoftDeleted(checkpointWatermark int64) { + m.Lock() + defer m.Unlock() + + for jobID, p := range m.protections { + // Condition: soft delete state AND checkpoint watermark > validTS + if p.SoftDelete && checkpointWatermark > p.ValidTS { + delete(m.protections, jobID) + logutil.Info( + "GC-Sync-Protection-Cleaned-Soft-Deleted", + zap.String("job-id", jobID), + zap.Int64("valid-ts", p.ValidTS), + zap.Int64("checkpoint-watermark", checkpointWatermark), + ) + } + } +} + +// CleanupExpired cleans up expired protections (TTL exceeded and not soft deleted) +// This handles crashed sync jobs that didn't unregister +func (m *SyncProtectionManager) CleanupExpired() { + m.Lock() + defer m.Unlock() + + now := time.Now() + for jobID, p := range m.protections { + validTime := time.Unix(0, p.ValidTS) + + // Non soft delete state, but TTL exceeded without renewal + if !p.SoftDelete && now.Sub(validTime) > m.ttl { + delete(m.protections, jobID) + logutil.Warn( + "GC-Sync-Protection-Force-Cleaned-Expired", + zap.String("job-id", jobID), + zap.Int64("valid-ts", p.ValidTS), + zap.Duration("age", now.Sub(validTime)), + zap.Duration("ttl", m.ttl), + ) + } + } +} + +// GetProtectionCount returns the number of protections +func (m *SyncProtectionManager) GetProtectionCount() int { + m.RLock() + defer m.RUnlock() + return len(m.protections) +} + +// GetProtectionCountByState returns the count of protections by state +func (m *SyncProtectionManager) GetProtectionCountByState() (active, softDeleted int) { + m.RLock() + defer m.RUnlock() + + for _, p := range m.protections { + if p.SoftDelete { + softDeleted++ + } else { + active++ + } + } + return +} + +// HasProtection checks if a job has protection +func (m *SyncProtectionManager) HasProtection(jobID string) bool { + m.RLock() + defer m.RUnlock() + _, ok := m.protections[jobID] + return ok +} + +// IsProtected checks if an object name is protected by any BloomFilter +func (m *SyncProtectionManager) IsProtected(objectName string) bool { + m.RLock() + defer m.RUnlock() + + if len(m.protections) == 0 { + return false + } + + for _, p := range m.protections { + // Use MayContainsKey for single element test + if result, err := p.BF.MayContainsKey([]byte(objectName)); err == nil && result { + return true + } + } + return false +} + +// FilterProtectedFiles filters out protected files from the list +// Returns files that are NOT protected (can be deleted) +func (m *SyncProtectionManager) FilterProtectedFiles(files []string) []string { + m.RLock() + defer m.RUnlock() + + if len(m.protections) == 0 || len(files) == 0 { + return files + } + + // Build result: files that are NOT protected + result := make([]string, 0, len(files)) + protectedCount := 0 + + for _, f := range files { + protected := false + + // Check against each BloomFilter + for _, p := range m.protections { + if contains, err := p.BF.MayContainsKey([]byte(f)); err == nil && contains { + protected = true + break + } + } + + if protected { + protectedCount++ + } else { + result = append(result, f) + } + } + + if protectedCount > 0 { + logutil.Info( + "GC-Sync-Protection-Filtered", + zap.Int("total", len(files)), + zap.Int("can-delete", len(result)), + zap.Int("protected", protectedCount), + ) + } + + return result +} diff --git a/pkg/vm/engine/tae/db/gc/v3/sync_protection_test.go b/pkg/vm/engine/tae/db/gc/v3/sync_protection_test.go new file mode 100644 index 0000000000000..fef0500e767fa --- /dev/null +++ b/pkg/vm/engine/tae/db/gc/v3/sync_protection_test.go @@ -0,0 +1,491 @@ +// Copyright 2021 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package gc + +import ( + "encoding/base64" + "testing" + "time" + + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/common" + "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/containers" + "github.com/matrixorigin/matrixone/pkg/vm/engine/tae/index" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// buildTestBF creates a BloomFilter from object names and returns base64 encoded data +// Uses index.BloomFilter (xorfilter based) which is deterministic across processes +func buildTestBF(t *testing.T, objects []string) string { + // Create a containers.Vector with all object names + vec := containers.MakeVector(types.T_varchar.ToType(), common.DefaultAllocator) + defer vec.Close() + + for _, obj := range objects { + vec.Append([]byte(obj), false) + } + + // Create BloomFilter using index.NewBloomFilter (xorfilter based) + bf, err := index.NewBloomFilter(vec) + require.NoError(t, err) + + data, err := bf.Marshal() + require.NoError(t, err) + + return base64.StdEncoding.EncodeToString(data) +} + +func TestSyncProtectionManager_RegisterAndUnregister(t *testing.T) { + mgr := NewSyncProtectionManager() + + jobID := "test-job-1" + objects := []string{"object1", "object2", "object3"} + bfData := buildTestBF(t, objects) + validTS := time.Now().UnixNano() + + // Test register + err := mgr.RegisterSyncProtection(jobID, bfData, validTS) + require.NoError(t, err) + assert.Equal(t, 1, mgr.GetProtectionCount()) + assert.True(t, mgr.HasProtection(jobID)) + + // Test duplicate register + err = mgr.RegisterSyncProtection(jobID, bfData, validTS) + require.Error(t, err) + assert.Contains(t, err.Error(), "already exists") + + // Test unregister (soft delete) + err = mgr.UnregisterSyncProtection(jobID) + require.NoError(t, err) + assert.Equal(t, 1, mgr.GetProtectionCount()) // Still exists, just soft deleted + + active, softDeleted := mgr.GetProtectionCountByState() + assert.Equal(t, 0, active) + assert.Equal(t, 1, softDeleted) + + // Test unregister non-existent job + err = mgr.UnregisterSyncProtection("non-existent") + require.Error(t, err) + assert.Contains(t, err.Error(), "not found") +} + +func TestSyncProtectionManager_GCRunningBlock(t *testing.T) { + mgr := NewSyncProtectionManager() + + jobID := "test-job-1" + bfData := buildTestBF(t, []string{"object1"}) + validTS := time.Now().UnixNano() + + // Set GC running + mgr.SetGCRunning(true) + assert.True(t, mgr.IsGCRunning()) + + // Register should fail when GC is running + err := mgr.RegisterSyncProtection(jobID, bfData, validTS) + require.Error(t, err) + assert.Contains(t, err.Error(), "GC is running") + + // Set GC not running + mgr.SetGCRunning(false) + assert.False(t, mgr.IsGCRunning()) + + // Register should succeed now + err = mgr.RegisterSyncProtection(jobID, bfData, validTS) + require.NoError(t, err) +} + +func TestSyncProtectionManager_Renew(t *testing.T) { + mgr := NewSyncProtectionManager() + + jobID := "test-job-1" + bfData := buildTestBF(t, []string{"object1"}) + validTS1 := time.Now().UnixNano() + + // Register + err := mgr.RegisterSyncProtection(jobID, bfData, validTS1) + require.NoError(t, err) + + // Renew + validTS2 := time.Now().Add(time.Minute).UnixNano() + err = mgr.RenewSyncProtection(jobID, validTS2) + require.NoError(t, err) + + // Renew non-existent job + err = mgr.RenewSyncProtection("non-existent", validTS2) + require.Error(t, err) + assert.Contains(t, err.Error(), "not found") + + // Soft delete and try to renew + err = mgr.UnregisterSyncProtection(jobID) + require.NoError(t, err) + + err = mgr.RenewSyncProtection(jobID, validTS2) + require.Error(t, err) + assert.Contains(t, err.Error(), "soft deleted") +} + +func TestSyncProtectionManager_CleanupSoftDeleted(t *testing.T) { + mgr := NewSyncProtectionManager() + + // Register multiple jobs + job1 := "job-1" + job2 := "job-2" + + validTS1 := int64(1000) + validTS2 := int64(2000) + + bfData1 := buildTestBF(t, []string{"obj1"}) + bfData2 := buildTestBF(t, []string{"obj2"}) + + err := mgr.RegisterSyncProtection(job1, bfData1, validTS1) + require.NoError(t, err) + err = mgr.RegisterSyncProtection(job2, bfData2, validTS2) + require.NoError(t, err) + + // Soft delete both + err = mgr.UnregisterSyncProtection(job1) + require.NoError(t, err) + err = mgr.UnregisterSyncProtection(job2) + require.NoError(t, err) + + assert.Equal(t, 2, mgr.GetProtectionCount()) + + // Cleanup with watermark = 1500 (only job1 should be cleaned) + mgr.CleanupSoftDeleted(1500) + assert.Equal(t, 1, mgr.GetProtectionCount()) + assert.False(t, mgr.HasProtection(job1)) + assert.True(t, mgr.HasProtection(job2)) + + // Cleanup with watermark = 2500 (job2 should be cleaned) + mgr.CleanupSoftDeleted(2500) + assert.Equal(t, 0, mgr.GetProtectionCount()) +} + +func TestSyncProtectionManager_CleanupExpired(t *testing.T) { + mgr := NewSyncProtectionManager() + mgr.ttl = 100 * time.Millisecond // Short TTL for testing + + jobID := "test-job-1" + bfData := buildTestBF(t, []string{"object1"}) + + // Register with old validTS + oldValidTS := time.Now().Add(-200 * time.Millisecond).UnixNano() + err := mgr.RegisterSyncProtection(jobID, bfData, oldValidTS) + require.NoError(t, err) + + assert.Equal(t, 1, mgr.GetProtectionCount()) + + // Cleanup expired + mgr.CleanupExpired() + assert.Equal(t, 0, mgr.GetProtectionCount()) +} + +func TestSyncProtectionManager_CleanupExpired_NotSoftDeleted(t *testing.T) { + mgr := NewSyncProtectionManager() + mgr.ttl = 100 * time.Millisecond + + jobID := "test-job-1" + bfData := buildTestBF(t, []string{"object1"}) + + // Register with old validTS + oldValidTS := time.Now().Add(-200 * time.Millisecond).UnixNano() + err := mgr.RegisterSyncProtection(jobID, bfData, oldValidTS) + require.NoError(t, err) + + // Soft delete it + err = mgr.UnregisterSyncProtection(jobID) + require.NoError(t, err) + + // CleanupExpired should NOT clean soft-deleted entries + mgr.CleanupExpired() + assert.Equal(t, 1, mgr.GetProtectionCount()) // Still exists +} + +func TestSyncProtectionManager_IsProtected(t *testing.T) { + mgr := NewSyncProtectionManager() + + jobID := "job-1" + bfData := buildTestBF(t, []string{"protected-obj"}) + validTS := time.Now().UnixNano() + + err := mgr.RegisterSyncProtection(jobID, bfData, validTS) + require.NoError(t, err) + + assert.True(t, mgr.IsProtected("protected-obj")) + // Note: BloomFilter may have false positives, so we can't assert False for unprotected +} + +func TestSyncProtectionManager_FilterProtectedFiles(t *testing.T) { + mgr := NewSyncProtectionManager() + + jobID := "job-1" + bfData := buildTestBF(t, []string{"protected1", "protected2"}) + validTS := time.Now().UnixNano() + + err := mgr.RegisterSyncProtection(jobID, bfData, validTS) + require.NoError(t, err) + + files := []string{"protected1", "protected2", "unprotected1", "unprotected2"} + canDelete := mgr.FilterProtectedFiles(files) + + // Protected files should be filtered out + assert.NotContains(t, canDelete, "protected1") + assert.NotContains(t, canDelete, "protected2") + // Unprotected files should remain (unless false positive) + // Note: Due to BloomFilter false positives, we can't guarantee unprotected files are in result +} + +func TestSyncProtectionManager_FilterProtectedFiles_NoProtection(t *testing.T) { + mgr := NewSyncProtectionManager() + + files := []string{"file1", "file2", "file3"} + canDelete := mgr.FilterProtectedFiles(files) + + assert.Equal(t, files, canDelete) +} + +func TestSyncProtectionManager_MaxCount(t *testing.T) { + mgr := NewSyncProtectionManager() + mgr.maxCount = 2 // Set low max for testing + + validTS := time.Now().UnixNano() + + // Register up to max + bfData1 := buildTestBF(t, []string{"obj1"}) + bfData2 := buildTestBF(t, []string{"obj2"}) + bfData3 := buildTestBF(t, []string{"obj3"}) + + err := mgr.RegisterSyncProtection("job-1", bfData1, validTS) + require.NoError(t, err) + err = mgr.RegisterSyncProtection("job-2", bfData2, validTS) + require.NoError(t, err) + + // Should fail when max reached + err = mgr.RegisterSyncProtection("job-3", bfData3, validTS) + require.Error(t, err) + assert.Contains(t, err.Error(), "max count reached") +} + +func TestSyncProtectionManager_ConcurrentAccess(t *testing.T) { + mgr := NewSyncProtectionManager() + + // Concurrent register/unregister + done := make(chan bool) + for i := 0; i < 10; i++ { + go func(id int) { + jobID := "job-" + string(rune('0'+id)) + bfData := buildTestBF(t, []string{"obj"}) + validTS := time.Now().UnixNano() + + _ = mgr.RegisterSyncProtection(jobID, bfData, validTS) + _ = mgr.RenewSyncProtection(jobID, validTS+1000) + _ = mgr.UnregisterSyncProtection(jobID) + _ = mgr.IsProtected("obj") + done <- true + }(i) + } + + for i := 0; i < 10; i++ { + <-done + } +} + +func TestSyncProtectionManager_FullWorkflow(t *testing.T) { + mgr := NewSyncProtectionManager() + + // Simulate sync job workflow + jobID := "sync-job-123" + objects := []string{"table1/obj1", "table1/obj2", "table2/obj1"} + bfData := buildTestBF(t, objects) + validTS := time.Now().UnixNano() + + // Step 1: Check GC not running, register protection + assert.False(t, mgr.IsGCRunning()) + err := mgr.RegisterSyncProtection(jobID, bfData, validTS) + require.NoError(t, err) + + // Step 2: Simulate GC starts (should not affect existing protection) + mgr.SetGCRunning(true) + + // Step 3: GC tries to delete files - protected files should be filtered + filesToDelete := []string{"table1/obj1", "table1/obj2", "table2/obj1", "table3/obj1"} + canDelete := mgr.FilterProtectedFiles(filesToDelete) + // Protected files should be filtered out + assert.NotContains(t, canDelete, "table1/obj1") + assert.NotContains(t, canDelete, "table1/obj2") + assert.NotContains(t, canDelete, "table2/obj1") + + // Step 4: GC ends + mgr.SetGCRunning(false) + + // Step 5: Sync job completes, soft delete protection + err = mgr.UnregisterSyncProtection(jobID) + require.NoError(t, err) + + // Step 6: Next GC cleans up soft-deleted protection when watermark > validTS + checkpointWatermark := validTS + 1000000 // Watermark > validTS + mgr.CleanupSoftDeleted(checkpointWatermark) + assert.Equal(t, 0, mgr.GetProtectionCount()) +} + +func TestSyncProtectionManager_CheckpointWatermarkEdgeCase(t *testing.T) { + mgr := NewSyncProtectionManager() + + jobID := "job-1" + bfData := buildTestBF(t, []string{"obj"}) + validTS := int64(1000) + + err := mgr.RegisterSyncProtection(jobID, bfData, validTS) + require.NoError(t, err) + err = mgr.UnregisterSyncProtection(jobID) + require.NoError(t, err) + + // Watermark == validTS: should NOT be cleaned (need strictly greater) + mgr.CleanupSoftDeleted(1000) + assert.Equal(t, 1, mgr.GetProtectionCount()) + + // Watermark > validTS: should be cleaned + mgr.CleanupSoftDeleted(1001) + assert.Equal(t, 0, mgr.GetProtectionCount()) +} + +func TestSyncProtectionManager_InvalidBFData(t *testing.T) { + mgr := NewSyncProtectionManager() + + jobID := "job-1" + validTS := time.Now().UnixNano() + + // Test empty BF + err := mgr.RegisterSyncProtection(jobID, "", validTS) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid sync protection") + + // Test invalid base64 + err = mgr.RegisterSyncProtection(jobID, "invalid-base64!!!", validTS) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid sync protection") + + // Test invalid BloomFilter data + invalidBF := base64.StdEncoding.EncodeToString([]byte("not a bloom filter")) + err = mgr.RegisterSyncProtection(jobID, invalidBF, validTS) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid sync protection") +} + +func TestSyncProtectionManager_FilterProtectedFiles_EmptyFiles(t *testing.T) { + mgr := NewSyncProtectionManager() + + jobID := "job-1" + bfData := buildTestBF(t, []string{"protected1"}) + validTS := time.Now().UnixNano() + + err := mgr.RegisterSyncProtection(jobID, bfData, validTS) + require.NoError(t, err) + + // Test with empty files list + result := mgr.FilterProtectedFiles([]string{}) + assert.Empty(t, result) +} + +func TestSyncProtectionManager_MultipleProtections(t *testing.T) { + mgr := NewSyncProtectionManager() + + // Register multiple protections + bfData1 := buildTestBF(t, []string{"obj1", "obj2"}) + bfData2 := buildTestBF(t, []string{"obj3", "obj4"}) + + validTS := time.Now().UnixNano() + + err := mgr.RegisterSyncProtection("job-1", bfData1, validTS) + require.NoError(t, err) + err = mgr.RegisterSyncProtection("job-2", bfData2, validTS) + require.NoError(t, err) + + // All objects should be protected + assert.True(t, mgr.IsProtected("obj1")) + assert.True(t, mgr.IsProtected("obj2")) + assert.True(t, mgr.IsProtected("obj3")) + assert.True(t, mgr.IsProtected("obj4")) + + // Filter should protect all + files := []string{"obj1", "obj2", "obj3", "obj4", "obj5"} + canDelete := mgr.FilterProtectedFiles(files) + assert.NotContains(t, canDelete, "obj1") + assert.NotContains(t, canDelete, "obj2") + assert.NotContains(t, canDelete, "obj3") + assert.NotContains(t, canDelete, "obj4") +} + +func TestSyncProtectionManager_IsProtected_NoProtections(t *testing.T) { + mgr := NewSyncProtectionManager() + + // No protections registered + assert.False(t, mgr.IsProtected("any-file")) +} + +func TestSyncProtectionManager_SetGCRunning(t *testing.T) { + mgr := NewSyncProtectionManager() + + // Initial state + assert.False(t, mgr.IsGCRunning()) + + // Set running + mgr.SetGCRunning(true) + assert.True(t, mgr.IsGCRunning()) + + // Set not running + mgr.SetGCRunning(false) + assert.False(t, mgr.IsGCRunning()) +} + +func TestSyncProtectionManager_GetProtectionCountByState_Empty(t *testing.T) { + mgr := NewSyncProtectionManager() + + active, softDeleted := mgr.GetProtectionCountByState() + assert.Equal(t, 0, active) + assert.Equal(t, 0, softDeleted) +} + +func TestSyncProtectionManager_CleanupExpired_NoExpired(t *testing.T) { + mgr := NewSyncProtectionManager() + mgr.ttl = time.Hour // Long TTL + + jobID := "test-job-1" + bfData := buildTestBF(t, []string{"object1"}) + validTS := time.Now().UnixNano() + + err := mgr.RegisterSyncProtection(jobID, bfData, validTS) + require.NoError(t, err) + + // Cleanup should not remove anything + mgr.CleanupExpired() + assert.Equal(t, 1, mgr.GetProtectionCount()) +} + +func TestSyncProtectionManager_CleanupSoftDeleted_NoSoftDeleted(t *testing.T) { + mgr := NewSyncProtectionManager() + + jobID := "job-1" + bfData := buildTestBF(t, []string{"obj"}) + validTS := int64(1000) + + err := mgr.RegisterSyncProtection(jobID, bfData, validTS) + require.NoError(t, err) + + // Not soft deleted, cleanup should not remove + mgr.CleanupSoftDeleted(2000) + assert.Equal(t, 1, mgr.GetProtectionCount()) +} diff --git a/pkg/vm/engine/tae/db/gc/v3/types.go b/pkg/vm/engine/tae/db/gc/v3/types.go index da23e712dec3e..7f0df8fd22f5c 100644 --- a/pkg/vm/engine/tae/db/gc/v3/types.go +++ b/pkg/vm/engine/tae/db/gc/v3/types.go @@ -159,6 +159,9 @@ type Cleaner interface { RemoveBackupProtection() GetBackupProtection() (protectedTS types.TS, lastUpdateTime time.Time, isActive bool) + // Sync protection methods (for cross-cluster sync) + GetSyncProtectionManager() *SyncProtectionManager + // For testing GetTablePK(tableId uint64) string } diff --git a/pkg/vm/engine/tae/db/gc/v3/window.go b/pkg/vm/engine/tae/db/gc/v3/window.go index d9303844283be..923dc4fe036a3 100644 --- a/pkg/vm/engine/tae/db/gc/v3/window.go +++ b/pkg/vm/engine/tae/db/gc/v3/window.go @@ -126,6 +126,7 @@ func (w *GCWindow) ExecuteGlobalCheckpointBasedGC( snapshotMeta *logtail.SnapshotMeta, iscpTables map[uint64]types.TS, checkpointCli checkpoint.Runner, + syncProtection *SyncProtectionManager, buffer *containers.OneSchemaBatchBuffer, cacheSize int, estimateRows int, @@ -149,6 +150,7 @@ func (w *GCWindow) ExecuteGlobalCheckpointBasedGC( iscpTables, snapshotMeta, checkpointCli, + syncProtection, buffer, false, mp, diff --git a/pkg/vm/engine/tae/rpc/handle_debug.go b/pkg/vm/engine/tae/rpc/handle_debug.go index 93975d4a1d438..c21d1d56dbfc4 100644 --- a/pkg/vm/engine/tae/rpc/handle_debug.go +++ b/pkg/vm/engine/tae/rpc/handle_debug.go @@ -17,6 +17,7 @@ package rpc import ( "bytes" "context" + "encoding/json" "fmt" "github.com/matrixorigin/matrixone/pkg/clusterservice" "github.com/matrixorigin/matrixone/pkg/pb/metadata" @@ -793,6 +794,71 @@ func (h *Handle) HandleDiskCleaner( case cmd_util.GCVerify: resp.ReturnStr = h.db.DiskCleaner.Verify(ctx) return + case cmd_util.RegisterSyncProtection: + // Register sync protection for cross-cluster sync + // value format: JSON {"job_id": "xxx", "bf": "base64_encoded_bloomfilter", "valid_ts": 1234567890} + if value == "" { + return nil, moerr.NewInvalidArgNoCtx(op, "empty value") + } + + var req cmd_util.SyncProtection + if err = json.Unmarshal([]byte(value), &req); err != nil { + logutil.Error( + "GC-Sync-Protection-Register-Parse-Error", + zap.String("value", value), + zap.Error(err), + ) + return nil, moerr.NewInvalidArgNoCtx(op, value) + } + + syncMgr := h.db.DiskCleaner.GetCleaner().GetSyncProtectionManager() + if err = syncMgr.RegisterSyncProtection(req.JobID, req.BF, req.ValidTS); err != nil { + return nil, err + } + resp.ReturnStr = `{"status": "ok"}` + return + case cmd_util.RenewSyncProtection: + // Renew sync protection valid timestamp + // value format: JSON {"job_id": "xxx", "valid_ts": 1234567890} + if value == "" { + return nil, moerr.NewInvalidArgNoCtx(op, "empty value") + } + var req cmd_util.SyncProtection + if err = json.Unmarshal([]byte(value), &req); err != nil { + logutil.Error( + "GC-Sync-Protection-Renew-Parse-Error", + zap.String("value", value), + zap.Error(err), + ) + return nil, moerr.NewInvalidArgNoCtx(op, value) + } + syncMgr := h.db.DiskCleaner.GetCleaner().GetSyncProtectionManager() + if err = syncMgr.RenewSyncProtection(req.JobID, req.ValidTS); err != nil { + return nil, err + } + resp.ReturnStr = `{"status": "ok"}` + return + case cmd_util.UnregisterSyncProtection: + // Unregister (soft delete) sync protection + // value format: JSON {"job_id": "xxx"} + if value == "" { + return nil, moerr.NewInvalidArgNoCtx(op, "empty value") + } + var req cmd_util.SyncProtection + if err = json.Unmarshal([]byte(value), &req); err != nil { + logutil.Error( + "GC-Sync-Protection-Unregister-Parse-Error", + zap.String("value", value), + zap.Error(err), + ) + return nil, moerr.NewInvalidArgNoCtx(op, value) + } + syncMgr := h.db.DiskCleaner.GetCleaner().GetSyncProtectionManager() + if err = syncMgr.UnregisterSyncProtection(req.JobID); err != nil { + return nil, err + } + resp.ReturnStr = `{"status": "ok"}` + return case cmd_util.AddChecker: break } diff --git a/test/distributed/cases/function/mo_ctl/mo_ctl_sync_protection.result b/test/distributed/cases/function/mo_ctl/mo_ctl_sync_protection.result new file mode 100644 index 0000000000000..7426a6a9eb1e1 --- /dev/null +++ b/test/distributed/cases/function/mo_ctl/mo_ctl_sync_protection.result @@ -0,0 +1,14 @@ +select mo_ctl('dn', 'diskcleaner', 'register_sync_protection.invalid_json'); +invalid argument register_sync_protection, bad value invalid_json +select mo_ctl('dn', 'diskcleaner', 'register_sync_protection.{"job_id":"test-job-1"}'); +invalid sync protection request +select mo_ctl('dn', 'diskcleaner', 'register_sync_protection.{"job_id":"test-job-2","bf":"invalid!!!","valid_ts":1234567890}'); +invalid sync protection request +select mo_ctl('dn', 'diskcleaner', 'renew_sync_protection.{"job_id":"non-existent","valid_ts":1234567890}'); +sync protection not found: non-existent +select mo_ctl('dn', 'diskcleaner', 'unregister_sync_protection.{"job_id":"non-existent"}'); +sync protection not found: non-existent +select mo_ctl('dn', 'diskcleaner', 'register_sync_protection.'); +invalid argument register_sync_protection, bad value empty value +select mo_ctl('dn', 'diskcleaner', 'unknown_sync_protection.{}'); +internal error: handleDiskCleaner: invalid operation! diff --git a/test/distributed/cases/function/mo_ctl/mo_ctl_sync_protection.test b/test/distributed/cases/function/mo_ctl/mo_ctl_sync_protection.test new file mode 100644 index 0000000000000..5838e459cfd07 --- /dev/null +++ b/test/distributed/cases/function/mo_ctl/mo_ctl_sync_protection.test @@ -0,0 +1,23 @@ +-- Test sync protection mo_ctl commands +-- This test verifies the sync protection mechanism for cross-cluster sync + +-- Test 1: Register sync protection with invalid JSON (should fail gracefully) +select mo_ctl('dn', 'diskcleaner', 'register_sync_protection.invalid_json'); + +-- Test 2: Register sync protection with missing fields (should fail) +select mo_ctl('dn', 'diskcleaner', 'register_sync_protection.{"job_id":"test-job-1"}'); + +-- Test 3: Register sync protection with invalid base64 BF (should fail) +select mo_ctl('dn', 'diskcleaner', 'register_sync_protection.{"job_id":"test-job-2","bf":"invalid!!!","valid_ts":1234567890}'); + +-- Test 4: Renew non-existent protection (should fail) +select mo_ctl('dn', 'diskcleaner', 'renew_sync_protection.{"job_id":"non-existent","valid_ts":1234567890}'); + +-- Test 5: Unregister non-existent protection (should fail) +select mo_ctl('dn', 'diskcleaner', 'unregister_sync_protection.{"job_id":"non-existent"}'); + +-- Test 6: Test with empty command +select mo_ctl('dn', 'diskcleaner', 'register_sync_protection.'); + +-- Test 7: Test unknown sync protection command +select mo_ctl('dn', 'diskcleaner', 'unknown_sync_protection.{}');