diff --git a/AnnService/inc/Core/Common.h b/AnnService/inc/Core/Common.h index afa8f1dc7..c555f3c5a 100644 --- a/AnnService/inc/Core/Common.h +++ b/AnnService/inc/Core/Common.h @@ -130,6 +130,8 @@ extern std::shared_ptr(*f_createIO)(); #define IOBINARY(ptr, func, bytes, ...) if (ptr->func(bytes, __VA_ARGS__) != bytes) return ErrorCode::DiskIOFail #define IOSTRING(ptr, func, ...) if (ptr->func(__VA_ARGS__) == 0) return ErrorCode::DiskIOFail +#define RETURN_IF_ERROR(expr) { ErrorCode ret = expr; if (ret != ErrorCode::Success) return ret; } +#define RETURN_IF_ERROR_WITH_LOG(expr, log_level, ...) { ErrorCode ret = expr; if (ret != ErrorCode::Success) { SPTAGLIB_LOG(log_level, __VA_ARGS__); return ret; } } extern Helper::LoggerHolder& GetLoggerHolder(); extern std::shared_ptr GetLogger(); @@ -150,7 +152,7 @@ class MyException : public std::exception #endif }; -enum class ErrorCode : std::uint16_t +enum class [[nodiscard]] ErrorCode : std::uint16_t { #define DefineErrorCode(Name, Value) Name = Value, #include "DefinitionList.h" diff --git a/AnnService/inc/Core/Common/IQuantizer.h b/AnnService/inc/Core/Common/IQuantizer.h index 3e069447e..aa20aa500 100644 --- a/AnnService/inc/Core/Common/IQuantizer.h +++ b/AnnService/inc/Core/Common/IQuantizer.h @@ -39,7 +39,7 @@ namespace SPTAG virtual ErrorCode LoadQuantizer(std::shared_ptr p_in) = 0; - virtual ErrorCode LoadQuantizer(uint8_t* raw_bytes) = 0; + virtual ErrorCode LoadQuantizer(uint8_t* raw_bytes, SizeType num_bytes) = 0; static std::shared_ptr LoadIQuantizer(std::shared_ptr p_in); diff --git a/AnnService/inc/Core/Common/NeighborhoodGraph.h b/AnnService/inc/Core/Common/NeighborhoodGraph.h index a347fc12c..853887728 100644 --- a/AnnService/inc/Core/Common/NeighborhoodGraph.h +++ b/AnnService/inc/Core/Common/NeighborhoodGraph.h @@ -506,6 +506,7 @@ break; newGraph->m_iGraphSize = R; newGraph->m_iNeighborhoodSize = m_iNeighborhoodSize; + ErrorCode ret = ErrorCode::Success; #pragma omp parallel for schedule(dynamic) for (SizeType i = 0; i < R; i++) { @@ -514,7 +515,11 @@ break; SizeType* outnodes = newGraph->m_pNeighborhoodGraph[i]; COMMON::QueryResultSet query((const T*)index->GetSample(indices[i]), m_iCEF + 1); - index->RefineSearchIndex(query, false); + ErrorCode internal_ret =index->RefineSearchIndex(query, false); + if (internal_ret != ErrorCode::Success) { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "RefineSearchIndex failed \n"); + ret = internal_ret; + } RebuildNeighbors(index, indices[i], outnodes, query.GetResults(), m_iCEF + 1); std::unordered_map::const_iterator iter; @@ -527,8 +532,8 @@ break; outnodes[m_iNeighborhoodSize - 1] = -2 - iter->second; } - if (output != nullptr) newGraph->SaveGraph(output); - return ErrorCode::Success; + if (output != nullptr) RETURN_IF_ERROR(newGraph->SaveGraph(output)); + return ret; } template diff --git a/AnnService/inc/Core/Common/OPQQuantizer.h b/AnnService/inc/Core/Common/OPQQuantizer.h index 250a29f7e..8bdd24cef 100644 --- a/AnnService/inc/Core/Common/OPQQuantizer.h +++ b/AnnService/inc/Core/Common/OPQQuantizer.h @@ -34,7 +34,7 @@ namespace SPTAG virtual ErrorCode LoadQuantizer(std::shared_ptr p_in); - virtual ErrorCode LoadQuantizer(std::uint8_t* raw_bytes); + virtual ErrorCode LoadQuantizer(std::uint8_t* raw_bytes, SizeType num_bytes); virtual SizeType ReconstructSize() const; @@ -164,13 +164,26 @@ namespace SPTAG } template - ErrorCode OPQQuantizer::LoadQuantizer(std::uint8_t* raw_bytes) + ErrorCode OPQQuantizer::LoadQuantizer(std::uint8_t* raw_bytes, SizeType num_bytes) { - PQQuantizer::LoadQuantizer(raw_bytes); + std::uint8_t* original_ptr = raw_bytes; + ErrorCode code = PQQuantizer::LoadQuantizer(raw_bytes, num_bytes); + if (code != ErrorCode::Success) + { + return code; + } + + // read OPQ matrix raw_bytes += sizeof(DimensionType) + sizeof(SizeType) + sizeof(DimensionType) + (sizeof(OPQMatrixType) * m_NumSubvectors * m_KsPerSubvector * m_DimPerSubvector); m_matrixDim = m_NumSubvectors * m_DimPerSubvector; m_OPQMatrix = std::make_unique(m_matrixDim * m_matrixDim); - std::memcpy(m_OPQMatrix.get(), raw_bytes, sizeof(OPQMatrixType) * m_matrixDim * m_matrixDim); + + SizeType matrix_bytes = sizeof(OPQMatrixType) * m_matrixDim * m_matrixDim; + if (raw_bytes - original_ptr + matrix_bytes > num_bytes) { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Not enough bytes for OPQ matrix\n"); + return ErrorCode::Fail; + } + std::memcpy(m_OPQMatrix.get(), raw_bytes, matrix_bytes); raw_bytes += sizeof(OPQMatrixType) * m_matrixDim * m_matrixDim; m_InitMatrixTranspose(); diff --git a/AnnService/inc/Core/Common/PQQuantizer.h b/AnnService/inc/Core/Common/PQQuantizer.h index 5bece0225..5e6e043f2 100644 --- a/AnnService/inc/Core/Common/PQQuantizer.h +++ b/AnnService/inc/Core/Common/PQQuantizer.h @@ -49,7 +49,7 @@ namespace SPTAG virtual ErrorCode LoadQuantizer(std::shared_ptr p_in); - virtual ErrorCode LoadQuantizer(std::uint8_t* raw_bytes); + virtual ErrorCode LoadQuantizer(std::uint8_t* raw_bytes, SizeType num_bytes); virtual DimensionType GetNumSubvectors() const; @@ -260,20 +260,38 @@ namespace SPTAG } template - ErrorCode PQQuantizer::LoadQuantizer(std::uint8_t* raw_bytes) + ErrorCode PQQuantizer::LoadQuantizer(std::uint8_t* raw_bytes, SizeType num_bytes) { + std::uint8_t* original_ptr = raw_bytes; SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "Loading Quantizer.\n"); m_NumSubvectors = *(DimensionType*)raw_bytes; raw_bytes += sizeof(DimensionType); + if (raw_bytes - original_ptr > num_bytes) { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Not enough bytes to read NumSubvectors.\n"); + return ErrorCode::MemoryOverFlow; + } SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "After read subvecs: %s.\n", std::to_string(m_NumSubvectors).c_str()); m_KsPerSubvector = *(SizeType*)raw_bytes; raw_bytes += sizeof(SizeType); + if (raw_bytes - original_ptr > num_bytes) { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Not enough bytes to read KsPerSubvector.\n"); + return ErrorCode::MemoryOverFlow; + } SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "After read ks: %s.\n", std::to_string(m_KsPerSubvector).c_str()); m_DimPerSubvector = *(DimensionType*)raw_bytes; raw_bytes += sizeof(DimensionType); + if (raw_bytes - original_ptr > num_bytes) { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Not enough bytes to read DimPerSubvector.\n"); + return ErrorCode::MemoryOverFlow; + } SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "After read dim: %s.\n", std::to_string(m_DimPerSubvector).c_str()); m_codebooks = std::make_unique(m_NumSubvectors * m_KsPerSubvector * m_DimPerSubvector); SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "sizeof(T): %s.\n", std::to_string(sizeof(T)).c_str()); + SizeType codebook_bytes = sizeof(T) * m_NumSubvectors * m_KsPerSubvector * m_DimPerSubvector; + if (raw_bytes - original_ptr + codebook_bytes > num_bytes) { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Not enough bytes to read codebooks.\n"); + return ErrorCode::MemoryOverFlow; + } std::memcpy(m_codebooks.get(), raw_bytes, sizeof(T) * m_NumSubvectors * m_KsPerSubvector * m_DimPerSubvector); SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "After read codebooks.\n"); diff --git a/AnnService/inc/Core/VectorIndex.h b/AnnService/inc/Core/VectorIndex.h index f3fd81b78..2cad1814e 100644 --- a/AnnService/inc/Core/VectorIndex.h +++ b/AnnService/inc/Core/VectorIndex.h @@ -188,7 +188,7 @@ class VectorIndex inline SizeType GetMetaMapping(std::string& meta) const; - void UpdateMetaMapping(const std::string& meta, SizeType i); + ErrorCode UpdateMetaMapping(const std::string& meta, SizeType i); void BuildMetaMapping(bool p_checkDeleted = true); diff --git a/AnnService/src/Core/BKT/BKTIndex.cpp b/AnnService/src/Core/BKT/BKTIndex.cpp index fbd428db6..7ac280b55 100644 --- a/AnnService/src/Core/BKT/BKTIndex.cpp +++ b/AnnService/src/Core/BKT/BKTIndex.cpp @@ -20,11 +20,13 @@ namespace SPTAG template ErrorCode Index::LoadConfig(Helper::IniReader& p_reader) { + ErrorCode code = ErrorCode::Success; #define DefineBKTParameter(VarName, VarType, DefaultValue, RepresentStr) \ - SetParameter(RepresentStr, \ + code = SetParameter(RepresentStr, \ p_reader.GetParameter("Index", \ RepresentStr, \ std::string(#DefaultValue)).c_str()); \ + if (code != ErrorCode::Success) return code; #include "inc/Core/BKT/ParameterDefinitionList.h" #undef DefineBKTParameter @@ -816,7 +818,7 @@ namespace SPTAG ptr->m_deletedID.Initialize(newR, m_iDataBlockSize, m_iDataCapacity, COMMON::Labelset::InvalidIDBehavior::AlwaysContains); COMMON::BKTree* newtree = &(ptr->m_pTrees); (*newtree).BuildTrees(ptr->m_pSamples, ptr->m_iDistCalcMethod, omp_get_num_threads()); - m_pGraph.RefineGraph(this, indices, reverseIndices, nullptr, &(ptr->m_pGraph), &(ptr->m_pTrees.GetSampleMap())); + if ((ret = m_pGraph.RefineGraph(this, indices, reverseIndices, nullptr, &(ptr->m_pGraph), &(ptr->m_pTrees.GetSampleMap()))) != ErrorCode::Success) return ret; if (HasMetaMapping()) ptr->BuildMetaMapping(false); ptr->m_bReady = true; return ret; @@ -875,18 +877,28 @@ namespace SPTAG template ErrorCode Index::DeleteIndex(const void* p_vectors, SizeType p_vectorNum) { const T* ptr_v = (const T*)p_vectors; + ErrorCode ret = ErrorCode::Success; #pragma omp parallel for schedule(dynamic) for (SizeType i = 0; i < p_vectorNum; i++) { COMMON::QueryResultSet query(ptr_v + i * GetFeatureDim(), m_pGraph.m_iCEF); - SearchIndex(query); + ErrorCode search_ret = SearchIndex(query); + if (search_ret != ErrorCode::Success || query.GetResultNum() == 0) { + SPTAGLIB_LOG(Helper::LogLevel::LL_Warning, "Cannot find vector to delete!\n"); + ret = search_ret; + } for (int j = 0; j < m_pGraph.m_iCEF; j++) { if (query.GetResult(j)->Dist < 1e-6) { - DeleteIndex(query.GetResult(j)->VID); + SizeType vid = query.GetResult(j)->VID; + ErrorCode delete_ret = DeleteIndex(vid); + if (delete_ret != ErrorCode::Success) { + SPTAGLIB_LOG(Helper::LogLevel::LL_Warning, "Cannot delete vector! ID: %llu\n", vid); + ret = delete_ret; + } } } } - return ErrorCode::Success; + return ret; } template @@ -940,7 +952,7 @@ namespace SPTAG for (SizeType i = begin; i < end; i++) { ByteArray meta = m_pMetadata->GetMetadata(i); std::string metastr((char*)meta.Data(), meta.Length()); - UpdateMetaMapping(metastr, i); + RETURN_IF_ERROR(UpdateMetaMapping(metastr, i)); } } } diff --git a/AnnService/src/Core/Common/IQuantizer.cpp b/AnnService/src/Core/Common/IQuantizer.cpp index ea6cf3910..c231edbc1 100644 --- a/AnnService/src/Core/Common/IQuantizer.cpp +++ b/AnnService/src/Core/Common/IQuantizer.cpp @@ -55,11 +55,14 @@ namespace SPTAG std::shared_ptr IQuantizer::LoadIQuantizer(SPTAG::ByteArray bytes) { auto raw_bytes = bytes.Data(); + SizeType num_bytes = bytes.Length(); QuantizerType quantizerType = *(QuantizerType*) raw_bytes; raw_bytes += sizeof(QuantizerType); + num_bytes -= sizeof(QuantizerType); VectorValueType reconstructType = *(VectorValueType*)raw_bytes; raw_bytes += sizeof(VectorValueType); + num_bytes -= sizeof(VectorValueType); SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "Loading quantizer of type %s with reconstructtype %s.\n", Helper::Convert::ConvertToString(quantizerType).c_str(), Helper::Convert::ConvertToString(reconstructType).c_str()); std::shared_ptr ret = nullptr; @@ -80,7 +83,7 @@ namespace SPTAG default: break; } - if (ret->LoadQuantizer(raw_bytes) != ErrorCode::Success) ret.reset(); + if (ret->LoadQuantizer(raw_bytes, num_bytes) != ErrorCode::Success) ret.reset(); return ret; case QuantizerType::OPQQuantizer: switch (reconstructType) { @@ -94,7 +97,7 @@ namespace SPTAG default: break; } - if (ret->LoadQuantizer(raw_bytes) != ErrorCode::Success) ret.reset(); + if (ret->LoadQuantizer(raw_bytes, num_bytes) != ErrorCode::Success) ret.reset(); return ret; } return ret; diff --git a/AnnService/src/Core/KDT/KDTIndex.cpp b/AnnService/src/Core/KDT/KDTIndex.cpp index ac7d4db35..bc24d430e 100644 --- a/AnnService/src/Core/KDT/KDTIndex.cpp +++ b/AnnService/src/Core/KDT/KDTIndex.cpp @@ -666,7 +666,7 @@ case VectorValueType::Name: \ for (SizeType i = begin; i < end; i++) { ByteArray meta = m_pMetadata->GetMetadata(i); std::string metastr((char*)meta.Data(), meta.Length()); - UpdateMetaMapping(metastr, i); + RETURN_IF_ERROR(UpdateMetaMapping(metastr, i)); } } } diff --git a/AnnService/src/Core/SPANN/SPANNIndex.cpp b/AnnService/src/Core/SPANN/SPANNIndex.cpp index aecd3ac23..535d63db3 100644 --- a/AnnService/src/Core/SPANN/SPANNIndex.cpp +++ b/AnnService/src/Core/SPANN/SPANNIndex.cpp @@ -66,7 +66,7 @@ namespace SPTAG for (int i = 0; i < 4; i++) { auto parameters = p_reader.GetParameters(sections[i].c_str()); for (auto iter = parameters.begin(); iter != parameters.end(); iter++) { - SetParameter(iter->first.c_str(), iter->second.c_str(), sections[i].c_str()); + RETURN_IF_ERROR(SetParameter(iter->first.c_str(), iter->second.c_str(), sections[i].c_str())); } } @@ -84,10 +84,10 @@ namespace SPTAG m_index->SetQuantizer(m_pQuantizer); if (m_index->LoadIndexDataFromMemory(p_indexBlobs) != ErrorCode::Success) return ErrorCode::Fail; - m_index->SetParameter("NumberOfThreads", std::to_string(m_options.m_iSSDNumberOfThreads)); + RETURN_IF_ERROR(m_index->SetParameter("NumberOfThreads", std::to_string(m_options.m_iSSDNumberOfThreads))); //m_index->SetParameter("MaxCheck", std::to_string(m_options.m_maxCheck)); //m_index->SetParameter("HashTableExponent", std::to_string(m_options.m_hashExp)); - m_index->UpdateIndex(); + RETURN_IF_ERROR(m_index->UpdateIndex()); m_index->SetReady(true); if (m_pQuantizer) @@ -113,10 +113,10 @@ namespace SPTAG m_index->SetQuantizer(m_pQuantizer); if (m_index->LoadIndexData(p_indexStreams) != ErrorCode::Success) return ErrorCode::Fail; - m_index->SetParameter("NumberOfThreads", std::to_string(m_options.m_iSSDNumberOfThreads)); + RETURN_IF_ERROR(m_index->SetParameter("NumberOfThreads", std::to_string(m_options.m_iSSDNumberOfThreads))); //m_index->SetParameter("MaxCheck", std::to_string(m_options.m_maxCheck)); //m_index->SetParameter("HashTableExponent", std::to_string(m_options.m_hashExp)); - m_index->UpdateIndex(); + RETURN_IF_ERROR(m_index->UpdateIndex()); m_index->SetReady(true); if (m_pQuantizer) @@ -161,7 +161,7 @@ namespace SPTAG #include "inc/Core/SPANN/ParameterDefinitionList.h" #undef DefineBuildHeadParameter - m_index->SaveConfig(p_configOut); + RETURN_IF_ERROR_WITH_LOG(m_index->SaveConfig(p_configOut), Helper::LogLevel::LL_Error, "Failed to save head index config.\n"); Helper::Convert::ConvertStringTo(m_index->GetParameter("HashTableExponent").c_str(), m_options.m_hashExp); IOSTRING(p_configOut, WriteString, "[BuildSSDIndex]\n"); @@ -200,7 +200,7 @@ namespace SPTAG else p_queryResults = new COMMON::QueryResultSet((const T*)p_query.GetTarget(), m_options.m_searchInternalResultNum); - m_index->SearchIndex(*p_queryResults); + RETURN_IF_ERROR(m_index->SearchIndex(*p_queryResults)); if (m_extraSearcher != nullptr) { auto workSpace = m_workSpaceFactory->GetWorkSpace(); @@ -751,7 +751,11 @@ namespace SPTAG bktFileNameBuilder << m_options.m_vectorPath << ".bkt." << m_options.m_iBKTKmeansK << "_" << m_options.m_iBKTLeafSize << "_" << m_options.m_iTreeNumber << "_" << m_options.m_iSamples << "_" << static_cast(m_options.m_distCalcMethod) << ".bin"; - bkt->SaveTrees(bktFileNameBuilder.str()); + ErrorCode saveTreesRet = bkt->SaveTrees(bktFileNameBuilder.str()); + if (saveTreesRet != ErrorCode::Success) { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Failed to save BKT to file %s\n", bktFileNameBuilder.str().c_str()); + return false; + } } SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "Finish generating BKT.\n"); @@ -851,11 +855,11 @@ namespace SPTAG auto dims = m_pQuantizer ? m_pQuantizer->GetNumSubvectors() : m_options.m_dim; m_index = SPTAG::VectorIndex::CreateInstance(m_options.m_indexAlgoType, valueType); - m_index->SetParameter("DistCalcMethod", SPTAG::Helper::Convert::ConvertToString(m_options.m_distCalcMethod)); + RETURN_IF_ERROR(m_index->SetParameter("DistCalcMethod", SPTAG::Helper::Convert::ConvertToString(m_options.m_distCalcMethod))); m_index->SetQuantizer(m_pQuantizer); for (const auto& iter : m_headParameters) { - m_index->SetParameter(iter.first.c_str(), iter.second.c_str()); + RETURN_IF_ERROR(m_index->SetParameter(iter.first.c_str(), iter.second.c_str())); } std::shared_ptr vectorOptions(new Helper::ReaderOptions(valueType, dims, VectorFileType::DEFAULT)); @@ -897,10 +901,10 @@ namespace SPTAG m_index->SetQuantizer(m_pQuantizer); if (!CheckHeadIndexType()) return ErrorCode::Fail; - m_index->SetParameter("NumberOfThreads", std::to_string(m_options.m_iSSDNumberOfThreads)); - m_index->SetParameter("MaxCheck", std::to_string(m_options.m_maxCheck)); - m_index->SetParameter("HashTableExponent", std::to_string(m_options.m_hashExp)); - m_index->UpdateIndex(); + RETURN_IF_ERROR(m_index->SetParameter("NumberOfThreads", std::to_string(m_options.m_iSSDNumberOfThreads))); + RETURN_IF_ERROR(m_index->SetParameter("MaxCheck", std::to_string(m_options.m_maxCheck))); + RETURN_IF_ERROR(m_index->SetParameter("HashTableExponent", std::to_string(m_options.m_hashExp))); + RETURN_IF_ERROR(m_index->UpdateIndex()); if (m_pQuantizer) { @@ -1021,11 +1025,20 @@ namespace SPTAG Index::UpdateIndex() { omp_set_num_threads(m_options.m_iSSDNumberOfThreads); - m_index->SetParameter("NumberOfThreads", std::to_string(m_options.m_iSSDNumberOfThreads)); + ErrorCode ret = m_index->SetParameter("NumberOfThreads", std::to_string(m_options.m_iSSDNumberOfThreads)); + if (ret != ErrorCode::Success) + { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Failed to set NumberOfThreads parameter to %d\n", m_options.m_iSSDNumberOfThreads); + return ret; + } //m_index->SetParameter("MaxCheck", std::to_string(m_options.m_maxCheck)); //m_index->SetParameter("HashTableExponent", std::to_string(m_options.m_hashExp)); - m_index->UpdateIndex(); - return ErrorCode::Success; + ret = m_index->UpdateIndex(); + if (ret != ErrorCode::Success) + { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "Failed to update index\n"); + } + return ret; } template @@ -1037,7 +1050,7 @@ namespace SPTAG else m_headParameters[p_param] = p_value; } else { - m_options.SetParameter(p_section, p_param, p_value); + RETURN_IF_ERROR(m_options.SetParameter(p_section, p_param, p_value)); } if (SPTAG::Helper::StrUtils::StrEqualIgnoreCase(p_param, "DistCalcMethod")) { if (m_pQuantizer) diff --git a/AnnService/src/Core/VectorIndex.cpp b/AnnService/src/Core/VectorIndex.cpp index 5c26429e9..2a08c926a 100644 --- a/AnnService/src/Core/VectorIndex.cpp +++ b/AnnService/src/Core/VectorIndex.cpp @@ -232,13 +232,14 @@ VectorIndex::GetMetaMapping(std::string& meta) const } -void +ErrorCode VectorIndex::UpdateMetaMapping(const std::string& meta, SizeType i) { MetadataMap* ptr = static_cast(m_pMetaToVec.get()); auto iter = ptr->find(meta); - if (iter != ptr->end()) DeleteIndex(iter->second);; + if (iter != ptr->end()) RETURN_IF_ERROR(DeleteIndex(iter->second)); (*ptr)[meta] = i; + return ErrorCode::Success; } @@ -331,7 +332,7 @@ VectorIndex::SaveIndex(const std::string& p_folderPath) if (!copyfile(file.c_str(), (newFolder + FolderSep + filename).c_str())) return ErrorCode::DiskIOFail; } - SetParameter("IndexDirectory", p_folderPath, "Base"); + RETURN_IF_ERROR(SetParameter("IndexDirectory", p_folderPath, "Base")); } ErrorCode ret = ErrorCode::Success; @@ -446,7 +447,7 @@ VectorIndex::BuildIndex(std::shared_ptr p_vectorSet, SPTAGLIB_LOG(Helper::LogLevel::LL_Info, "Build meta mapping...\n"); BuildMetaMapping(false); } - BuildIndex(p_vectorSet->GetData(), p_vectorSet->Count(), p_vectorSet->Dimension(), p_normalized, p_shareOwnership); + RETURN_IF_ERROR_WITH_LOG(BuildIndex(p_vectorSet->GetData(), p_vectorSet->Count(), p_vectorSet->Dimension(), p_normalized, p_shareOwnership), Helper::LogLevel::LL_Error, "Building index failed.\n"); return ErrorCode::Success; } @@ -454,12 +455,17 @@ VectorIndex::BuildIndex(std::shared_ptr p_vectorSet, ErrorCode VectorIndex::SearchIndex(const void* p_vector, int p_vectorCount, int p_neighborCount, bool p_withMeta, BasicResult* p_results) const { size_t vectorSize = GetValueTypeSize(GetVectorValueType()) * GetFeatureDim(); + ErrorCode ret = ErrorCode::Success; #pragma omp parallel for schedule(dynamic,10) for (int i = 0; i < p_vectorCount; i++) { QueryResult res((char*)p_vector + i * vectorSize, p_neighborCount, p_withMeta, p_results + i * p_neighborCount); - SearchIndex(res); + ErrorCode ret_internal = SearchIndex(res); + if (ret_internal != ErrorCode::Success) { + SPTAGLIB_LOG(Helper::LogLevel::LL_Error, "SearchIndex failed for vector %d\n", i); + ret = ret_internal; + } } - return ErrorCode::Success; + return ret; } @@ -500,7 +506,11 @@ VectorIndex::MergeIndex(VectorIndex* p_addindex, int p_threadnum, IAbortOperatio ByteArray meta = p_addindex->GetMetadata(i); std::uint64_t offsets[2] = { 0, meta.Length() }; std::shared_ptr p_metaSet(new MemMetadataSet(meta, ByteArray((std::uint8_t*)offsets, sizeof(offsets), false), 1)); - AddIndex(p_addindex->GetSample(i), 1, p_addindex->GetFeatureDim(), p_metaSet); + ErrorCode add_ret = AddIndex(p_addindex->GetSample(i), 1, p_addindex->GetFeatureDim(), p_metaSet); + if (add_ret != ErrorCode::Success) { + SPTAGLIB_LOG(Helper::LogLevel::LL_Warning, "Cannot add vector! ID: %llu\n", i); + ret = add_ret; + } } if (p_abort != nullptr && p_abort->ShouldAbort()) @@ -517,7 +527,11 @@ VectorIndex::MergeIndex(VectorIndex* p_addindex, int p_threadnum, IAbortOperatio if (p_addindex->ContainSample(i)) { - AddIndex(p_addindex->GetSample(i), 1, p_addindex->GetFeatureDim(), nullptr); + ErrorCode add_ret = AddIndex(p_addindex->GetSample(i), 1, p_addindex->GetFeatureDim(), nullptr); + if (add_ret != ErrorCode::Success) { + SPTAGLIB_LOG(Helper::LogLevel::LL_Warning, "Cannot add vector! ID: %llu\n", i); + ret = add_ret; + } } if (p_abort != nullptr && p_abort->ShouldAbort())