infiniflow · yuzhichang · May 29, 2025 · May 15, 2025 · May 16, 2025 · May 16, 2025
diff --git a/src/executor/operator/physical_scan/physical_knn_scan.cpp b/src/executor/operator/physical_scan/physical_knn_scan.cpp
@@ -640,11 +640,7 @@ void PhysicalKnnScan::ExecuteInternalByColumnDataTypeAndQueryDataType(QueryConte
             if (!status.ok()) {
                 UnrecoverableError(status.message());
             }
-            SharedPtr<MemIndex> mem_index;
-            status = segment_index_meta->GetMemIndex(mem_index);
-            if (!status.ok()) {
-                UnrecoverableError(status.message());
-            }
+            SharedPtr<MemIndex> mem_index = segment_index_meta->GetMemIndex();
             return std::make_tuple(chunk_ids_ptr, mem_index);
         };
 

diff --git a/src/executor/operator/physical_scan/physical_match_sparse_scan.cpp b/src/executor/operator/physical_scan/physical_match_sparse_scan.cpp
@@ -582,12 +582,6 @@ void PhysicalMatchSparseScan::ExecuteInnerT(DistFunc *dist_func,
             if (!status.ok()) {
                 UnrecoverableError(status.message());
             }
-            SharedPtr<MemIndex> mem_index;
-            status = segment_index_meta->GetMemIndex(mem_index);
-            if (!status.ok()) {
-                UnrecoverableError(status.message());
-            }
-
             for (ChunkID chunk_id : *chunk_ids_ptr) {
                 ChunkIndexMeta chunk_index_meta(chunk_id, *segment_index_meta);
                 BufferObj *index_buffer = nullptr;
@@ -604,8 +598,12 @@ void PhysicalMatchSparseScan::ExecuteInnerT(DistFunc *dist_func,
                 bmp_search(*bmp_index, 0, false, filter);
 #endif
             }
-            if (mem_index && mem_index->memory_bmp_index_) {
-                bmp_search(mem_index->memory_bmp_index_->get(), 0, true, filter);
+            SharedPtr<MemIndex> mem_index = segment_index_meta->GetMemIndex();
+            if (mem_index) {
+                SharedPtr<BMPIndexInMem> bmp_index = mem_index->GetBMPIndex();
+                if (bmp_index) {
+                    bmp_search(bmp_index->get(), 0, true, filter);
+                }
             }
         };
 

diff --git a/src/executor/operator/physical_scan/physical_match_tensor_scan.cpp b/src/executor/operator/physical_scan/physical_match_tensor_scan.cpp
@@ -388,14 +388,10 @@ void PhysicalMatchTensorScan::ExecuteInner(QueryContext *query_context, MatchTen
             if (!status.ok()) {
                 UnrecoverableError(status.message());
             }
-            SharedPtr<MemIndex> mem_index;
-            status = segment_index_meta.GetMemIndex(mem_index);
-            if (!status.ok()) {
-                UnrecoverableError(status.message());
-            }
-
+            SharedPtr<MemIndex> mem_index = segment_index_meta.GetMemIndex();
+            SharedPtr<EMVBIndexInMem> emvb_index_in_mem = mem_index == nullptr ? nullptr : mem_index->GetEMVBIndex();
             // 1. in mem index
-            if (const EMVBIndexInMem *emvb_index_in_mem = mem_index->memory_emvb_index_.get(); emvb_index_in_mem) {
+            if (emvb_index_in_mem) {
                 // TODO: fix the parameters
                 const auto result =
                     emvb_index_in_mem->SearchWithBitmask(reinterpret_cast<const f32 *>(calc_match_tensor_expr_->query_embedding_.ptr),

diff --git a/src/main/infinity_context.cpp b/src/main/infinity_context.cpp
@@ -498,6 +498,7 @@ void InfinityContext::UnInit() {
     //    commiting_thread_pool_.stop(true);
     //    hnsw_build_thread_pool_.stop(true);
 
+    VirtualStore::MunmapAllFiles();
     session_mgr_.reset();
     resource_manager_.reset();
     LOG_INFO("Infinity context is un-initialized.");

diff --git a/src/planner/optimizer/index_scan/index_filter_evaluators.cpp b/src/planner/optimizer/index_scan/index_filter_evaluators.cpp
@@ -781,13 +781,12 @@ Bitmask ExecuteSingleRangeT(const Pair<ConvertToOrderedType<ColumnValueType>, Co
             }
             trunk_readers.emplace_back(MakeUnique<TrunkReaderT<ColumnValueType>>(segment_row_count, index_buffer));
         }
-        SharedPtr<MemIndex> mem_index;
-        status = index_meta->GetMemIndex(mem_index);
-        if (!status.ok()) {
-            UnrecoverableError(status.message());
-        }
-        if (mem_index && mem_index->memory_secondary_index_) {
-            trunk_readers.emplace_back(MakeUnique<TrunkReaderM<ColumnValueType>>(segment_row_count, mem_index->memory_secondary_index_));
+        SharedPtr<MemIndex> mem_index = index_meta->GetMemIndex();
+        if (mem_index) {
+            SharedPtr<SecondaryIndexInMem> secondary_index = mem_index->GetSecondaryIndex();
+            if (secondary_index) {
+                trunk_readers.emplace_back(MakeUnique<TrunkReaderM<ColumnValueType>>(segment_row_count, secondary_index));
+            }
         }
     }
     // output result

diff --git a/src/storage/buffer/buffer_obj.cpp b/src/storage/buffer/buffer_obj.cpp
@@ -208,9 +208,6 @@ bool BufferObj::Save(const FileWorkerSaveCtx &ctx) {
         buffer_mgr_->MoveTemp(this);
         file_worker_->MoveFile();
         type_ = BufferType::kPersistent;
-    } else if (type_ == BufferType::kMmap) {
-        String error_message = fmt::format("Invalid buffer type: mmap, {}", GetFilename());
-        UnrecoverableError(error_message);
     }
     return write;
 }

diff --git a/src/storage/buffer/file_worker/bmp_index_file_worker.cpp b/src/storage/buffer/file_worker/bmp_index_file_worker.cpp
@@ -186,7 +186,7 @@ bool BMPIndexFileWorker::ReadFromMmapImpl(const void *ptr, SizeT size) {
         UnrecoverableError("Data is already allocated.");
     }
 #ifdef INDEX_HANDLER
-    mmap_data_ = reinterpret_cast<u8 *>(new BMPHandlerPtr(BMPHandler::Make(index_base_.get(), column_def_.get()).release()));
+    mmap_data_ = reinterpret_cast<u8 *>(new BMPHandlerPtr(BMPHandler::Make(index_base_.get(), column_def_.get(), false).release()));
     auto *bmp_handler = reinterpret_cast<BMPHandlerPtr *>(mmap_data_);
     (*bmp_handler)->LoadFromPtr(static_cast<const char *>(ptr), size);
 #else

diff --git a/src/storage/buffer/file_worker/file_worker.cpp b/src/storage/buffer/file_worker/file_worker.cpp
@@ -244,7 +244,7 @@ void FileWorker::CleanupFile(KVInstance *kv_instance) const {
 void FileWorker::CleanupTempFile() const {
     String path = fmt::format("{}/{}", ChooseFileDir(true), *file_name_);
     if (VirtualStore::Exists(path)) {
-        LOG_INFO(fmt::format("Clean file: {}", path));
+        LOG_INFO(fmt::format("Clean temp file: {}", path));
         VirtualStore::DeleteFile(path);
     } else {
         String error_message = fmt::format("Cleanup: File {} not found for deletion", path);

diff --git a/src/storage/buffer/file_worker/var_file_worker.cpp b/src/storage/buffer/file_worker/var_file_worker.cpp
@@ -106,6 +106,8 @@ void VarFileWorker::ReadFromFileImpl(SizeT file_size, bool from_spill) {
     if (file_size < buffer_size_) {
         String error_message = fmt::format("File: {} size {} is smaller than buffer size {}.", GetFilePath(), file_size, buffer_size_);
         UnrecoverableError(error_message);
+    } else {
+        buffer_size_ = file_size;
     }
 
     auto buffer = MakeUnique<char[]>(buffer_size_);
@@ -125,6 +127,8 @@ bool VarFileWorker::ReadFromMmapImpl(const void *ptr, SizeT file_size) {
     if (file_size < buffer_size_) {
         String error_message = fmt::format("File size {} is smaller than buffer size {}.", file_size, buffer_size_);
         UnrecoverableError(error_message);
+    } else {
+        buffer_size_ = file_size;
     }
     auto *var_buffer = new VarBuffer(buffer_obj_, static_cast<const char *>(ptr), buffer_size_);
     mmap_data_ = reinterpret_cast<u8 *>(var_buffer);

diff --git a/src/storage/catalog/catalog_cache.cpp b/src/storage/catalog/catalog_cache.cpp
@@ -30,7 +30,14 @@ namespace infinity {
 
 TableCache::TableCache(u64 table_id, SegmentID unsealed_segment_id, SegmentOffset unsealed_segment_offset, SegmentID next_segment_id)
     : prepare_segment_id_(unsealed_segment_id), prepare_segment_offset_(unsealed_segment_offset), commit_segment_id_(unsealed_segment_id),
-      commit_segment_offset_(unsealed_segment_offset), table_id_(table_id), next_segment_id_(next_segment_id) {}
+      commit_segment_offset_(unsealed_segment_offset), table_id_(table_id), next_segment_id_(next_segment_id) {
+    if (commit_segment_offset_ != 0) {
+        // Used when system is restarted and there's an unsealed segment.
+        unsealed_segment_cache_ = MakeShared<SegmentCache>(prepare_segment_id_, commit_segment_offset_);
+        unsealed_segment_cache_->sealed_ = false;
+        LOG_INFO(fmt::format("TableCache initialized with unsealed_segment_cache_({}, {})", prepare_segment_id_, commit_segment_offset_));
+    }
+}
 
 SharedPtr<AppendPrepareInfo> TableCache::PrepareAppendNolock(SizeT row_count, TransactionID txn_id) {
     if (row_count > MAX_BLOCK_CAPACITY or row_count == 0) {
@@ -40,19 +47,24 @@ SharedPtr<AppendPrepareInfo> TableCache::PrepareAppendNolock(SizeT row_count, Tr
     SharedPtr<AppendPrepareInfo> append_info = MakeShared<AppendPrepareInfo>();
     append_info->transaction_id_ = txn_id;
     if (unsealed_segment_cache_ == nullptr) {
+        // Used when system is restarted and all segments are sealed.
         unsealed_segment_cache_ = MakeShared<SegmentCache>(next_segment_id_, row_count);
         unsealed_segment_cache_->sealed_ = false;
-
         // Update prepare info
         prepare_segment_id_ = next_segment_id_;
         prepare_segment_offset_ = row_count;
 
         append_info->ranges_.emplace_back(RowID(next_segment_id_, 0), row_count);
+        LOG_DEBUG(fmt::format("TableCache.PrepareAppendNolock allocated range({}.{}, {})", next_segment_id_, 0, row_count));
         ++next_segment_id_;
     } else {
         if (unsealed_segment_cache_->row_count_ + row_count < DEFAULT_SEGMENT_CAPACITY) {
             // Don't need to add a new segment
             append_info->ranges_.emplace_back(RowID(unsealed_segment_cache_->segment_id_, unsealed_segment_cache_->row_count_), row_count);
+            LOG_DEBUG(fmt::format("TableCache.PrepareAppendNolock allocated range({}.{}, {})",
+                                  unsealed_segment_cache_->segment_id_,
+                                  unsealed_segment_cache_->row_count_,
+                                  row_count));
             unsealed_segment_cache_->row_count_ += row_count;
 
             prepare_segment_offset_ += row_count;
@@ -62,11 +74,19 @@ SharedPtr<AppendPrepareInfo> TableCache::PrepareAppendNolock(SizeT row_count, Tr
             unsealed_segment_cache_->row_count_ += row_count;
             unsealed_segment_cache_->sealed_ = true;
             segment_cache_map_.emplace(unsealed_segment_cache_->segment_id_, unsealed_segment_cache_);
+            LOG_DEBUG(fmt::format("TableCache.PrepareAppendNolock allocated range({}.{}, {})",
+                                  unsealed_segment_cache_->segment_id_,
+                                  unsealed_segment_cache_->row_count_,
+                                  row_count));
 
             prepare_segment_offset_ += row_count;
         } else {
             SizeT first_row_count = DEFAULT_SEGMENT_CAPACITY - unsealed_segment_cache_->row_count_;
             append_info->ranges_.emplace_back(RowID(unsealed_segment_cache_->segment_id_, unsealed_segment_cache_->row_count_), first_row_count);
+            LOG_DEBUG(fmt::format("TableCache.PrepareAppendNolock allocated first range({}.{}, {})",
+                                  unsealed_segment_cache_->segment_id_,
+                                  unsealed_segment_cache_->row_count_,
+                                  first_row_count));
             unsealed_segment_cache_->row_count_ += first_row_count;
             unsealed_segment_cache_->sealed_ = true;
             segment_cache_map_.emplace(unsealed_segment_cache_->segment_id_, unsealed_segment_cache_);
@@ -80,6 +100,7 @@ SharedPtr<AppendPrepareInfo> TableCache::PrepareAppendNolock(SizeT row_count, Tr
             unsealed_segment_cache_ = MakeShared<SegmentCache>(next_segment_id_, second_row_count);
             unsealed_segment_cache_->sealed_ = false;
             append_info->ranges_.emplace_back(RowID(next_segment_id_, 0), second_row_count);
+            LOG_DEBUG(fmt::format("TableCache.PrepareAppendNolock allocated second range({}.{}, {})", next_segment_id_, 0, first_row_count));
             ++next_segment_id_;
         }
     }

diff --git a/src/storage/catalog/mem_index.cppm b/src/storage/catalog/mem_index.cppm
@@ -41,6 +41,31 @@ export struct MemIndex {
     void ClearMemIndex();
 
     BaseMemIndex *GetBaseMemIndex(const MemIndexID &mem_index_id);
+    SharedPtr<HnswIndexInMem> GetHnswIndex() {
+        std::unique_lock<std::mutex> lock(mtx_);
+        return memory_hnsw_index_;
+    }
+    SharedPtr<IVFIndexInMem> GetIVFIndex() {
+        std::unique_lock<std::mutex> lock(mtx_);
+        return memory_ivf_index_;
+    }
+    SharedPtr<MemoryIndexer> GetFulltextIndex() {
+        std::unique_lock<std::mutex> lock(mtx_);
+        return memory_indexer_;
+    }
+    SharedPtr<SecondaryIndexInMem> GetSecondaryIndex() {
+        std::unique_lock<std::mutex> lock(mtx_);
+        return memory_secondary_index_;
+    }
+    SharedPtr<EMVBIndexInMem> GetEMVBIndex() {
+        std::unique_lock<std::mutex> lock(mtx_);
+        return memory_emvb_index_;
+    }
+
+    SharedPtr<BMPIndexInMem> GetBMPIndex() {
+        std::unique_lock<std::mutex> lock(mtx_);
+        return memory_bmp_index_;
+    }
 
     std::mutex mtx_; // Used by append / mem index dump / clear
 

diff --git a/src/storage/catalog/meta/column_meta.cpp b/src/storage/catalog/meta/column_meta.cpp
@@ -50,11 +50,12 @@ Status ColumnMeta::SetChunkOffset(SizeT chunk_offset) {
 }
 
 Status ColumnMeta::InitSet() {
-    Status status = SetChunkOffset(0);
-    if (!status.ok()) {
-        return status;
-    }
+    // Status status = SetChunkOffset(0);
+    // if (!status.ok()) {
+    //     return status;
+    // }
 
+    Status status;
     SharedPtr<ColumnDef> col_def;
     {
         SharedPtr<Vector<SharedPtr<ColumnDef>>> column_defs_ptr;
@@ -145,10 +146,10 @@ Status ColumnMeta::LoadSet() {
         auto filename = MakeShared<String>(fmt::format("col_{}_out_0", col_def->id()));
 
         SizeT chunk_offset = 0;
-        status = this->GetChunkOffset(chunk_offset);
-        if (!status.ok()) {
-            return status;
-        }
+        // status = this->GetChunkOffset(chunk_offset);
+        // if (!status.ok()) {
+        //     return status;
+        // }
 
         auto outline_file_worker = MakeUnique<VarFileWorker>(MakeShared<String>(InfinityContext::instance().config()->DataDir()),
                                                              MakeShared<String>(InfinityContext::instance().config()->TempDir()),

diff --git a/src/storage/catalog/meta/meta_key.cpp b/src/storage/catalog/meta/meta_key.cpp
@@ -107,6 +107,8 @@ String PmPathMetaKey::ToString() const { return fmt::format("pm_path: {}:{}", Ke
 
 String PmObjectMetaKey::ToString() const { return fmt::format("pm_object: {}:{}", KeyEncode::PMObjectStatKey(object_key_), value_); }
 
+String DropMetaKey::ToString() const { return fmt::format("drop_key: drop|{}:{}", object_key_, value_); }
+
 nlohmann::json DBMetaKey::ToJson() const {
     nlohmann::json json_res;
     json_res["db_id"] = std::stoull(db_id_str_);
@@ -250,6 +252,13 @@ nlohmann::json PmObjectMetaKey::ToJson() const {
     return json_res;
 }
 
+nlohmann::json DropMetaKey::ToJson() const {
+    nlohmann::json json_res;
+    json_res["drop_key"] = object_key_;
+    json_res["value"] = nlohmann::json::parse(value_);
+    return json_res;
+}
+
 SharedPtr<MetaKey> MetaParse(const String &key, const String &value) {
     Vector<String> fields = infinity::Partition(key, '|');
 
@@ -409,6 +418,13 @@ SharedPtr<MetaKey> MetaParse(const String &key, const String &value) {
         UnrecoverableError(fmt::format("Unexpected key: {}:{}", key, value));
     }
 
+    if (fields[0] == "drop") {
+        const String &object_key = fields[1];
+        SharedPtr<DropMetaKey> drop_meta_key = MakeShared<DropMetaKey>(object_key);
+        drop_meta_key->value_ = value;
+        return drop_meta_key;
+    }
+
     const String &tag_name_str = fields[0];
     SharedPtr<SystemTagMetaKey> system_tag_meta_key = MakeShared<SystemTagMetaKey>(tag_name_str);
     system_tag_meta_key->value_ = value;

diff --git a/src/storage/catalog/meta/meta_key.cppm b/src/storage/catalog/meta/meta_key.cppm
@@ -284,6 +284,16 @@ export struct PmObjectMetaKey final : public MetaKey {
     nlohmann::json ToJson() const final;
 };
 
+export struct DropMetaKey final : public MetaKey {
+    DropMetaKey(String object_key) : MetaKey(MetaType::kDrop), object_key_(std::move(object_key)) {}
+
+    String object_key_{};
+    String value_{};
+
+    String ToString() const final;
+    nlohmann::json ToJson() const final;
+};
+
 export struct SystemTagMetaKey final : public MetaKey {
     SystemTagMetaKey(String tag_name) : MetaKey(MetaType::kSystemTag), tag_name_(std::move(tag_name)) {}
 

diff --git a/src/storage/catalog/meta/meta_type.cppm b/src/storage/catalog/meta/meta_type.cppm
@@ -40,6 +40,7 @@ export enum class MetaType {
     kSystemTag,
     kPmPath,
     kPmObject,
+    kDrop,
 };
 
 }