llvm · cachemeifyoucan · Oct 29, 2024
diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt
@@ -837,6 +837,13 @@ option (LLVM_ENABLE_SPHINX "Use Sphinx to generate llvm documentation." OFF)
 option (LLVM_ENABLE_OCAMLDOC "Build OCaml bindings documentation." ON)
 option (LLVM_ENABLE_BINDINGS "Build bindings." ON)
 
+if(UNIX AND CMAKE_SIZEOF_VOID_P GREATER_EQUAL 8)
+  set(LLVM_ENABLE_ONDISK_CAS_default ON)
+else()
+  set(LLVM_ENABLE_ONDISK_CAS_default OFF)
+endif()
+option(LLVM_ENABLE_ONDISK_CAS "Build OnDiskCAS." ${LLVM_ENABLE_ONDISK_CAS_default})
+
 set(LLVM_INSTALL_DOXYGEN_HTML_DIR "${CMAKE_INSTALL_DOCDIR}/llvm/doxygen-html"
     CACHE STRING "Doxygen-generated HTML documentation install directory")
 set(LLVM_INSTALL_OCAMLDOC_HTML_DIR "${CMAKE_INSTALL_DOCDIR}/llvm/ocaml-html"

diff --git a/llvm/include/llvm/CAS/MappedFileRegionBumpPtr.h b/llvm/include/llvm/CAS/MappedFileRegionBumpPtr.h
@@ -0,0 +1,110 @@
+//===- MappedFileRegionBumpPtr.h --------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CAS_MAPPEDFILEREGIONBUMPPTR_H
+#define LLVM_CAS_MAPPEDFILEREGIONBUMPPTR_H
+
+#include "llvm/Support/Alignment.h"
+#include "llvm/Support/FileSystem.h"
+#include <atomic>
+
+namespace llvm::cas {
+
+/// Allocator for an owned mapped file region that supports thread-safe and
+/// process-safe bump pointer allocation.
+///
+/// This allocator is designed to create a sparse file when supported by the
+/// filesystem's \c ftruncate so that it can be used with a large maximum size.
+/// It will also attempt to shrink the underlying file down to its current
+/// allocation size when the last concurrent mapping is closed.
+///
+/// Process-safe. Uses file locks when resizing the file during initialization
+/// and destruction.
+///
+/// Thread-safe, assuming all threads use the same instance to talk to a given
+/// file/mapping. Unsafe to have multiple instances talking to the same file
+/// in the same process since file locks will misbehave. Clients should
+/// coordinate (somehow).
+///
+/// \note Currently we allocate the whole file without sparseness on Windows.
+///
+/// Provides 8-byte alignment for all allocations.
+class MappedFileRegionBumpPtr {
+public:
+  using RegionT = sys::fs::mapped_file_region;
+
+  /// Create a \c MappedFileRegionBumpPtr.
+  ///
+  /// \param Path the path to open the mapped region.
+  /// \param Capacity the maximum size for the mapped file region.
+  /// \param BumpPtrOffset the offset at which to store the bump pointer.
+  /// \param NewFileConstructor is for constructing new files. It has exclusive
+  /// access to the file. Must call \c initializeBumpPtr.
+  static Expected<MappedFileRegionBumpPtr>
+  create(const Twine &Path, uint64_t Capacity, int64_t BumpPtrOffset,
+         function_ref<Error(MappedFileRegionBumpPtr &)> NewFileConstructor);
+
+  /// Finish initializing the bump pointer. Must be called by
+  /// \c NewFileConstructor.
+  void initializeBumpPtr(int64_t BumpPtrOffset);
+
+  /// Minimum alignment for allocations, currently hardcoded to 8B.
+  static constexpr Align getAlign() {
+    // Trick Align into giving us '8' as a constexpr.
+    struct alignas(8) T {};
+    static_assert(alignof(T) == 8, "Tautology failed?");
+    return Align::Of<T>();
+  }
+
+  /// Allocate at least \p AllocSize. Rounds up to \a getAlign().
+  char *allocate(uint64_t AllocSize) {
+    return data() + allocateOffset(AllocSize);
+  }
+  /// Allocate, returning the offset from \a data() instead of a pointer.
+  int64_t allocateOffset(uint64_t AllocSize);
+
+  char *data() const { return Region.data(); }
+  uint64_t size() const { return *BumpPtr; }
+  uint64_t capacity() const { return Region.size(); }
+
+  RegionT &getRegion() { return Region; }
+
+  ~MappedFileRegionBumpPtr() { destroyImpl(); }
+
+  MappedFileRegionBumpPtr() = default;
+  MappedFileRegionBumpPtr(MappedFileRegionBumpPtr &&RHS) { moveImpl(RHS); }
+  MappedFileRegionBumpPtr &operator=(MappedFileRegionBumpPtr &&RHS) {
+    destroyImpl();
+    moveImpl(RHS);
+    return *this;
+  }
+
+  MappedFileRegionBumpPtr(const MappedFileRegionBumpPtr &) = delete;
+  MappedFileRegionBumpPtr &operator=(const MappedFileRegionBumpPtr &) = delete;
+
+private:
+  void destroyImpl();
+  void moveImpl(MappedFileRegionBumpPtr &RHS) {
+    std::swap(Region, RHS.Region);
+    std::swap(BumpPtr, RHS.BumpPtr);
+    std::swap(Path, RHS.Path);
+    std::swap(FD, RHS.FD);
+    std::swap(SharedLockFD, RHS.SharedLockFD);
+  }
+
+private:
+  RegionT Region;
+  std::atomic<int64_t> *BumpPtr = nullptr;
+  std::string Path;
+  std::optional<int> FD;
+  std::optional<int> SharedLockFD;
+};
+
+} // namespace llvm::cas
+
+#endif // LLVM_CAS_MAPPEDFILEREGIONBUMPPTR_H
diff --git a/llvm/lib/CAS/CMakeLists.txt b/llvm/lib/CAS/CMakeLists.txt
@@ -1,9 +1,15 @@
+if (LLVM_ENABLE_ONDISK_CAS)
+  add_definitions(-DLLVM_ENABLE_ONDISK_CAS=1)
+endif()
+
 add_llvm_component_library(LLVMCAS
   ActionCache.cpp
   ActionCaches.cpp
   BuiltinCAS.cpp
   InMemoryCAS.cpp
+  MappedFileRegionBumpPtr.cpp
   ObjectStore.cpp
+  OnDiskCommon.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/CAS

diff --git a/llvm/lib/CAS/MappedFileRegionBumpPtr.cpp b/llvm/lib/CAS/MappedFileRegionBumpPtr.cpp
@@ -0,0 +1,233 @@
+//===- MappedFileRegionBumpPtr.cpp ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+/// \file
+///
+/// A bump pointer allocator, backed by a memory-mapped file.
+///
+/// The effect we want is:
+///
+/// 1. If it doesn't exist, create the file with an initial size.
+/// 2. Reserve virtual memory large enough for the max file size.
+/// 3. Map the file into memory in the reserved region.
+/// 4. Increase the file size and update the mapping when necessary.
+///
+/// However, updating the mapping is challenging when it needs to work portably,
+/// and across multiple processes without locking for every read. Our current
+/// implementation strategy is:
+///
+/// 1. Use \c ftruncate (\c sys::fs::resize_file) to grow the file to its max
+///    size (typically several GB). Many modern filesystems will create a sparse
+///    file, so that the trailing unused pages do not take space on disk.
+/// 2. Call \c mmap (\c sys::fs::mapped_file_region)
+/// 3. [Automatic as part of 2.]
+/// 4. [Automatic as part of 2.]
+///
+/// Additionally, we attempt to resize the file to its actual data size when
+/// closing the mapping, if this is the only concurrent instance. This is done
+/// using file locks. Shrinking the file mitigates problems with having large
+/// files: on filesystems without sparse files it avoids unnecessary space use;
+/// it also avoids allocating the full size if another process copies the file,
+/// which typically loses sparseness. These mitigations only work while the file
+/// is not in use.
+///
+/// FIXME: we assume that all concurrent users of the file will use the same
+/// value for Capacity. Otherwise a process with a larger capacity can write
+/// data that is "out of bounds" for processes with smaller capacity. Currently
+/// this is true in the CAS.
+///
+/// To support resizing, we use two separate file locks:
+/// 1. We use a shared reader lock on a ".shared" file until destruction.
+/// 2. We use a lock on the main file during initialization - shared to check
+///    the status, upgraded to exclusive to resize/initialize the file.
+///
+/// Then during destruction we attempt to get exclusive access on (1), which
+/// requires no concurrent readers. If so, we shrink the file. Using two
+/// separate locks simplifies the implementation and enables it to work on
+/// platforms (e.g. Windows) where a shared/reader lock prevents writing.
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CAS/MappedFileRegionBumpPtr.h"
+#include "OnDiskCommon.h"
+
+using namespace llvm;
+using namespace llvm::cas;
+using namespace llvm::cas::ondisk;
+
+namespace {
+struct FileLockRAII {
+  std::string Path;
+  int FD;
+  enum LockKind { Shared, Exclusive };
+  std::optional<LockKind> Locked;
+
+  FileLockRAII(std::string Path, int FD) : Path(std::move(Path)), FD(FD) {}
+  ~FileLockRAII() { consumeError(unlock()); }
+
+  Error lock(LockKind LK) {
+    if (std::error_code EC = lockFileThreadSafe(FD, LK == Exclusive))
+      return createFileError(Path, EC);
+    Locked = LK;
+    return Error::success();
+  }
+
+  Error unlock() {
+    if (Locked) {
+      Locked = std::nullopt;
+      if (std::error_code EC = unlockFileThreadSafe(FD))
+        return createFileError(Path, EC);
+    }
+    return Error::success();
+  }
+};
+} // end anonymous namespace
+
+Expected<MappedFileRegionBumpPtr> MappedFileRegionBumpPtr::create(
+    const Twine &Path, uint64_t Capacity, int64_t BumpPtrOffset,
+    function_ref<Error(MappedFileRegionBumpPtr &)> NewFileConstructor) {
+  MappedFileRegionBumpPtr Result;
+  Result.Path = Path.str();
+  // Open the main file.
+  int FD;
+  if (std::error_code EC = sys::fs::openFileForReadWrite(
+          Result.Path, FD, sys::fs::CD_OpenAlways, sys::fs::OF_None))
+    return createFileError(Path, EC);
+  Result.FD = FD;
+
+  // Open the shared lock file. See file comment for details of locking scheme.
+  SmallString<128> SharedLockPath(Result.Path);
+  SharedLockPath.append(".shared");
+  int SharedLockFD;
+  if (std::error_code EC = sys::fs::openFileForReadWrite(
+          SharedLockPath, SharedLockFD, sys::fs::CD_OpenAlways,
+          sys::fs::OF_None))
+    return createFileError(SharedLockPath, EC);
+  Result.SharedLockFD = SharedLockFD;
+
+  // Take shared/reader lock that will be held until we close the file; unlocked
+  // by destroyImpl.
+  if (std::error_code EC =
+          lockFileThreadSafe(SharedLockFD, /*Exclusive=*/false))
+    return createFileError(Path, EC);
+
+  // Take shared/reader lock for initialization.
+  FileLockRAII InitLock(Result.Path, FD);
+  if (Error E = InitLock.lock(FileLockRAII::Shared))
+    return std::move(E);
+
+  sys::fs::file_t File = sys::fs::convertFDToNativeFile(FD);
+  sys::fs::file_status Status;
+  if (std::error_code EC = sys::fs::status(File, Status))
+    return createFileError(Result.Path, EC);
+
+  if (Status.getSize() < Capacity) {
+    // Lock the file exclusively so only one process will do the initialization.
+    if (Error E = InitLock.unlock())
+      return std::move(E);
+    if (Error E = InitLock.lock(FileLockRAII::Exclusive))
+      return std::move(E);
+    // Retrieve the current size now that we have exclusive access.
+    if (std::error_code EC = sys::fs::status(File, Status))
+      return createFileError(Result.Path, EC);
+  }
+
+  // At this point either the file is still under-sized, or we have the size for
+  // the completely initialized file.
+
+  if (Status.getSize() < Capacity) {
+    // We are initializing the file; it may be empty, or may have been shrunk
+    // during a previous close.
+    // FIXME: Detect a case where someone opened it with a smaller capacity.
+    // FIXME: On Windows we should use FSCTL_SET_SPARSE and FSCTL_SET_ZERO_DATA
+    // to make this a sparse region, if supported.
+    if (std::error_code EC = sys::fs::resize_file(FD, Capacity))
+      return createFileError(Result.Path, EC);
+  } else {
+    // Someone else initialized it.
+    Capacity = Status.getSize();
+  }
+
+  // Create the mapped region.
+  {
+    std::error_code EC;
+    sys::fs::mapped_file_region Map(
+        File, sys::fs::mapped_file_region::readwrite, Capacity, 0, EC);
+    if (EC)
+      return createFileError(Result.Path, EC);
+    Result.Region = std::move(Map);
+  }
+
+  if (Status.getSize() == 0) {
+    // We are creating a new file; run the constructor.
+    if (Error E = NewFileConstructor(Result))
+      return std::move(E);
+  } else {
+    Result.initializeBumpPtr(BumpPtrOffset);
+  }
+
+  return Result;
+}
+
+void MappedFileRegionBumpPtr::destroyImpl() {
+  if (!FD)
+    return;
+
+  // Drop the shared lock indicating we are no longer accessing the file.
+  if (SharedLockFD)
+    (void)unlockFileThreadSafe(*SharedLockFD);
+
+  // Attempt to truncate the file if we can get exclusive access. Ignore any
+  // errors.
+  if (BumpPtr) {
+    assert(SharedLockFD && "Must have shared lock file open");
+    if (tryLockFileThreadSafe(*SharedLockFD) == std::error_code()) {
+      assert(size() <= capacity());
+      (void)sys::fs::resize_file(*FD, size());
+      (void)unlockFileThreadSafe(*SharedLockFD);
+    }
+  }
+
+  auto Close = [](std::optional<int> &FD) {
+    if (FD) {
+      sys::fs::file_t File = sys::fs::convertFDToNativeFile(*FD);
+      sys::fs::closeFile(File);
+      FD = std::nullopt;
+    }
+  };
+
+  // Close the file and shared lock.
+  Close(FD);
+  Close(SharedLockFD);
+}
+
+void MappedFileRegionBumpPtr::initializeBumpPtr(int64_t BumpPtrOffset) {
+  assert(capacity() < (uint64_t)INT64_MAX && "capacity must fit in int64_t");
+  int64_t BumpPtrEndOffset = BumpPtrOffset + sizeof(decltype(*BumpPtr));
+  assert(BumpPtrEndOffset <= (int64_t)capacity() &&
+         "Expected end offset to be pre-allocated");
+  assert(isAligned(Align::Of<decltype(*BumpPtr)>(), BumpPtrOffset) &&
+         "Expected end offset to be aligned");
+  BumpPtr = reinterpret_cast<decltype(BumpPtr)>(data() + BumpPtrOffset);
+
+  int64_t ExistingValue = 0;
+  if (!BumpPtr->compare_exchange_strong(ExistingValue, BumpPtrEndOffset))
+    assert(ExistingValue >= BumpPtrEndOffset &&
+           "Expected 0, or past the end of the BumpPtr itself");
+}
+
+int64_t MappedFileRegionBumpPtr::allocateOffset(uint64_t AllocSize) {
+  AllocSize = alignTo(AllocSize, getAlign());
+  int64_t OldEnd = BumpPtr->fetch_add(AllocSize);
+  int64_t NewEnd = OldEnd + AllocSize;
+  if (LLVM_UNLIKELY(NewEnd > (int64_t)capacity())) {
+    // Try to return the allocation.
+    (void)BumpPtr->compare_exchange_strong(OldEnd, NewEnd);
+    report_fatal_error(
+        errorCodeToError(std::make_error_code(std::errc::not_enough_memory)));
+  }
+  return OldEnd;
+}