From 1988dbbefc19edd315ae5760ad41bc9a93dcaf49 Mon Sep 17 00:00:00 2001
From: Amos Brocco <amos.brocco@gmail.com>
Date: Fri, 23 Feb 2024 14:23:13 +0100
Subject: [PATCH 1/7] Add __aarch64__ support for CRC32

---
 CMakeLists.txt          |  6 ++++++
 jump/jumpengine.h       |  9 +--------
 memento/mementoengine.h |  8 +-------
 power/powerengine.h     | 10 ++--------
 utils.cpp               | 30 ++++++++++++++++++++++++++++++
 utils.h                 | 23 +++++++++++++++++++++++
 6 files changed, 63 insertions(+), 23 deletions(-)
 create mode 100644 utils.cpp
 create mode 100644 utils.h

diff --git a/CMakeLists.txt b/CMakeLists.txt
index f3d25ff..54f9db7 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -35,6 +35,8 @@ add_executable(speed_test speed_test.cpp
     memento/mashtable.h
     jump/jumpengine.h
     power/powerengine.h
+    utils.h
+    utils.cpp
     )
 
 add_executable(balance balance.cpp
@@ -46,6 +48,8 @@ add_executable(balance balance.cpp
     memento/mashtable.h
     jump/jumpengine.h
     power/powerengine.h
+    utils.h
+    utils.cpp
     )
 
 add_executable(monotonicity monotonicity.cpp
@@ -57,6 +61,8 @@ add_executable(monotonicity monotonicity.cpp
     memento/mashtable.h
     jump/jumpengine.h
     power/powerengine.h
+    utils.h
+    utils.cpp
     )
 
 add_executable(mashtable_test mashtable_test.cpp memento/mashtable.h)
diff --git a/jump/jumpengine.h b/jump/jumpengine.h
index dba6369..7b7fa12 100644
--- a/jump/jumpengine.h
+++ b/jump/jumpengine.h
@@ -16,6 +16,7 @@
 #ifndef JUMPENGINE_H
 #define JUMPENGINE_H
 #include <cstdint>
+#include "../utils.h"
 
 class JumpEngine final {
 public:
@@ -23,14 +24,6 @@ class JumpEngine final {
         : m_num_buckets{working_set}
     {}
 
-    // From AnchorHash
-    static uint32_t crc32c_sse42_u64(uint64_t key, uint64_t seed) {
-        __asm__ volatile("crc32q %[key], %[seed];"
-                         : [seed] "+r"(seed)
-                         : [key] "rm"(key));
-        return seed;
-    }
-
     /**
    * Returns the bucket where the given key should be mapped.
    * This implementations is the same as provided by Jump authors
diff --git a/memento/mementoengine.h b/memento/mementoengine.h
index 8bcbe04..0c99b0b 100644
--- a/memento/mementoengine.h
+++ b/memento/mementoengine.h
@@ -16,6 +16,7 @@
 #ifndef MEMENTOENGINE_H
 #define MEMENTOENGINE_H
 #include "memento.h"
+#include "../utils.h"
 #include <string_view>
 #include <xxhash.h>
 
@@ -195,13 +196,6 @@ class MementoEngine final {
   uint32_t bArraySize() const noexcept { return m_bArraySize; }
 
 private:
-  // From AnchorHash
-  static uint32_t crc32c_sse42_u64(uint64_t key, uint64_t seed) {
-    __asm__ volatile("crc32q %[key], %[seed];"
-                     : [seed] "+r"(seed)
-                     : [key] "rm"(key));
-    return seed;
-  }
 
   // From Jump paper
   static int32_t JumpConsistentHash(uint64_t key, int32_t num_buckets) {
diff --git a/power/powerengine.h b/power/powerengine.h
index 26f067d..e2741f4 100644
--- a/power/powerengine.h
+++ b/power/powerengine.h
@@ -17,6 +17,7 @@
 #define POWERENGINE_H
 #include <cmath>
 #include <cstdint>
+#include "../utils.h"
 #include "pcg_random.hpp"
 
 class PowerEngine final {
@@ -25,14 +26,6 @@ class PowerEngine final {
         : m_n{working_nodes}, m_m{smallestPow2(m_n)}, m_mH{m_m >> 1}, m_mHm1{m_mH - 1}, m_mm1{m_m - 1}
     {}
 
-    // From AnchorHash
-    static uint32_t crc32c_sse42_u64(uint64_t key, uint64_t seed) {
-        __asm__ volatile("crc32q %[key], %[seed];"
-                         : [seed] "+r"(seed)
-                         : [key] "rm"(key));
-        return seed;
-    }
-
     /**
    * Returns the bucket where the given key should be mapped.
    * This implementations is the same as provided by Power authors
@@ -142,6 +135,7 @@ class PowerEngine final {
             //          U denotes the next random number from a generator U (0, 1)
             //          that generates random numbers
             //          uniformly over range (0, 1) and deterministically based on the given key.
+            rng.seed(key);
             auto u = (static_cast<double>(rng())/static_cast<double>(rng.max()));
             // (...) 2. Compute r = min{j: U>(x+1)/(j+1)
             auto r = (uint32_t) ceil((static_cast<uint64_t>(x) + 1) / u) - 1;
diff --git a/utils.cpp b/utils.cpp
new file mode 100644
index 0000000..2c79ede
--- /dev/null
+++ b/utils.cpp
@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) 2023 Amos Brocco.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "utils.h"
+
+uint32_t crc32c_sse42_u64(uint64_t key, uint64_t seed) {
+#ifdef __x86_64
+    __asm__ volatile("crc32q %[key], %[seed];"
+                     : [seed] "+r"(seed)
+                     : [key] "rm"(key));
+#elif __aarch64__
+    __asm__ volatile("crc32x %[key], %[seed];"
+                     : [seed] "+r"(seed)
+                     : [key] "rm"(key));
+#endif
+    return seed;
+}
diff --git a/utils.h b/utils.h
new file mode 100644
index 0000000..21a816b
--- /dev/null
+++ b/utils.h
@@ -0,0 +1,23 @@
+#ifndef UTILS_H
+#define UTILS_H
+/*
+ * Copyright (c) 2023 Amos Brocco.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <stdint.h>
+
+uint32_t crc32c_sse42_u64(uint64_t key, uint64_t seed);
+
+#endif // UTILS_H

From 3b38f6b27a319a30ee3667b064970db17396edaf Mon Sep 17 00:00:00 2001
From: Amos Brocco <amos.brocco@gmail.com>
Date: Fri, 1 Mar 2024 13:42:16 +0100
Subject: [PATCH 2/7] Fix CRC32 for ARM64

---
 anchor/misc/crc32c_sse42_u64.h | 26 +++++++++++++++++++++-----
 utils.cpp                      | 18 ++++++++++++++----
 2 files changed, 35 insertions(+), 9 deletions(-)

diff --git a/anchor/misc/crc32c_sse42_u64.h b/anchor/misc/crc32c_sse42_u64.h
index b5a2fff..a0d8503 100644
--- a/anchor/misc/crc32c_sse42_u64.h
+++ b/anchor/misc/crc32c_sse42_u64.h
@@ -1,10 +1,26 @@
 // Harware based crc calculation
+#include <stdint.h>
 
 static inline uint32_t crc32c_sse42_u64(uint64_t key, uint64_t seed) {
-	__asm__ volatile(
-			"crc32q %[key], %[seed];"
-			: [seed] "+r" (seed)
-			: [key] "rm" (key));
-	return seed;
+#ifdef __x86_64
+    __asm__ volatile("crc32q %[key], %[seed];"
+                     : [seed] "+r"(seed)
+                     : [key] "rm"(key));
+    return seed;
+#elif __aarch64__
+    uint32_t crc = 0xFFFFFFFF; // Inizializza il CRC con il valore iniziale
+
+    // Calcola il CRC utilizzando l'istruzione assembly crc32cx
+    asm volatile(
+        "crc32cx %w0, %w0, %x1" // Calcola il CRC tra il valore corrente e key
+        : "+r" (crc)           // Output: il valore aggiornato del CRC
+        : "r" (key)            // Input: il valore di key
+        );
+
+    // Applica il seed al risultato finale
+    crc ^= seed;
+
+    return crc;
+#endif
 }
 
diff --git a/utils.cpp b/utils.cpp
index 2c79ede..d4d2c5c 100644
--- a/utils.cpp
+++ b/utils.cpp
@@ -21,10 +21,20 @@ uint32_t crc32c_sse42_u64(uint64_t key, uint64_t seed) {
     __asm__ volatile("crc32q %[key], %[seed];"
                      : [seed] "+r"(seed)
                      : [key] "rm"(key));
+    return seed;
 #elif __aarch64__
-    __asm__ volatile("crc32x %[key], %[seed];"
-                     : [seed] "+r"(seed)
-                     : [key] "rm"(key));
+    uint32_t crc = 0xFFFFFFFF; // Inizializza il CRC con il valore iniziale
+
+    // Calcola il CRC utilizzando l'istruzione assembly crc32cx
+    asm volatile(
+        "crc32cx %w0, %w0, %x1" // Calcola il CRC tra il valore corrente e key
+        : "+r" (crc)           // Output: il valore aggiornato del CRC
+        : "r" (key)            // Input: il valore di key
+        );
+
+    // Applica il seed al risultato finale
+    crc ^= seed;
+
+    return crc;
 #endif
-    return seed;
 }

From 5ff93e2eeed4958a4f1fbb45cbd1330f4ac9dd20 Mon Sep 17 00:00:00 2001
From: Amos Brocco <amos.brocco@gmail.com>
Date: Thu, 28 Mar 2024 11:01:36 +0100
Subject: [PATCH 3/7] Add binomialhash

---
 CMakeLists.txt            |   3 +
 balance.cpp               |  22 +++--
 binomial/binomialengine.h | 172 ++++++++++++++++++++++++++++++++++++++
 monotonicity.cpp          |   4 +
 power/powerengine.h       |  10 ++-
 speed_test.cpp            |   7 +-
 6 files changed, 207 insertions(+), 11 deletions(-)
 create mode 100644 binomial/binomialengine.h

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 54f9db7..aa3901a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -35,6 +35,7 @@ add_executable(speed_test speed_test.cpp
     memento/mashtable.h
     jump/jumpengine.h
     power/powerengine.h
+    binomial/binomialengine.h
     utils.h
     utils.cpp
     )
@@ -48,6 +49,7 @@ add_executable(balance balance.cpp
     memento/mashtable.h
     jump/jumpengine.h
     power/powerengine.h
+    binomial/binomialengine.h
     utils.h
     utils.cpp
     )
@@ -61,6 +63,7 @@ add_executable(monotonicity monotonicity.cpp
     memento/mashtable.h
     jump/jumpengine.h
     power/powerengine.h
+    binomial/binomialengine.h
     utils.h
     utils.cpp
     )
diff --git a/balance.cpp b/balance.cpp
index a4ee01f..ec0276e 100644
--- a/balance.cpp
+++ b/balance.cpp
@@ -26,6 +26,7 @@
 #include "memento/mementoengine.h"
 #include "jump/jumpengine.h"
 #include "power/powerengine.h"
+#include "binomial/binomialengine.h"
 #include <fmt/core.h>
 #include <fstream>
 #include <unordered_map>
@@ -84,16 +85,19 @@ int bench(const std::string_view name, const std::string &filename,
 
   // check load balancing
   double mean = (double)num_keys / (working_set - num_removals);
-
+  double min_k = num_keys + 1;
+  double max_k = 0;
+  double var = 0;
   double lb = 0;
   for (uint32_t i = 0; i < anchor_set; i++) {
 
     if (bucket_status[i]) {
-
+        min_k = anchor_ansorbed_keys[i] < min_k ? anchor_ansorbed_keys[i] : min_k;
+        max_k = anchor_ansorbed_keys[i] > max_k ? anchor_ansorbed_keys[i] : max_k;
       if (anchor_ansorbed_keys[i] / mean > lb) {
         lb = anchor_ansorbed_keys[i] / mean;
       }
-
+      var += pow(anchor_ansorbed_keys[i]-mean, 2);
     }
 
     else {
@@ -108,11 +112,13 @@ int bench(const std::string_view name, const std::string &filename,
 #ifdef USE_PCG32
   fmt::println("{}: LB is {}\n", name, lb);
   results_file << name << ": "
-               << "Balance: " << lb << "\tPCG32\n";
+               << "Balance: " << lb << ", Max: " << max_k << ", Min: " << min_k
+               << " Av: " << mean << " Var: " << var/(num_keys -1) << "\tPCG32\n";
 #else
-  fmt::println("{}: LB is {}\n", name, lb);
+  fmt::println("{}: LB is {}, Average: {}, Max: {}, Min: {}, Var: {}\n", name, lb, mean, max_k, min_k, var/(num_keys -1));
   results_file << name << ": "
-               << "Balance: " << lb << "\trand()\n";
+               << "Balance: " << lb << ", Max: " << max_k << ", Min: " << min_k
+               << " Av: " << mean << " Var: " << var/(num_keys -1) << "\trand()\n";
 #endif
 
   ////////////////////////////////////////////////////////////////////
@@ -212,6 +218,10 @@ int main(int argc, char *argv[]) {
       return bench<PowerEngine>("PowerEngine", filename,
                                anchor_set, working_set,
                                num_removals, num_keys);
+  } else if (algorithm == "binomial") {
+      return bench<BinomialEngine>("BinomialEngine", filename,
+                                anchor_set, working_set,
+                                num_removals, num_keys);
   } else {
     fmt::println("Unknown algorithm {}", algorithm);
     return 2;
diff --git a/binomial/binomialengine.h b/binomial/binomialengine.h
new file mode 100644
index 0000000..b54fb20
--- /dev/null
+++ b/binomial/binomialengine.h
@@ -0,0 +1,172 @@
+/*
+ * Copyright (c) 2024 Amos Brocco.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef BINOMIALENGINE_H
+#define BINOMIALENGINE_H
+#include <cstdint>
+#include "../utils.h"
+#include "pcg_random.hpp"
+
+
+class BinomialEngine final {
+public:
+    BinomialEngine(uint32_t, uint32_t working_nodes)
+        : m_size{working_nodes}
+    {
+        // Computes the next power of 2 greater than m_size
+        int highestOneBit = BinomialEngine::highestOneBit(m_size);
+        if(m_size > highestOneBit ) {
+            highestOneBit = highestOneBit << 1;
+        }
+        // If K is the power of 2 greater than m_size
+        // the upper tree filter is K-1
+        m_upperTreeFilter = highestOneBit - 1;
+        // the lower tree filter is K/2
+        m_lowerTreeFilter = m_upperTreeFilter >> 1;
+    }
+
+
+    /**
+   * Returns the bucket where the given key should be mapped.
+   * This implementations is the same as provided by Power authors
+   *
+   * @param key the key to map
+   * @param seed the initial seed for CRC32c
+   * @return the related bucket
+   */
+    uint32_t getBucketCRC32c(uint64_t key, uint64_t seed) noexcept
+    {   
+        if( m_size < 2 )
+            return 0;
+
+        auto hash = crc32c_sse42_u64(key, seed);
+
+        /* If the cluster counts only one node we return such a node. */
+
+        /* We get a position within the upper tree based on the value of the key hash. */
+        uint32_t bucket = (uint32_t) hash & m_upperTreeFilter;
+
+        /* We relocate the bucket randomly inside the same tree level. */
+        bucket = relocateInsideLevel( bucket, hash );
+
+        /* If the final position is valid, we return it. */
+        if( bucket < m_size )
+            return bucket;
+
+        /*
+         * Otherwise, we get a new random position in the upper tree
+         * and return it if in the range [lowerTreeFilter+1,size-1].
+         * We repeat the operation twice (if needed) to get a better balance.
+         */
+        long h = hash;
+        for( uint32_t i = 0; i < 2; ++i )
+        {
+            m_rng.seed(m_upperTreeFilter, h);
+            h = m_rng();
+            bucket = (uint32_t) h & m_upperTreeFilter;
+
+            if( bucket > m_lowerTreeFilter && bucket < m_size )
+                return bucket;
+
+        }
+
+        /*
+         * Finally, if none of the previous operations succeed,
+         * we remap the key in the range covered by the lower tree,
+         * which is guaranteed valid.
+         */
+        bucket = (uint32_t) hash & m_lowerTreeFilter;
+        return relocateInsideLevel( bucket, hash );
+
+    }
+
+    /**
+   * Adds a new bucket to the engine.
+   *
+   * @return the added bucket
+   */
+    uint32_t addBucket() noexcept {
+        uint32_t newBucket = m_size;
+        if( ++m_size > m_upperTreeFilter )
+        {
+            m_upperTreeFilter = (m_upperTreeFilter << 1) | 1;
+            m_lowerTreeFilter = (m_lowerTreeFilter << 1) | 1;
+        }
+        return newBucket;
+    }
+
+    /**
+   * Removes the given bucket from the engine.
+   * Since Power does not support random removals, it will always remove the
+   * last bucket.
+   *
+   * @return the removed bucket
+   */
+    uint32_t removeBucket(uint32_t) noexcept
+    {
+        if( --m_size <= m_lowerTreeFilter )
+        {
+            m_lowerTreeFilter = m_lowerTreeFilter >> 1;
+            m_upperTreeFilter = m_upperTreeFilter >> 1;
+        }
+
+        return m_size;
+
+    }
+
+private:
+
+    uint32_t relocateInsideLevel( uint32_t bucket, uint64_t hash ) noexcept
+    {
+
+        /*
+         * If the bucket is {@code 0} or {@code 1}, we are in the root
+         * of the tree. Therefore, no relocation is needed.
+         */
+        if( bucket < 2 )
+            return bucket;
+
+        uint32_t levelBaseIndex = BinomialEngine::highestOneBit( bucket );
+        uint32_t levelFilter = levelBaseIndex - 1;
+
+
+        m_rng.seed(levelFilter, hash);
+        uint64_t levelHash = m_rng();
+        uint32_t levelIndex = (uint32_t) levelHash & levelFilter;
+
+        return levelBaseIndex + levelIndex;
+    }
+
+    /*
+     * Returns an integer with at most a single one-bit which is in
+     * the position of the highest-order one-bit of the input value.
+     */
+    static unsigned int highestOneBit(unsigned int i) noexcept {
+        i |= (i >> 1);
+        i |= (i >> 2);
+        i |= (i >> 4);
+        i |= (i >> 8);
+        i |= (i >> 16);
+        return i - (i >> 1);
+    }
+
+    pcg32 m_rng;
+    uint32_t m_size;
+    uint32_t m_upperTreeFilter;
+    uint32_t m_lowerTreeFilter;
+
+};
+
+#endif // BINOMIALENGINE_H
diff --git a/monotonicity.cpp b/monotonicity.cpp
index 2b902ae..16534f9 100644
--- a/monotonicity.cpp
+++ b/monotonicity.cpp
@@ -26,6 +26,7 @@
 #include "memento/mashtable.h"
 #include "memento/mementoengine.h"
 #include "power/powerengine.h"
+#include "binomial/binomialengine.h"
 #include <fmt/core.h>
 #include <fstream>
 #include <gtl/phmap.hpp>
@@ -274,6 +275,9 @@ int main(int argc, char *argv[]) {
   } else if (algorithm == "power") {
     return bench<PowerEngine>("PowerEngine", filename, anchor_set, working_set,
                               num_removals, num_keys);
+  } else if (algorithm == "binomial") {
+      return bench<BinomialEngine>("BinomialEngine", filename, anchor_set, working_set,
+                                num_removals, num_keys);
   } else {
     fmt::println("Unknown algorithm {}", algorithm);
     return 2;
diff --git a/power/powerengine.h b/power/powerengine.h
index e2741f4..ec0dcfb 100644
--- a/power/powerengine.h
+++ b/power/powerengine.h
@@ -36,20 +36,19 @@ class PowerEngine final {
    */
     uint32_t getBucketCRC32c(uint64_t key, uint64_t seed) noexcept
     {
-        pcg32 rng;
         auto k = crc32c_sse42_u64(key, seed);
         // r1 = f (key, m) (we pass m-1 because f expects that)
-        auto r1 = f(k, m_mm1, rng);
+        auto r1 = f(k, m_mm1, m_rng);
         if (r1 < m_n) {
             return r1;
         }
         // r2 = g(key, n, m/2 − 1)
-        auto r2 = g(k, m_n, m_mHm1, rng);
+        auto r2 = g(k, m_n, m_mHm1, m_rng);
         if (r2 > m_mHm1) {
             return r2;
         }
         // f (key, m/2) (we pass m/2-1 because f expects that)
-        return f(k, m_mHm1, rng);
+        return f(k, m_mHm1, m_rng);
     }
 
     /**
@@ -150,6 +149,9 @@ class PowerEngine final {
         }
     }
 
+    /* Random number generator */
+    pcg32 m_rng;
+
     /* Number of nodes  in the cluster */
     uint32_t m_n;
 
diff --git a/speed_test.cpp b/speed_test.cpp
index fdd5a39..c2b8a9f 100644
--- a/speed_test.cpp
+++ b/speed_test.cpp
@@ -19,6 +19,7 @@
 #include "memento/mementoengine.h"
 #include "jump/jumpengine.h"
 #include "power/powerengine.h"
+#include "binomial/binomialengine.h"
 #ifdef USE_PCG32
 #include "pcg_random.hpp"
 #include <random>
@@ -292,7 +293,11 @@ int main(int argc, char *argv[]) {
       return bench<PowerEngine>("PowerEngine", filename,
                                anchor_set, working_set,
                                num_removals, num_keys);
-  } else {
+  } else if (algorithm == "binomial") {
+      return bench<BinomialEngine>("BinomialEngine", filename,
+                                anchor_set, working_set,
+                                num_removals, num_keys);
+  }else {
     fmt::println("Unknown algorithm {}", algorithm);
     return 2;
   }

From 3a2f1b900d2b38aff14a918e40d6278d4df15b6f Mon Sep 17 00:00:00 2001
From: Amos Brocco <amos.brocco@gmail.com>
Date: Thu, 28 Mar 2024 11:17:26 +0100
Subject: [PATCH 4/7] Fix comment

---
 binomial/binomialengine.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/binomial/binomialengine.h b/binomial/binomialengine.h
index b54fb20..424881e 100644
--- a/binomial/binomialengine.h
+++ b/binomial/binomialengine.h
@@ -31,9 +31,9 @@ class BinomialEngine final {
             highestOneBit = highestOneBit << 1;
         }
         // If K is the power of 2 greater than m_size
-        // the upper tree filter is K-1
+        // the upper tree filter is L=(K-1)
         m_upperTreeFilter = highestOneBit - 1;
-        // the lower tree filter is K/2
+        // the lower tree filter is L/2
         m_lowerTreeFilter = m_upperTreeFilter >> 1;
     }
 

From 425781630634d5fc81cc938e45ef09977cf3e61b Mon Sep 17 00:00:00 2001
From: Amos Brocco <amos.brocco@gmail.com>
Date: Thu, 18 Apr 2024 09:57:36 +0200
Subject: [PATCH 5/7] Fix variance

---
 balance.cpp | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/balance.cpp b/balance.cpp
index ec0276e..1bd76b1 100644
--- a/balance.cpp
+++ b/balance.cpp
@@ -97,6 +97,9 @@ int bench(const std::string_view name, const std::string &filename,
       if (anchor_ansorbed_keys[i] / mean > lb) {
         lb = anchor_ansorbed_keys[i] / mean;
       }
+      if (working_set <= 128) {
+          std::cout << i << ":" << anchor_ansorbed_keys[i] << " (" << (((static_cast<double>(anchor_ansorbed_keys[i]) - mean) / mean)*100.0) << "%)" << std::endl;
+      }
       var += pow(anchor_ansorbed_keys[i]-mean, 2);
     }
 
@@ -115,7 +118,7 @@ int bench(const std::string_view name, const std::string &filename,
                << "Balance: " << lb << ", Max: " << max_k << ", Min: " << min_k
                << " Av: " << mean << " Var: " << var/(num_keys -1) << "\tPCG32\n";
 #else
-  fmt::println("{}: LB is {}, Average: {}, Max: {}, Min: {}, Var: {}\n", name, lb, mean, max_k, min_k, var/(num_keys -1));
+  fmt::println("{}: LB is {}, Average: {}, Max: {}, Min: {}, Var: {}\n", name, lb, mean, max_k, min_k, var/working_set);
   results_file << name << ": "
                << "Balance: " << lb << ", Max: " << max_k << ", Min: " << min_k
                << " Av: " << mean << " Var: " << var/(num_keys -1) << "\trand()\n";

From b1089129dccb2f40443749600d2eef35703cfde9 Mon Sep 17 00:00:00 2001
From: Amos Brocco <amos.brocco@gmail.com>
Date: Thu, 18 Apr 2024 12:55:45 +0200
Subject: [PATCH 6/7] Add sd

---
 balance.cpp | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/balance.cpp b/balance.cpp
index 1bd76b1..a55aa24 100644
--- a/balance.cpp
+++ b/balance.cpp
@@ -116,12 +116,13 @@ int bench(const std::string_view name, const std::string &filename,
   fmt::println("{}: LB is {}\n", name, lb);
   results_file << name << ": "
                << "Balance: " << lb << ", Max: " << max_k << ", Min: " << min_k
-               << " Av: " << mean << " Var: " << var/(num_keys -1) << "\tPCG32\n";
+               << " Av: " << mean << " Var: " << var/working_set << " sd: " << std::sqrt(var/working_set) << "\tPCG32\n";
 #else
-  fmt::println("{}: LB is {}, Average: {}, Max: {}, Min: {}, Var: {}\n", name, lb, mean, max_k, min_k, var/working_set);
+  fmt::println("{}: LB is {}, Average: {}, Max: {}, Min: {}, Var: {} sd: {}\n",
+               name, lb, mean, max_k, min_k, var/working_set, std::sqrt(var/working_set));
   results_file << name << ": "
                << "Balance: " << lb << ", Max: " << max_k << ", Min: " << min_k
-               << " Av: " << mean << " Var: " << var/(num_keys -1) << "\trand()\n";
+               << " Av: " << mean << " Var: " << var/working_set << " sd: " << std::sqrt(var/working_set) << "\trand()\n";
 #endif
 
   ////////////////////////////////////////////////////////////////////

From 0de96ea3e631dd8b936603372332055d82739b37 Mon Sep 17 00:00:00 2001
From: Amos Brocco <amos.brocco@gmail.com>
Date: Thu, 25 Apr 2024 15:58:49 +0200
Subject: [PATCH 7/7] Dynamic iter number for Binomial

---
 CMakeLists.txt            |  3 +++
 binomial/binomialengine.h | 41 +++++++++++++++++++++++++++------------
 power/powerengine.h       | 24 ++++++++++++++++++++++-
 3 files changed, 55 insertions(+), 13 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index aa3901a..1c322bb 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -36,6 +36,7 @@ add_executable(speed_test speed_test.cpp
     jump/jumpengine.h
     power/powerengine.h
     binomial/binomialengine.h
+    flip/flipengine.h
     utils.h
     utils.cpp
     )
@@ -50,6 +51,7 @@ add_executable(balance balance.cpp
     jump/jumpengine.h
     power/powerengine.h
     binomial/binomialengine.h
+    flip/flipengine.h
     utils.h
     utils.cpp
     )
@@ -64,6 +66,7 @@ add_executable(monotonicity monotonicity.cpp
     jump/jumpengine.h
     power/powerengine.h
     binomial/binomialengine.h
+    flip/flipengine.h
     utils.h
     utils.cpp
     )
diff --git a/binomial/binomialengine.h b/binomial/binomialengine.h
index 424881e..88467fa 100644
--- a/binomial/binomialengine.h
+++ b/binomial/binomialengine.h
@@ -16,6 +16,8 @@
 #ifndef BINOMIALENGINE_H
 #define BINOMIALENGINE_H
 #include <cstdint>
+#include <cstdlib>
+#include <string>
 #include "../utils.h"
 #include "pcg_random.hpp"
 
@@ -30,11 +32,23 @@ class BinomialEngine final {
         if(m_size > highestOneBit ) {
             highestOneBit = highestOneBit << 1;
         }
-        // If K is the power of 2 greater than m_size
-        // the upper tree filter is L=(K-1)
         m_upperTreeFilter = highestOneBit - 1;
-        // the lower tree filter is L/2
         m_lowerTreeFilter = m_upperTreeFilter >> 1;
+
+        // Read BINOMIAL_HASH_STEP2_ITERATIONS
+        auto val{getenv("BINOMIAL_HASH_STEP2_ITERATIONS")};
+        if (val) {
+            try {
+                auto iters{std::stoi(val)};
+                m_iter = iters;
+            } catch(...) {
+                std::cerr << "Invalid BINOMIAL_HASH_STEP2_ITERATIONS: " << val << std::endl;
+            }
+            std::cerr << "Using BINOMIAL_HASH_STEP2_ITERATIONS=" << m_iter << std::endl;
+        } else {
+            m_iter = m_size <= m_lowerTreeFilter + (m_lowerTreeFilter >> 1) + 1 ? 2 : 1;
+            std::cerr << "Using default BINOMIAL_HASH_STEP2_ITERATIONS=" << m_iter << std::endl;
+        }
     }
 
 
@@ -47,14 +61,13 @@ class BinomialEngine final {
    * @return the related bucket
    */
     uint32_t getBucketCRC32c(uint64_t key, uint64_t seed) noexcept
-    {   
+    {
+        /* If the cluster counts only one node we return such a node. */
         if( m_size < 2 )
             return 0;
 
         auto hash = crc32c_sse42_u64(key, seed);
 
-        /* If the cluster counts only one node we return such a node. */
-
         /* We get a position within the upper tree based on the value of the key hash. */
         uint32_t bucket = (uint32_t) hash & m_upperTreeFilter;
 
@@ -66,12 +79,12 @@ class BinomialEngine final {
             return bucket;
 
         /*
-         * Otherwise, we get a new random position in the upper tree
+         * Otherwise, we try to get a new random position in the upper tree
          * and return it if in the range [lowerTreeFilter+1,size-1].
-         * We repeat the operation twice (if needed) to get a better balance.
          */
+
         long h = hash;
-        for( uint32_t i = 0; i < 2; ++i )
+        for( uint32_t i = 0; i < m_iter; ++i )
         {
             m_rng.seed(m_upperTreeFilter, h);
             h = m_rng();
@@ -104,6 +117,7 @@ class BinomialEngine final {
             m_upperTreeFilter = (m_upperTreeFilter << 1) | 1;
             m_lowerTreeFilter = (m_lowerTreeFilter << 1) | 1;
         }
+        m_iter = m_size <= m_lowerTreeFilter + (m_lowerTreeFilter >> 1) + 1 ? 2 : 1;
         return newBucket;
     }
 
@@ -121,7 +135,7 @@ class BinomialEngine final {
             m_lowerTreeFilter = m_lowerTreeFilter >> 1;
             m_upperTreeFilter = m_upperTreeFilter >> 1;
         }
-
+        m_iter = m_size <= m_lowerTreeFilter + (m_lowerTreeFilter >> 1) + 1 ? 2 : 1;
         return m_size;
 
     }
@@ -138,10 +152,13 @@ class BinomialEngine final {
         if( bucket < 2 )
             return bucket;
 
+        /* First bucket at that level */
         uint32_t levelBaseIndex = BinomialEngine::highestOneBit( bucket );
-        uint32_t levelFilter = levelBaseIndex - 1;
 
+        /* Number of buckets at that level minus 1 */
+        uint32_t levelFilter = levelBaseIndex - 1;
 
+        /* Compute a random bucket at that level */
         m_rng.seed(levelFilter, hash);
         uint64_t levelHash = m_rng();
         uint32_t levelIndex = (uint32_t) levelHash & levelFilter;
@@ -166,7 +183,7 @@ class BinomialEngine final {
     uint32_t m_size;
     uint32_t m_upperTreeFilter;
     uint32_t m_lowerTreeFilter;
-
+    uint32_t m_iter{1};
 };
 
 #endif // BINOMIALENGINE_H
diff --git a/power/powerengine.h b/power/powerengine.h
index ec0dcfb..997a2ba 100644
--- a/power/powerengine.h
+++ b/power/powerengine.h
@@ -110,6 +110,12 @@ class PowerEngine final {
         auto j = (sizeof(kBits)<<3) - __builtin_clz(kBits) - 1;
         // (...) computes 2^j
         auto h = static_cast<uint32_t>(1) << j;
+
+#ifndef POWER_BASE_LOGIC
+        auto h2 =  crc32c_sse42_u64(key, h);
+        auto h3 = (uint32_t) h2 & (h-1);
+        return h+h3;
+#else
         // [must return] a pseudo-random integer deterministacally based on
         // the values of key and j
         rng.seed(key, j);
@@ -117,6 +123,7 @@ class PowerEngine final {
         // probability
         auto r = h + (rng() & (h - 1));
         return r;
+#endif
     }
 
     /**
@@ -129,12 +136,26 @@ class PowerEngine final {
      */
     static uint32_t g(uint32_t key, uint32_t n, uint32_t s, pcg32& rng) {
         auto x = s; // (...) Initially, x is set to the value of s
+        rng.seed(key);
+
+#ifndef POWER_BASE_LOGIC
+        auto candidate{s};
+        uint32_t next{0};
+        for(;;) {
+            auto u = (static_cast<double>(rng())/static_cast<double>(rng.max()));
+            next = (uint32_t) ((candidate + 1) / u);
+            if (next >= 0 && next < n) {
+                candidate = next;
+            } else {
+                return candidate;
+            }
+        }
+#else
         for (;;) {
             // (...) 1. Generate U
             //          U denotes the next random number from a generator U (0, 1)
             //          that generates random numbers
             //          uniformly over range (0, 1) and deterministically based on the given key.
-            rng.seed(key);
             auto u = (static_cast<double>(rng())/static_cast<double>(rng.max()));
             // (...) 2. Compute r = min{j: U>(x+1)/(j+1)
             auto r = (uint32_t) ceil((static_cast<uint64_t>(x) + 1) / u) - 1;
@@ -147,6 +168,7 @@ class PowerEngine final {
                 return x;
             }
         }
+#endif
     }
 
     /* Random number generator */