Fixed bug in dense bitset which causes it to use much much more memory than needed.

ylow · ylow · commit 07628dfd6706 · 2012-03-12T15:05:31.000-04:00
Added a test
diff --git a/src/graphlab/util/dense_bitset.hpp b/src/graphlab/util/dense_bitset.hpp
@@ -76,7 +76,7 @@ namespace graphlab {
     inline void resize(size_t n) {
       len = n;
       //need len bits
-      arrlen = next_powerof2(n) / sizeof(size_t) + 1;
+      arrlen = (n / (sizeof(size_t) * 8)) + (n % (sizeof(size_t) * 8) > 0);
       array = (size_t*)realloc(array, sizeof(size_t) * arrlen);
     }
   
@@ -296,20 +296,6 @@ namespace graphlab {
 
   private:
    
-    inline size_t next_powerof2(size_t val) {
-      --val;
-      val = val | (val >> 1);
-      val = val | (val >> 2);
-      val = val | (val >> 4);
-      val = val | (val >> 8);
-      val = val | (val >> 16);
-#ifdef _LP64
-      val = val | (val >> 32);
-#endif
-      return val + 1; 
-    }
-  
- 
     inline static void bit_to_pos(uint32_t b, uint32_t& arrpos, uint32_t& bitpos) {
       // the compiler better optimize this...
       arrpos = b / (8 * sizeof(size_t));
@@ -551,20 +537,21 @@ namespace graphlab {
     
     /// Serializes this bitset to an archive
     inline void save(oarchive& oarc) const {
-      oarc <<len << arrlen;
-      if (arrlen > 0) serialize(oarc, array, arrlen*sizeof(size_t));
+      //oarc <<len << arrlen;
+      //if (arrlen > 0)
+      serialize(oarc, array, arrlen*sizeof(size_t));
     }
 
     /// Deserializes this bitset from an archive
     inline void load(iarchive& iarc) {
-      size_t l;
+      /*size_t l;
       size_t arl;
       iarc >> l >> arl;
       ASSERT_EQ(l, len);
-      ASSERT_EQ(arl, arrlen);
-      if (arrlen > 0) {
-        deserialize(iarc, array, arrlen*sizeof(size_t));
-      }
+      ASSERT_EQ(arl, arrlen);*/
+      //if (arrlen > 0) {
+      deserialize(iarc, array, arrlen*sizeof(size_t));
+      //}
     }
 
     size_t popcount() const {
@@ -577,21 +564,6 @@ namespace graphlab {
     }
 
   private:
-   
-    inline static size_t next_powerof2(size_t val) {
-      --val;
-      val = val | (val >> 1);
-      val = val | (val >> 2);
-      val = val | (val >> 4);
-      val = val | (val >> 8);
-      val = val | (val >> 16);
-#ifdef _LP64
-      val = val | (val >> 32);
-#endif
-      return val + 1; 
-    }
-  
- 
     inline static void bit_to_pos(uint32_t b, uint32_t &arrpos, uint32_t &bitpos) {
       // the compiler better optimize this...
       arrpos = b / (8 * sizeof(size_t));
@@ -616,11 +588,11 @@ namespace graphlab {
     }
 
     static const size_t arrlen;
-    size_t array[len / sizeof(size_t) + 1];
+    size_t array[len / (sizeof(size_t) * 8) + (len % (sizeof(size_t) * 8) > 0)];
   };
 
   template<int len>
-  const size_t fixed_dense_bitset<len>::arrlen = len / sizeof(size_t) + 1;
+  const size_t fixed_dense_bitset<len>::arrlen = len / (sizeof(size_t) * 8) + (len % (sizeof(size_t) * 8) > 0);
 }
 #endif
 
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
@@ -12,7 +12,7 @@ ADD_CXXTEST(factor_test.cxx)
 ADD_CXXTEST(small_map_test.cxx)
 ADD_CXXTEST(small_set_test.cxx)
 ADD_CXXTEST(fast_multinomial_test.cxx)
-
+ADD_CXXTEST(dense_bitset_test.cxx)
 
 # ADD_CXXTEST(graphlab_test.cxx)
 ADD_CXXTEST(serializetests.cxx)
diff --git a/tests/dense_bitset_test.cxx b/tests/dense_bitset_test.cxx
@@ -8,6 +8,7 @@ class DenseBitsetTestSuite : public CxxTest::TestSuite {
   void test_densebitset(void) {
     dense_bitset d;
     d.resize(100);
+    d.clear();
     uint32_t probelocations[7] = {0, 10, 12, 50, 66, 81, 99};
     // test setting
     for (size_t i= 0;i < 7; ++i) {
@@ -34,7 +35,21 @@ class DenseBitsetTestSuite : public CxxTest::TestSuite {
       TS_ASSERT_EQUALS(iter, probelocations[ctr]);
       ++ctr;
     }
+    
+    std::stringstream strm;
+    graphlab::oarchive oarc(strm);
+    oarc << d;
+    strm.flush();
+    graphlab::iarchive iarc(strm);
+    dense_bitset d2;
+    iarc >> d2;
+
 
+    for (size_t i = 0;i< 100; ++i) {
+      bool inprobe=false;
+      for (size_t j = 0;j <7; ++j) inprobe |= (probelocations[j] == i);
+      TS_ASSERT_EQUALS(d2.get(i), inprobe);
+    }
     // testclearing
     for (size_t i= 0;i < 7; ++i) {
       d.clear_bit(probelocations[i]);
@@ -75,13 +90,31 @@ class DenseBitsetTestSuite : public CxxTest::TestSuite {
       ++ctr;
     }
 
+    std::stringstream strm;
+    graphlab::oarchive oarc(strm);
+    oarc << d;
+    strm.flush();
+    graphlab::iarchive iarc(strm);
+    fixed_dense_bitset<100> d2;
+    iarc >> d2;
+
+
+    for (size_t i = 0;i< 100; ++i) {
+      bool inprobe=false;
+      for (size_t j = 0;j <7; ++j) inprobe |= (probelocations[j] == i);
+      TS_ASSERT_EQUALS(d2.get(i), inprobe);
+    }
+    
+    
     // testclearing
     for (size_t i= 0;i < 7; ++i) {
       d.clear_bit(probelocations[i]);
     }
     for (size_t i = 0;i< 100; ++i) {
       TS_ASSERT_EQUALS(d.get(i), false);
     }
+
+
   }
 
 
