Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit b4884cb

Browse files
zeeboStorj Robot
zeebo
authored and
Storj Robot
committed
storagenode/hashstore: reduce open file handles
this makes it so that when a log file is over the max log size, we close it. for reads, we always open a new file handle through a small lru of default size 10. this should reduce memory usage on windows which uses a lot of memory per file handle for some reason, while still having good performance everywhere. Change-Id: I746b82cc7ef9bfa84ca5219b043d683b196e0c1f
1 parent d808055 commit b4884cb

File tree

15 files changed

+684
-192
lines changed

15 files changed

+684
-192
lines changed

‎cmd/write-hashtbl/main.go‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ func (c *cmdRoot) Execute(ctx context.Context) (err error) {
102102
if err != nil {
103103
return errs.Wrap(err)
104104
}
105-
defer tcons.Close()
105+
defer tcons.Cancel()
106106

107107
for _, file := range files {
108108
_, _ = fmt.Fprintf(clingy.Stdout(ctx), "Processing %s...\n", file)
@@ -124,9 +124,7 @@ func (c *cmdRoot) Execute(ctx context.Context) (err error) {
124124
if err != nil {
125125
return err
126126
}
127-
tbl.Close()
128-
129-
return nil
127+
return tbl.Close()
130128
}
131129

132130
func isLogFile(path string) (uint64, bool) {

‎storagenode/hashstore/config.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ type CompactionCfg struct {
7070
type StoreCfg struct {
7171
FlushSemaphore int `help:"controls the number of concurrent flushes to log files" default:"0" hidden:"true"`
7272
SyncWrites bool `help:"if set, writes to the log file and table are fsync'd to disk" default:"false"`
73+
OpenFileCache int `help:"number of open file handles to cache for reads" default:"10"`
7374
}
7475

7576
// MmapCfg is the configuration for mmap usage.
@@ -107,6 +108,7 @@ func CreateDefaultConfig(kind TableKind, mmap bool) Config {
107108
Store: StoreCfg{
108109
FlushSemaphore: 0,
109110
SyncWrites: false,
111+
OpenFileCache: 10,
110112
},
111113
Compaction: CompactionCfg{
112114
MaxLogSize: 1073741824,

‎storagenode/hashstore/crash_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,13 +46,13 @@ func runCrashServer(ctx context.Context) error {
4646
if err != nil {
4747
return errs.Wrap(err)
4848
}
49-
defer db.Close()
49+
defer func() { _ = db.Close() }()
5050

5151
// launch a goroutine to try to read from stdin so that if the parent
5252
// process dies, we close the db and eventually exit.
5353
go func() {
5454
_, _ = os.Stdin.Read(make([]byte, 1))
55-
db.Close()
55+
_ = db.Close()
5656
}()
5757

5858
mu := new(sync.Mutex) // protects writing to stdout
@@ -187,7 +187,7 @@ func TestCorrectDuringCrashes(t *testing.T) {
187187
// open the database and ensure every key that was printed is readable.
188188
db, err := New(ctx, CreateDefaultConfig(TableKind_HashTbl, false), dir, "", nil, nil, nil)
189189
assert.NoError(t, err)
190-
defer db.Close()
190+
defer assertClose(t, db)
191191

192192
for _, key := range keys {
193193
r, err := db.Read(ctx, key)

‎storagenode/hashstore/db.go‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,10 @@ type DB struct {
4646
lastRestore func(context.Context) time.Time
4747

4848
closed drpcsignal.Signal // closed state
49+
cloErr error // close error
4950
cloMu sync.Mutex // synchronizes closing
50-
wg sync.WaitGroup // waitgroup for background goroutines
51+
52+
wg sync.WaitGroup // waitgroup for background goroutines
5153

5254
mu sync.Mutex // protects the following fields
5355
compact *compactState // set if compaction is in progress
@@ -86,7 +88,7 @@ func New(
8688
}
8789
defer func() {
8890
if err != nil {
89-
d.Close()
91+
_ = d.Close()
9092
}
9193
}()
9294

@@ -212,12 +214,12 @@ func (d *DB) Stats() (DBStats, StoreStats, StoreStats) {
212214
}
213215

214216
// Close closes down the database and blocks until all background processes have stopped.
215-
func (d *DB) Close() {
217+
func (d *DB) Close() error {
216218
d.cloMu.Lock()
217219
defer d.cloMu.Unlock()
218220

219221
if !d.closed.Set(Error.New("db closed")) {
220-
return
222+
return d.cloErr
221223
}
222224

223225
d.mu.Lock()
@@ -231,15 +233,19 @@ func (d *DB) Close() {
231233
}
232234

233235
// close down the stores now that compaction is finished.
236+
var eg errs.Group
234237
if d.active != nil {
235-
d.active.Close()
238+
eg.Add(d.active.Close())
236239
}
237240
if d.passive != nil {
238-
d.passive.Close()
241+
eg.Add(d.passive.Close())
239242
}
240243

241244
// wait for any background goroutines to finish.
242245
d.wg.Wait()
246+
247+
d.cloErr = eg.Err()
248+
return d.cloErr
243249
}
244250

245251
// Create adds an entry to the database with the given key and expiration time. Close or Cancel

‎storagenode/hashstore/db_test.go‎

Lines changed: 57 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"errors"
99
"io"
1010
"io/fs"
11+
"runtime"
1112
"sync/atomic"
1213
"testing"
1314
"time"
@@ -61,14 +62,67 @@ func testDB_BasicOperation(t *testing.T, cfg Config) {
6162
assert.Error(t, err)
6263
assert.That(t, errors.Is(err, fs.ErrNotExist))
6364

64-
// create and read should fail after close.
65+
// create, read and compact should fail after close.
6566
db.Close()
6667

6768
_, err = db.Read(ctx, newKey())
6869
assert.Error(t, err)
6970

7071
_, err = db.Create(ctx, newKey(), time.Time{})
7172
assert.Error(t, err)
73+
74+
err = db.Compact(ctx)
75+
assert.Error(t, err)
76+
}
77+
78+
func TestDB_ConcurrentOperation(t *testing.T) {
79+
forAllTables(t, testDB_ConcurrentOperation)
80+
}
81+
func testDB_ConcurrentOperation(t *testing.T, cfg Config) {
82+
cfg.Compaction.MaxLogSize = 1 << 10 // 1KiB
83+
84+
db := newTestDB(t, cfg, nil, nil)
85+
defer db.Close()
86+
87+
procs := runtime.GOMAXPROCS(-1)
88+
keysCh := make(chan []Key, procs)
89+
for range procs {
90+
go func() {
91+
var keys []Key
92+
for i := 0; ; i++ {
93+
// stop when we have ~100 log files and, if not short, compaction has happened
94+
stats, _, _ := db.Stats()
95+
if stats.NumLogs >= 100 && (testing.Short() || stats.Compactions > 0) {
96+
break
97+
}
98+
99+
keys = append(keys, db.AssertCreate())
100+
101+
// add about 10% concurrent reads
102+
if mwc.Intn(10) == 0 {
103+
db.AssertRead(keys[mwc.Intn(len(keys))])
104+
}
105+
}
106+
keysCh <- keys
107+
}()
108+
}
109+
110+
// collect all the keys created by the goroutines.
111+
var allKeys []Key
112+
for range procs {
113+
allKeys = append(allKeys, <-keysCh...)
114+
}
115+
116+
// ensure we can read all the keys back.
117+
for _, key := range allKeys {
118+
db.AssertRead(key)
119+
}
120+
121+
// ensure we can still read all the keys back after reopen.
122+
db.AssertReopen()
123+
for _, key := range allKeys {
124+
db.AssertRead(key)
125+
}
72126
}
73127

74128
func TestDB_TrashStats(t *testing.T) {
@@ -420,7 +474,7 @@ func benchmarkDB(b *testing.B, cfg Config) {
420474

421475
db, err := New(ctx, cfg, b.TempDir(), "", nil, nil, nil)
422476
assert.NoError(b, err)
423-
defer db.Close()
477+
defer assertClose(b, db)
424478

425479
b.SetBytes(int64(size))
426480
b.ReportAllocs()
@@ -446,7 +500,7 @@ func benchmarkDB(b *testing.B, cfg Config) {
446500

447501
db, err := New(ctx, cfg, b.TempDir(), "", nil, nil, nil)
448502
assert.NoError(b, err)
449-
defer db.Close()
503+
defer assertClose(b, db)
450504

451505
// write at most ~100MB of keys or 1000 keys, whichever is smaller. this keeps the benchmark
452506
// time reasonable.

‎storagenode/hashstore/hashtbl.go‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ type HashTbl struct {
3131
opMu rwMutex // protects operations
3232

3333
closed drpcsignal.Signal // closed state
34+
cloErr error // close error
3435
cloMu sync.Mutex // synchronizes closing
3536

3637
buffer *rwBigPageCache // buffer for inserts
@@ -148,6 +149,12 @@ func OpenHashTbl(ctx context.Context, fh *os.File, cfg MmapCfg) (_ *HashTbl, err
148149
slotMask: 1<<logSlots - 1,
149150
header: header,
150151
}
152+
defer func() {
153+
if err != nil {
154+
h.fh = nil // don't close the file handle if we're returning an error
155+
_ = h.Close()
156+
}
157+
}()
151158

152159
if cfg.Mmap && platform.MmapSupported {
153160
data, err := platform.Mmap(fh, int(size))
@@ -210,12 +217,12 @@ func (h *HashTbl) Header() TblHeader { return h.header }
210217
func (h *HashTbl) Handle() *os.File { return h.fh }
211218

212219
// Close closes the hash table and returns when no more operations are running.
213-
func (h *HashTbl) Close() {
220+
func (h *HashTbl) Close() error {
214221
h.cloMu.Lock()
215222
defer h.cloMu.Unlock()
216223

217224
if !h.closed.Set(Error.New("hashtbl closed")) {
218-
return
225+
return h.cloErr
219226
}
220227

221228
// grab the lock to ensure all operations have finished before closing the file handle.
@@ -227,7 +234,11 @@ func (h *HashTbl) Close() {
227234
h.mmap = nil
228235
}
229236

230-
_ = h.fh.Close()
237+
if h.fh != nil {
238+
h.cloErr = h.fh.Close()
239+
}
240+
241+
return h.cloErr
231242
}
232243

233244
// slotForKey computes the slot for the given key.
@@ -568,10 +579,10 @@ func (c *HashTblConstructor) valid() error {
568579
return nil
569580
}
570581

571-
// Close signals that we're done with the HashTblConstructor. It should always be called.
572-
func (c *HashTblConstructor) Close() {
582+
// Cancel signals that we're done with the HashTblConstructor. It should always be called.
583+
func (c *HashTblConstructor) Cancel() {
573584
if c.h != nil {
574-
c.h.Close()
585+
_ = c.h.Close()
575586
c.h = nil
576587
}
577588
}

0 commit comments

Comments
 (0)