Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 04bd4cf

Browse files
wj596wj596
committed
v1.1.0-dev
1 parent 7811b73 commit 04bd4cf

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

88 files changed

+4179
-2769
lines changed

‎config/app_config.go

Lines changed: 33 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -19,13 +19,11 @@
1919
package config
2020

2121
import (
22+
"github.com/juju/errors"
23+
"gopkg.in/yaml.v2"
2224
"io/ioutil"
2325
"log"
2426
"path/filepath"
25-
"runtime"
26-
27-
"github.com/juju/errors"
28-
"gopkg.in/yaml.v2"
2927

3028
"go-mysql-transfer/util/fileutils"
3129
"go-mysql-transfer/util/sysutils"
@@ -38,16 +36,22 @@ const (
3836
_prometheusExporterPort = 9595
3937
_webPort = 8060
4038
_rpcPort = 7060
39+
40+
_batchBulkSize = 100
41+
_batchCoroutines = 3
42+
_streamBulkSize = 100
43+
_streamFlushInterval = 200
4144
)
4245

4346
var _instance *AppConfig
4447

4548
// AppConfig 应用配置
4649
type AppConfig struct {
47-
DataDir string `yaml:"data_dir"`
48-
Maxprocs int `yaml:"maxprocs"` // 最大协程数，默认CPU核心数*2
49-
BulkSize int64 `yaml:"bulk_size"`
50-
FlushBulkInterval int `yaml:"flush_bulk_interval"`
50+
DataDir string `yaml:"data_dir"`
51+
BatchCoroutines int `yaml:"batch_coroutines"` // 批处理最大协程数，默认3
52+
BatchBulkSize int `yaml:"batch_bulk_size"` // 批处理每批提交条数，默认100
53+
StreamBulkSize int `yaml:"stream_bulk_size"` // 实时数据每批提交条数，默认100
54+
StreamFlushInterval int `yaml:"flush_bulk_interval"` // 实时数据刷新周期，单位毫秒，默认200毫秒
5155

5256
EnablePrometheusExporter bool `yaml:"enable_prometheus_exporter"` // 启用prometheus exporter，默认false
5357
PrometheusExporterPort int `yaml:"prometheus_exporter_addr"` // prometheus exporter端口
@@ -96,8 +100,17 @@ func checkConfig(c *AppConfig) error {
96100
if c.RpcPort == 0 {
97101
c.RpcPort = _rpcPort
98102
}
99-
if c.Maxprocs <= 0 {
100-
c.Maxprocs = runtime.NumCPU() * 2
103+
if c.BatchCoroutines <= 0 {
104+
c.BatchCoroutines = _batchCoroutines
105+
}
106+
if c.BatchBulkSize <= 0 {
107+
c.BatchBulkSize = _batchBulkSize
108+
}
109+
if c.StreamBulkSize <= 0 {
110+
c.StreamBulkSize = _streamBulkSize
111+
}
112+
if c.StreamFlushInterval <= 0 {
113+
c.StreamFlushInterval = _streamFlushInterval
101114
}
102115

103116
// Logger
@@ -155,16 +168,20 @@ func (c *AppConfig) GetDataDir() string {
155168
return c.DataDir
156169
}
157170

158-
func (c *AppConfig) GetMaxprocs() int {
159-
return c.Maxprocs
171+
func (c *AppConfig) GetBatchCoroutines() int {
172+
return c.BatchCoroutines
173+
}
174+
175+
func (c *AppConfig) GetBatchBulkSize() int {
176+
return c.BatchBulkSize
160177
}
161178

162-
func (c *AppConfig) GetBulkSize() int64 {
163-
return c.BulkSize
179+
func (c *AppConfig) GetStreamBulkSize() int {
180+
return c.StreamBulkSize
164181
}
165182

166-
func (c *AppConfig) GetFlushBulkInterval() int {
167-
return c.FlushBulkInterval
183+
func (c *AppConfig) GetStreamFlushInterval() int {
184+
return c.StreamFlushInterval
168185
}
169186

170187
func (c *AppConfig) IsEnablePrometheusExporter() bool {

‎dao/bolt/test_before.go renamed to ‎dao/before_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package bolt
1+
package dao
22

33
import (
44
"testing"

‎dao/bolt.go

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
package dao
2+
3+
import (
4+
"encoding/binary"
5+
"fmt"
6+
"path/filepath"
7+
8+
"github.com/juju/errors"
9+
"go.etcd.io/bbolt"
10+
11+
"go-mysql-transfer/config"
12+
"go-mysql-transfer/util/fileutils"
13+
)
14+
15+
const (
16+
_storePath = "db"
17+
_fileMode = 0600
18+
_metadataFileName = "metadata.db"
19+
//_commitFileName = "commit.db"
20+
)
21+
22+
var (
23+
_mdb *bbolt.DB
24+
_cdb *bbolt.DB
25+
26+
_sourceBucket = []byte("source")
27+
_endpointBucket = []byte("endpoint")
28+
_pipelineBucket = []byte("pipeline")
29+
_ruleBucket = []byte("rule")
30+
31+
_positionBucket = []byte("position")
32+
_streamStateBucket = []byte("streamState")
33+
34+
//_commitBucket = []byte("commit")
35+
)
36+
37+
func initBolt(config *config.AppConfig) error {
38+
storePath := filepath.Join(config.GetDataDir(), _storePath)
39+
if err := fileutils.MkdirIfNecessary(storePath); err != nil {
40+
return errors.New(fmt.Sprintf("create metadataFilePath : %s", err.Error()))
41+
}
42+
43+
var err error
44+
45+
metadataFilePath := filepath.Join(storePath, _metadataFileName)
46+
_mdb, err = bbolt.Open(metadataFilePath, _fileMode, bbolt.DefaultOptions)
47+
if err != nil {
48+
return errors.New(fmt.Sprintf("open boltdb: %s", err.Error()))
49+
}
50+
51+
//recordFilePath := filepath.Join(storePath, _commitFileName)
52+
//_cdb, err = bbolt.Open(recordFilePath, _fileMode, bbolt.DefaultOptions)
53+
//if err != nil {
54+
// return errors.New(fmt.Sprintf("open boltdb: %s", err.Error()))
55+
//}
56+
57+
err = _mdb.Update(func(tx *bbolt.Tx) error {
58+
if _, err = tx.CreateBucketIfNotExists(_sourceBucket); err != nil {
59+
return err
60+
}
61+
if _, err = tx.CreateBucketIfNotExists(_endpointBucket); err != nil {
62+
return err
63+
}
64+
if _, err = tx.CreateBucketIfNotExists(_ruleBucket); err != nil {
65+
return err
66+
}
67+
if _, err = tx.CreateBucketIfNotExists(_pipelineBucket); err != nil {
68+
return err
69+
}
70+
if _, err = tx.CreateBucketIfNotExists(_positionBucket); err != nil {
71+
return err
72+
}
73+
74+
if _, err = tx.CreateBucketIfNotExists(_positionBucket); err != nil {
75+
return err
76+
}
77+
if _, err = tx.CreateBucketIfNotExists(_streamStateBucket); err != nil {
78+
return err
79+
}
80+
81+
return nil
82+
})
83+
if err != nil {
84+
return errors.New(fmt.Sprintf("create bucket: %s", err.Error()))
85+
}
86+
87+
//err = _cdb.Update(func(tx *bbolt.Tx) error {
88+
// tx.CreateBucketIfNotExists(_commitBucket)
89+
// return nil
90+
//})
91+
//if err != nil {
92+
// return errors.New(fmt.Sprintf("create bucket: %s", err.Error()))
93+
//}
94+
95+
return nil
96+
}
97+
98+
func closeBolt() {
99+
if _mdb != nil {
100+
_mdb.Close()
101+
}
102+
103+
if _cdb != nil {
104+
_cdb.Close()
105+
}
106+
}
107+
108+
func marshalId(id uint64) []byte {
109+
buf := make([]byte, 8)
110+
binary.BigEndian.PutUint64(buf, id)
111+
return buf
112+
}

‎dao/bolt/bolt.go

Lines changed: 0 additions & 111 deletions
This file was deleted.

‎dao/bolt/source_dao_test.go

Lines changed: 0 additions & 70 deletions
This file was deleted.

0 commit comments

Comments
 (0)