Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 5b2f993

Browse files
gongweibaogongweibao
committed
Merge branch 'develop' of https://github.com/PaddlePaddle/Paddle into fixdownloadbug
2 parents 48fdfd5 + c5dc0b7 commit 5b2f993

44 files changed

Lines changed: 1643 additions & 232 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎cmake/external/eigen.cmake‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,17 @@ INCLUDE_DIRECTORIES(${EIGEN_SOURCE_DIR}/src/eigen3)
77
ExternalProject_Add(
88
eigen3
99
${EXTERNAL_PROJECT_LOG_ARGS}
10-
URL "https://bitbucket.org/eigen/eigen/get/3.3.4.tar.gz"
11-
URL_MD5 "1a47e78efe365a97de0c022d127607c3"
10+
# for latest version, please get from official website
11+
# URL "https://bitbucket.org/eigen/eigen/get/3.3.4.tar.gz"
12+
# URL_MD5 "1a47e78efe365a97de0c022d127607c3"
13+
14+
# for no-ssl http support, please get from bazel's mirror
15+
# URL "http://mirror.bazel.build/bitbucket.org/eigen/eigen/get/f3a22f35b044.tar.gz"
16+
# URL_MD5 "4645c66075982da6fa0bcf6b20f3e8f7"
17+
18+
# get from github mirror
19+
GIT_REPOSITORY "https://github.com/RLovelett/eigen.git"
20+
GIT_TAG "a46d2e7337c4656f00abe54a8115f6d76153a048"
1221
PREFIX ${EIGEN_SOURCE_DIR}
1322
UPDATE_COMMAND ""
1423
CONFIGURE_COMMAND ""

‎cmake/external/protobuf.cmake‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,10 @@
1313
# limitations under the License.
1414

1515
INCLUDE(ExternalProject)
16+
# Always invoke `FIND_PACKAGE(Protobuf)` for importing function protobuf_generate_cpp
17+
FIND_PACKAGE(Protobuf QUIET)
18+
SET(PROTOBUF_FOUND "OFF")
19+
1620

1721
# Print and set the protobuf library information,
1822
# finish this cmake process and exit from this file.
@@ -39,12 +43,19 @@ macro(PROMPT_PROTOBUF_LIB)
3943
ADD_LIBRARY(protobuf_lite ${protobuf_LIBTYPE} IMPORTED GLOBAL)
4044
SET_PROPERTY(TARGET protobuf_lite PROPERTY IMPORTED_LOCATION ${PROTOBUF_LITE_LIBRARY})
4145

42-
ADD_LIBRARY(protoc ${protobuf_LIBTYPE} IMPORTED GLOBAL)
43-
SET_PROPERTY(TARGET protoc PROPERTY IMPORTED_LOCATION ${PROTOC_LIBRARY})
46+
ADD_LIBRARY(libprotoc ${protobuf_LIBTYPE} IMPORTED GLOBAL)
47+
SET_PROPERTY(TARGET libprotoc PROPERTY IMPORTED_LOCATION ${PROTOC_LIBRARY})
48+
49+
ADD_EXECUTABLE(protoc IMPORTED GLOBAL)
50+
SET_PROPERTY(TARGET protoc PROPERTY IMPORTED_LOCATION ${PROTOBUF_PROTOC_EXECUTABLE})
51+
# FIND_Protobuf.cmake uses `Protobuf_PROTOC_EXECUTABLE`.
52+
# make `protobuf_generate_cpp` happy.
53+
SET(Protobuf_PROTOC_EXECUTABLE ${PROTOBUF_PROTOC_EXECUTABLE})
4454

4555
FOREACH(dep ${protobuf_DEPS})
4656
ADD_DEPENDENCIES(protobuf ${dep})
4757
ADD_DEPENDENCIES(protobuf_lite ${dep})
58+
ADD_DEPENDENCIES(libprotoc ${dep})
4859
ADD_DEPENDENCIES(protoc ${dep})
4960
ENDFOREACH()
5061

‎cmake/generic.cmake‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,9 @@
8787
# go_library(example SHARED)
8888
#
8989

90+
# including binary directory for generated headers.
91+
include_directories(${CMAKE_BINARY_DIR})
92+
9093
if(NOT APPLE)
9194
find_package(Threads REQUIRED)
9295
link_libraries(${CMAKE_THREAD_LIBS_INIT})
@@ -331,3 +334,13 @@ function(go_test TARGET_NAME)
331334
add_custom_target(${TARGET_NAME} ALL DEPENDS ${TARGET_NAME}_timestamp ${go_test_DEPS})
332335
add_test(${TARGET_NAME} ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME})
333336
endfunction(go_test)
337+
338+
function(proto_library TARGET_NAME)
339+
set(oneValueArgs "")
340+
set(multiValueArgs SRCS)
341+
cmake_parse_arguments(proto_library "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
342+
set(proto_srcs)
343+
set(proto_hdrs)
344+
protobuf_generate_cpp(proto_srcs proto_hdrs ${proto_library_SRCS})
345+
cc_library(${TARGET_NAME} SRCS ${proto_srcs} DEPS protobuf)
346+
endfunction()

‎doc/CMakeLists.txt‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,6 @@ sphinx_add_target(paddle_docs
2727
${CMAKE_CURRENT_SOURCE_DIR}
2828
${SPHINX_HTML_DIR_EN})
2929

30-
add_dependencies(paddle_docs
31-
gen_proto_py)
32-
33-
3430
# configured documentation tools and intermediate build results
3531
set(BINARY_BUILD_DIR_CN "${CMAKE_CURRENT_BINARY_DIR}/cn/_build")
3632

@@ -51,6 +47,3 @@ sphinx_add_target(paddle_docs_cn
5147
${SPHINX_CACHE_DIR_CN}
5248
${CMAKE_CURRENT_SOURCE_DIR}
5349
${SPHINX_HTML_DIR_CN})
54-
55-
add_dependencies(paddle_docs_cn
56-
gen_proto_py)

‎doc/design/scope.md‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ class Scope {
4141
const Variable* GetVariable(const std::string& name) const;
4242

4343
private:
44-
std::unordered_map<std::string, std::unique_ptr<Vairable>> vars_;
44+
std::unordered_map<std::string, std::unique_ptr<Variable>> vars_;
4545
};
4646
```
4747
@@ -59,9 +59,9 @@ class Scope {
5959
Scope(const std::shared_ptr<Scope>& scope): parent_(scope) {}
6060
6161
Variable* GetVariable(const std::string& name) const {
62-
Variable* var = GetVarLocally(name);
63-
if (var != nullptr) {
64-
return var;
62+
auto it = vars_.find(name);
63+
if (it != vars_.end()) {
64+
return it->second.get();
6565
} else if (parent_ != nullptr) {
6666
return parent_->GetVariable(name);
6767
} else {
@@ -97,8 +97,8 @@ class Scope {
9797
// return nullptr if not found.
9898
Variable* GetVariable(const std::string& name) const;
9999

100-
// return Error if already contains same name variable.
101-
Error CreateVariable(const std::string& name);
100+
// return if already contains same name variable.
101+
Variable* CreateVariable(const std::string& name);
102102

103103
private:
104104
std::shared_ptr<Scope> parent_;

‎go/master/c/client.go‎

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,13 @@ typedef int paddle_master_client;
1313
import "C"
1414

1515
import (
16+
"strings"
1617
"sync"
18+
"time"
1719
"unsafe"
1820

1921
"github.com/PaddlePaddle/Paddle/go/master"
22+
"github.com/coreos/etcd/clientv3"
2023
log "github.com/sirupsen/logrus"
2124
)
2225

@@ -48,16 +51,33 @@ func remove(client C.paddle_master_client) *master.Client {
4851
return h
4952
}
5053

51-
type addresser string
52-
53-
func (a addresser) Address() string {
54-
return string(a)
54+
//export paddle_new_etcd_master_client
55+
func paddle_new_etcd_master_client(etcdEndpoints *C.char, timeout int, bufSize int) C.paddle_master_client {
56+
p := C.GoString(etcdEndpoints)
57+
cli, err := clientv3.New(clientv3.Config{
58+
Endpoints: strings.Split(p, ","),
59+
DialTimeout: time.Second * time.Duration(timeout),
60+
})
61+
if err != nil {
62+
panic(err)
63+
}
64+
ch := make(chan string, 1)
65+
a, err := master.GetKey(cli, master.DefaultAddrPath, timeout)
66+
if err != nil {
67+
panic(err)
68+
}
69+
ch <- a
70+
go master.WatchKey(cli, master.DefaultAddrPath, ch)
71+
c := master.NewClient(ch, bufSize)
72+
return add(c)
5573
}
5674

5775
//export paddle_new_master_client
5876
func paddle_new_master_client(addr *C.char, bufSize int) C.paddle_master_client {
5977
a := C.GoString(addr)
60-
c := master.NewClient(addresser(a), bufSize)
78+
ch := make(chan string, 1)
79+
ch <- a
80+
c := master.NewClient(ch, bufSize)
6181
return add(c)
6282
}
6383

‎go/master/client.go‎

Lines changed: 4 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -2,18 +2,12 @@ package master
22

33
import (
44
"os"
5-
"time"
65

76
"github.com/PaddlePaddle/Paddle/go/connection"
87
"github.com/PaddlePaddle/recordio"
98
log "github.com/sirupsen/logrus"
109
)
1110

12-
// Addresser provide the address of the master server.
13-
type Addresser interface {
14-
Address() string
15-
}
16-
1711
// Client is the client of the master server.
1812
type Client struct {
1913
conn *connection.Conn
@@ -24,11 +18,11 @@ type Client struct {
2418
//
2519
// bufSize is the record buffer size. NextRecord will read from this
2620
// buffer.
27-
func NewClient(addr Addresser, bufSize int) *Client {
21+
func NewClient(addrCh <-chan string, bufSize int) *Client {
2822
c := &Client{}
2923
c.conn = connection.New()
3024
c.ch = make(chan []byte, bufSize)
31-
go c.monitorMaster(addr)
25+
go c.monitorMaster(addrCh)
3226
go c.getRecords()
3327
return c
3428
}
@@ -72,12 +66,10 @@ func (c *Client) getRecords() {
7266
}
7367
}
7468

75-
func (c *Client) monitorMaster(addr Addresser) {
69+
func (c *Client) monitorMaster(addrCh <-chan string) {
7670
lastMaster := ""
77-
monitor := func() {
78-
// get the lastest address of the master server,
71+
for curMaster := range addrCh {
7972
// connect to the new address once address changed.
80-
curMaster := addr.Address()
8173
if curMaster != lastMaster {
8274
if curMaster == "" {
8375
err := c.conn.Close()
@@ -94,18 +86,10 @@ func (c *Client) monitorMaster(addr Addresser) {
9486
// to retry next time.
9587
curMaster = lastMaster
9688
}
97-
9889
}
9990
}
100-
10191
lastMaster = curMaster
10292
}
103-
104-
monitor()
105-
ticker := time.NewTicker(10 * time.Second)
106-
for _ = range ticker.C {
107-
monitor()
108-
}
10993
}
11094

11195
// SetDataset set dataset for the master server to dispatch.

‎go/master/client_internal_test.go‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,6 @@ func init() {
2626
log.SetLevel(log.ErrorLevel)
2727
}
2828

29-
type TestAddresser string
30-
31-
func (a TestAddresser) Address() string {
32-
return string(a)
33-
}
34-
3529
func TestGetFinishTask(t *testing.T) {
3630
const path = "/tmp/master_client_test_0"
3731

@@ -45,7 +39,6 @@ func TestGetFinishTask(t *testing.T) {
4539
if err != nil {
4640
panic(err)
4741
}
48-
4942
go func(l net.Listener) {
5043
s, err := NewService(&InMemStore{}, chunkPerTask, time.Second, 1)
5144
if err != nil {
@@ -82,9 +75,11 @@ func TestGetFinishTask(t *testing.T) {
8275
// Manually intialize client to avoid calling c.getRecords()
8376
c := &Client{}
8477
c.conn = connection.New()
85-
go c.monitorMaster(TestAddresser(fmt.Sprintf(":%d", p)))
78+
addr := fmt.Sprintf(":%d", p)
79+
ch := make(chan string, 1)
80+
ch <- addr
81+
go c.monitorMaster(ch)
8682
c.SetDataset([]string{path})
87-
8883
checkOnePass := func(i int) {
8984
var tasks []Task
9085
for idx := 0; idx < totalTask; idx++ {

‎go/master/client_test.go‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ func TestNextRecord(t *testing.T) {
2020
path = "/tmp/master_client_TestFull"
2121
total = 50
2222
)
23-
2423
l, err := net.Listen("tcp", ":0")
2524
if err != nil {
2625
panic(err)
@@ -31,7 +30,6 @@ func TestNextRecord(t *testing.T) {
3130
if err != nil {
3231
panic(err)
3332
}
34-
3533
go func(l net.Listener) {
3634
s, err := master.NewService(&master.InMemStore{}, 10, time.Second, 1)
3735
if err != nil {
@@ -63,10 +61,10 @@ func TestNextRecord(t *testing.T) {
6361
}
6462
w.Close()
6563
f.Close()
66-
67-
c := master.NewClient(master.TestAddresser(fmt.Sprintf(":%d", p)), 10)
64+
curAddr := make(chan string, 1)
65+
curAddr <- fmt.Sprintf(":%d", p)
66+
c := master.NewClient(curAddr, 10)
6867
c.SetDataset([]string{path})
69-
7068
for pass := 0; pass < 50; pass++ {
7169
received := make(map[byte]bool)
7270
for i := 0; i < total; i++ {

‎go/master/etcd_client.go‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,3 +142,31 @@ func (e *EtcdClient) Load() ([]byte, error) {
142142
state := kvs[0].Value
143143
return state, nil
144144
}
145+
146+
// GetKey gets the value by the specify key.
147+
func GetKey(c *clientv3.Client, key string, timeout int) (string, error) {
148+
ctx, cancel := context.WithTimeout(context.Background(), time.Second*time.Duration(timeout))
149+
resp, err := c.Get(ctx, key)
150+
cancel()
151+
if err != nil {
152+
return "", err
153+
}
154+
kvs := resp.Kvs
155+
if len(kvs) == 0 {
156+
return "", nil
157+
}
158+
v := kvs[0].Value
159+
return string(v), nil
160+
}
161+
162+
// WatchKey watches the specify key and send to valChan if there is some event.
163+
func WatchKey(c *clientv3.Client, key string, valChan chan<- string) {
164+
rch := c.Watch(context.Background(), key)
165+
for wresp := range rch {
166+
for _, ev := range wresp.Events {
167+
// if received event is DELETE, the value will be an empty string
168+
log.Infof("received event %s, %q : %q\n", ev.Type, ev.Kv.Key, ev.Kv.Value)
169+
valChan <- string(ev.Kv.Value)
170+
}
171+
}
172+
}

0 commit comments

Comments
 (0)