Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit dc60671

Browse files
lcy-sesolcy-seso
committed
Merge branch 'develop' into enable_error_clipping_for_fc
2 parents 2033642 + 77604bc commit dc60671

41 files changed

Lines changed: 353 additions & 124 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.pre-commit-config.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,3 +21,10 @@
2121
sha: 28c0ea8a67a3e2dbbf4822ef44e85b63a0080a29
2222
hooks:
2323
- id: clang-formater
24+
- repo: https://github.com/dnephin/pre-commit-golang
25+
sha: e4693a4c282b4fc878eda172a929f7a6508e7d16
26+
hooks:
27+
- id: go-fmt
28+
files: (.*\.go)
29+
- id: go-lint
30+
files: (.*\.go)

‎.travis.yml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,16 +33,17 @@ addons:
3333
- ccache
3434
before_install:
3535
- if [[ "$JOB" == "check_style" ]]; then sudo ln -s /usr/bin/clang-format-3.8 /usr/bin/clang-format; fi
36-
# Paddle is using protobuf 3.1 currently. Protobuf 3.2 breaks the compatibility. So we specify the python
36+
# Paddle is using protobuf 3.1 currently. Protobuf 3.2 breaks the compatibility. So we specify the python
3737
# protobuf version.
3838
- pip install numpy wheel 'protobuf==3.1' sphinx==1.5.6 recommonmark sphinx-rtd-theme==0.1.9 virtualenv pre-commit requests==2.9.2 LinkChecker
3939
- pip install rarfile
40+
- curl https://glide.sh/get | bash
4041
- eval "$(GIMME_GO_VERSION=1.8.3 gimme)"
4142
- |
4243
function timeout() { perl -e 'alarm shift; exec @ARGV' "$@"; }
4344
script:
4445
- |
45-
export WITH_GOLANG=ON && timeout 2580 paddle/scripts/travis/${JOB}.sh # 43min timeout
46+
timeout 2580 paddle/scripts/travis/${JOB}.sh # 43min timeout
4647
RESULT=$?; if [ $RESULT -eq 0 ] || [ $RESULT -eq 142 ]; then true; else false; fi;
4748
notifications:
4849
email:

‎cmake/external/protobuf.cmake‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,65 @@ INCLUDE(ExternalProject)
1717
FIND_PACKAGE(Protobuf QUIET)
1818
SET(PROTOBUF_FOUND "OFF")
1919

20+
if(NOT COMMAND protobuf_generate_python) # before cmake 3.4, protobuf_genrerate_python is not defined.
21+
function(protobuf_generate_python SRCS)
22+
# shameless copy from https://github.com/Kitware/CMake/blob/master/Modules/FindProtobuf.cmake
23+
if(NOT ARGN)
24+
message(SEND_ERROR "Error: PROTOBUF_GENERATE_PYTHON() called without any proto files")
25+
return()
26+
endif()
27+
28+
if(PROTOBUF_GENERATE_CPP_APPEND_PATH)
29+
# Create an include path for each file specified
30+
foreach(FIL ${ARGN})
31+
get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
32+
get_filename_component(ABS_PATH ${ABS_FIL} PATH)
33+
list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
34+
if(${_contains_already} EQUAL -1)
35+
list(APPEND _protobuf_include_path -I ${ABS_PATH})
36+
endif()
37+
endforeach()
38+
else()
39+
set(_protobuf_include_path -I ${CMAKE_CURRENT_SOURCE_DIR})
40+
endif()
41+
42+
if(DEFINED PROTOBUF_IMPORT_DIRS AND NOT DEFINED Protobuf_IMPORT_DIRS)
43+
set(Protobuf_IMPORT_DIRS "${PROTOBUF_IMPORT_DIRS}")
44+
endif()
45+
46+
if(DEFINED Protobuf_IMPORT_DIRS)
47+
foreach(DIR ${Protobuf_IMPORT_DIRS})
48+
get_filename_component(ABS_PATH ${DIR} ABSOLUTE)
49+
list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
50+
if(${_contains_already} EQUAL -1)
51+
list(APPEND _protobuf_include_path -I ${ABS_PATH})
52+
endif()
53+
endforeach()
54+
endif()
55+
56+
set(${SRCS})
57+
foreach(FIL ${ARGN})
58+
get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
59+
get_filename_component(FIL_WE ${FIL} NAME_WE)
60+
if(NOT PROTOBUF_GENERATE_CPP_APPEND_PATH)
61+
get_filename_component(FIL_DIR ${FIL} DIRECTORY)
62+
if(FIL_DIR)
63+
set(FIL_WE "${FIL_DIR}/${FIL_WE}")
64+
endif()
65+
endif()
66+
67+
list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}_pb2.py")
68+
add_custom_command(
69+
OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}_pb2.py"
70+
COMMAND ${Protobuf_PROTOC_EXECUTABLE} --python_out ${CMAKE_CURRENT_BINARY_DIR} ${_protobuf_include_path} ${ABS_FIL}
71+
DEPENDS ${ABS_FIL} ${Protobuf_PROTOC_EXECUTABLE}
72+
COMMENT "Running Python protocol buffer compiler on ${FIL}"
73+
VERBATIM )
74+
endforeach()
75+
76+
set(${SRCS} ${${SRCS}} PARENT_SCOPE)
77+
endfunction()
78+
endif()
2079

2180
# Print and set the protobuf library information,
2281
# finish this cmake process and exit from this file.

‎cmake/generic.cmake‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -194,7 +194,7 @@ function(cc_test TARGET_NAME)
194194
add_executable(${TARGET_NAME} ${cc_test_SRCS})
195195
target_link_libraries(${TARGET_NAME} ${cc_test_DEPS} gtest gtest_main)
196196
add_dependencies(${TARGET_NAME} ${cc_test_DEPS} gtest gtest_main)
197-
add_test(${TARGET_NAME} ${TARGET_NAME})
197+
add_test(NAME ${TARGET_NAME} COMMAND ${TARGET_NAME} WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR})
198198
endif()
199199
endfunction(cc_test)
200200

@@ -281,10 +281,11 @@ function(go_library TARGET_NAME)
281281

282282
file(GLOB GO_SOURCE RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "*.go")
283283
string(REPLACE "${PADDLE_GO_PATH}/" "" CMAKE_CURRENT_SOURCE_REL_DIR ${CMAKE_CURRENT_SOURCE_DIR})
284+
# FIXME: link path
284285
add_custom_command(TARGET ${TARGET_NAME} POST_BUILD
285286
COMMAND rm "${${TARGET_NAME}_LIB_PATH}"
286287
# Golang build source code
287-
COMMAND env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} build ${BUILD_MODE}
288+
COMMAND GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} build ${BUILD_MODE}
288289
-o "${${TARGET_NAME}_LIB_PATH}"
289290
"./${CMAKE_CURRENT_SOURCE_REL_DIR}/${GO_SOURCE}"
290291
# must run under GOPATH
@@ -299,11 +300,13 @@ function(go_binary TARGET_NAME)
299300
cmake_parse_arguments(go_binary "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
300301
string(REPLACE "${PADDLE_GO_PATH}/" "" CMAKE_CURRENT_SOURCE_REL_DIR ${CMAKE_CURRENT_SOURCE_DIR})
301302

303+
# FIXME: link path
302304
add_custom_command(OUTPUT ${TARGET_NAME}_timestamp
303-
COMMAND env GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} build
305+
COMMAND env LIBRARY_PATH=${CMAKE_BINARY_DIR}/go/pserver/client/c/:$ENV{LIBRARY_PATH}
306+
GOPATH=${GOPATH} ${CMAKE_Go_COMPILER} build
304307
-o "${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME}"
305308
"./${CMAKE_CURRENT_SOURCE_REL_DIR}/${go_binary_SRCS}"
306-
WORKING_DIRECTORY "${PADDLE_IN_GOPATH}/go")
309+
WORKING_DIRECTORY "${PADDLE_IN_GOPATH}/go")
307310
# TODO: don't know what ${TARGET_NAME}_link does
308311
add_custom_target(${TARGET_NAME} ALL DEPENDS go_vendor ${TARGET_NAME}_timestamp ${go_binary_DEPS})
309312
install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME} DESTINATION bin)
@@ -332,3 +335,12 @@ function(proto_library TARGET_NAME)
332335
protobuf_generate_cpp(proto_srcs proto_hdrs ${proto_library_SRCS})
333336
cc_library(${TARGET_NAME} SRCS ${proto_srcs} DEPS ${proto_library_DEPS} protobuf)
334337
endfunction()
338+
339+
function(py_proto_compile TARGET_NAME)
340+
set(oneValueArgs "")
341+
set(multiValueArgs SRCS)
342+
cmake_parse_arguments(py_proto_compile "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
343+
set(py_srcs)
344+
protobuf_generate_python(py_srcs ${py_proto_compile_SRCS})
345+
add_custom_target(${TARGET_NAME} ALL DEPENDS ${py_srcs})
346+
endfunction()

‎go/cmd/master/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,4 +12,4 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15-
go_binary(master SRC master.go)
15+
go_binary(master SRC master.go DEPS paddle_go_optimizer)

‎go/cmd/pserver/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,4 +12,4 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15-
go_binary(pserver SRCS pserver.go)
15+
go_binary(pserver SRCS pserver.go DEPS paddle_go_optimizer)

‎go/pserver/client/c/CMakeLists.txt‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
cc_library(paddle_go_optimizer DEPS paddle_optimizer paddle_proto glog gflags protobuf)
2+
target_link_libraries(paddle_go_optimizer stdc++ m)
23
go_library(paddle_pserver_cclient STATIC DEPS paddle_go_optimizer)
34
if(WITH_TESTING)
4-
add_subdirectory(test)
5+
# FIXME: this test requires pserver which is not managed by the test
6+
# we need some kind of e2e testing machanism.
7+
# add_subdirectory(test)
58
endif()

‎go/pserver/client/c/test/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
1-
cc_test(test_cclient SRCS test_cclient.c DEPS paddle_pserver_cclient)
1+
cc_test(test_cclient SRCS test_cclient.c DEPS paddle_pserver_cclient paddle_go_optimizer)
22
add_style_check_target(test_cclient test_cclient.c)

‎go/pserver/optimizer.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ package pserver
22

33
// #cgo CFLAGS: -I ../../
44
// //FIXME: ldflags contain "build" path
5-
// #cgo LDFLAGS: ../../build/go/pserver/client/c/libpaddle_go_optimizer.a -lstdc++ -lm
5+
// #cgo LDFLAGS: ${SRCDIR}/../../build/go/pserver/client/c/libpaddle_go_optimizer.a -lstdc++ -lm
66
// #include "paddle/optimizer/optimizer.h"
77
// #include <stdlib.h>
88
// #include <string.h>
@@ -56,8 +56,8 @@ func newOptimizer(paramWithConfigs ParameterWithConfig) *optimizer {
5656

5757
func (o *optimizer) GetWeights() []byte {
5858
var buffer unsafe.Pointer
59-
buffer_len := C.paddle_optimizer_get_weights(o.opt, &buffer)
60-
return cArrayToSlice(buffer, int(buffer_len)*C.sizeof_float)
59+
bufferLen := C.paddle_optimizer_get_weights(o.opt, &buffer)
60+
return cArrayToSlice(buffer, int(bufferLen)*C.sizeof_float)
6161
}
6262

6363
func (o *optimizer) UpdateParameter(g Gradient) error {

‎go/pserver/service.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,10 @@ import (
1010
type ElementType int
1111

1212
const (
13+
// AlreadyInitialized is true if pserver is initialized
1314
AlreadyInitialized = "pserver already initialized"
14-
Uninitialized = "pserver not fully initialized"
15+
// Uninitialized is true if pserver not fully initialized
16+
Uninitialized = "pserver not fully initialized"
1517
)
1618

1719
// Supported element types
@@ -55,7 +57,7 @@ func NewService(idx int) (*Service, error) {
5557
s := &Service{
5658
idx: idx,
5759
}
58-
s.optMap = make(map[string]*optimizer)
60+
s.optMap = make(map[string]*optimizer)
5961
s.initialized = make(chan struct{})
6062
return s, nil
6163
}

0 commit comments

Comments
 (0)