配置化

zhima1234 · zhima1234 · commit 66b8ce39803b · 2017-05-23T11:37:10.000+08:00
diff --git a/config/db.sql b/config/db.sql
@@ -290,6 +290,7 @@ CREATE TABLE IF NOT EXISTS `crawl_rule` (
   `author` varchar(127) NOT NULL DEFAULT '' COMMENT '文章作者规则',
   `pub_date` varchar(127) NOT NULL DEFAULT '' COMMENT '发布时间规则',
   `content` varchar(127) NOT NULL DEFAULT '' COMMENT '正文规则',
+  `ext` varchar(1022) NOT NULL DEFAULT '' COMMENT '扩展，比如附加 css 等，json格式',
   `op_user` varchar(20) NOT NULL DEFAULT '' COMMENT '操作人',
   `ctime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
   PRIMARY KEY (`id`),
diff --git a/config/env.sample.ini b/config/env.sample.ini
@@ -58,6 +58,9 @@ spec = 0 0 */1 * * ?
 iframe_deny = github.com,robbinfan.com,www.zhihu.com,google.com,golang.org,evernote.com,blogspot.com
 reddit_path = /r/golang/new/
 project_url = http://www.oschina.net/project/lang/358/go?tag=0&os=0&sort=time
+article_title_pattern = (?i)go|golang|goroutine|channel
+preset_users = polaris
+preset_uids = 1
 
 [search]
 engine_url = http://127.0.0.1:7070/solr/studygolang
diff --git a/src/http/controller/admin/project.go b/src/http/controller/admin/project.go
@@ -76,7 +76,7 @@ func (ProjectController) CrawlProject(ctx echo.Context) error {
 
 		var errMsg string
 		for _, projectUrl := range urls {
-			err := logic.DefaultProject.ParseProjectList(strings.TrimSpace(projectUrl))
+			err := logic.DefaultProject.ParseOneProject(strings.TrimSpace(projectUrl))
 			if err != nil {
 				errMsg = err.Error()
 			}
diff --git a/src/logic/project.go b/src/logic/project.go
@@ -20,6 +20,7 @@ import (
 
 	"github.com/PuerkitoBio/goquery"
 	"github.com/lunny/html2md"
+	"github.com/polaris1119/config"
 	"github.com/polaris1119/logger"
 	"golang.org/x/net/context"
 )
@@ -264,7 +265,7 @@ const OsChinaDomain = "http://www.oschina.net"
 // ProjectLogoPrefix 开源项目 logo 前缀
 const ProjectLogoPrefix = "plogo"
 
-var PresetUsernames = []string{"polaris", "blov", "agolangf", "xuanbao"}
+var PresetUsernames = config.ConfigFile.MustValueArray("crawl", "preset_users", ",")
 
 // ParseOneProject 处理单个 project
 func (ProjectLogic) ParseOneProject(projectUrl string) error {
diff --git a/src/logic/reddit.go b/src/logic/reddit.go
@@ -21,6 +21,7 @@ import (
 
 	"github.com/PuerkitoBio/goquery"
 	"github.com/polaris1119/config"
+	"github.com/polaris1119/goutils"
 	"github.com/polaris1119/logger"
 )
 
@@ -85,7 +86,7 @@ func (this *RedditLogic) newDocumentFromResp(url string) (*goquery.Document, err
 	return goquery.NewDocumentFromResponse(resp)
 }
 
-var PresetUids = []int{1, 1747, 1748, 1827}
+var PresetUids = config.ConfigFile.MustValueArray("crawl", "preset_uids", ",")
 
 var resourceRe = regexp.MustCompile(`\n\n`)
 
@@ -177,7 +178,7 @@ func (this *RedditLogic) dealRedditOneResource(contentSelection *goquery.Selecti
 
 	resource.Title = title
 	resource.Url = resourceUrl
-	resource.Uid = PresetUids[rand.Intn(4)]
+	resource.Uid = goutils.MustInt(PresetUids[rand.Intn(len(PresetUids))])
 
 	ctime := time.Now()
 	datetime, ok := contentSelection.Find(".tagline time").Attr("datetime")

Original file line number	Diff line number	Diff line change
`@@ -76,7 +76,7 @@ func (ProjectController) CrawlProject(ctx echo.Context) error {`
`76`	`76`
`77`	`77`	`var errMsg string`
`78`	`78`	`for _, projectUrl := range urls {`
`79`		`- err := logic.DefaultProject.ParseProjectList(strings.TrimSpace(projectUrl))`
	`79`	`+ err := logic.DefaultProject.ParseOneProject(strings.TrimSpace(projectUrl))`
`80`	`80`	`if err != nil {`
`81`	`81`	`errMsg = err.Error()`
`82`	`82`	`}`