自动抓取更通用

zhima1234 · zhima1234 · commit 1225c3af1133 · 2017-05-22T16:56:58.000+08:00
diff --git a/config/env.sample.ini b/config/env.sample.ini
@@ -54,9 +54,10 @@ https_domain = xxx
 
 ; 抓取程序
 [crawl]
-listen = 5050
 spec = 0 0 */1 * * ?
 iframe_deny = github.com,robbinfan.com,www.zhihu.com,google.com,golang.org,evernote.com,blogspot.com
+reddit_path = /r/golang/new/
+project_url = http://www.oschina.net/project/lang/358/go?tag=0&os=0&sort=time
 
 [search]
 engine_url = http://127.0.0.1:7070/solr/studygolang
diff --git a/src/logic/auto_crawl.go b/src/logic/auto_crawl.go
@@ -18,11 +18,12 @@ import (
 	"strings"
 
 	"github.com/PuerkitoBio/goquery"
+	"github.com/polaris1119/config"
 	"github.com/polaris1119/logger"
 	"golang.org/x/net/context"
 )
 
-const titlePattern = "(?i)go|golang|goroutine|channel"
+var titlePattern = config.ConfigFile.MustValue("crawl", "article_title_pattern")
 
 type AutoCrawlLogic struct{}
 
@@ -32,7 +33,7 @@ func (self AutoCrawlLogic) DoCrawl(isAll bool) error {
 	autoCrawlConfList := make([]*model.AutoCrawlRule, 0)
 	err := MasterDB.Where("status=?", model.AutoCrawlOn).Find(&autoCrawlConfList)
 	if err != nil {
-		logger.Errorln("ArticleLogic FindBy Error:", err)
+		logger.Errorln("AutoCrawlLogic FindBy Error:", err)
 		return err
 	}
 
@@ -163,7 +164,7 @@ func (self AutoCrawlLogic) parseArticleList(strUrl string, autoCrawlConf *model.
 		aSelection := contentSelection.Find(resultSelector)
 
 		// 搜索时，避免搜到垃圾，对标题进一步判断
-		if isSearch {
+		if isSearch && titlePattern != "" {
 			title := aSelection.Text()
 
 			matched, err := regexp.MatchString(titlePattern, title)
diff --git a/src/logic/reddit.go b/src/logic/reddit.go
@@ -20,24 +20,29 @@ import (
 	"model"
 
 	"github.com/PuerkitoBio/goquery"
+	"github.com/polaris1119/config"
 	"github.com/polaris1119/logger"
 )
 
 type RedditLogic struct {
 	domain string
-	golang string
+	path   string
 }
 
-var DefaultReddit = RedditLogic{
-	domain: "https://www.reddit.com",
-	golang: "/r/golang/new/",
+var DefaultReddit = newRedditLogic()
+
+func newRedditLogic() *RedditLogic {
+	return &RedditLogic{
+		domain: "https://www.reddit.com",
+		path:   config.ConfigFile.MustValue("crawl", "reddit_path"),
+	}
 }
 
 // Parse 获取url对应的资源并根据规则进行解析
 func (this *RedditLogic) Parse(redditUrl string) error {
 	redditUrl = strings.TrimSpace(redditUrl)
 	if redditUrl == "" {
-		redditUrl = this.domain + this.golang
+		redditUrl = this.domain + this.path
 	} else if !strings.HasPrefix(redditUrl, "https") {
 		redditUrl = "https://" + redditUrl
 	}
@@ -126,7 +131,7 @@ func (this *RedditLogic) dealRedditOneResource(contentSelection *goquery.Selecti
 		var doc *goquery.Document
 
 		if doc, err = goquery.NewDocument(resourceUrl); err != nil {
-			return errors.New("goquery reddit.com/r/golang self newdocument error:" + err.Error())
+			return errors.New("goquery reddit.com" + this.path + " self newdocument error:" + err.Error())
 		}
 
 		content, err := doc.Find("#siteTable .usertext .md").Html()
@@ -152,7 +157,7 @@ func (this *RedditLogic) dealRedditOneResource(contentSelection *goquery.Selecti
 		})
 
 		if strings.TrimSpace(content) == "" {
-			return errors.New("goquery reddit.com/r/golang self newdocument(" + resourceUrl + ") error: content is empty")
+			return errors.New("goquery reddit.com" + this.path + " self newdocument(" + resourceUrl + ") error: content is empty")
 		}
 
 		resource.Content = content
diff --git a/src/server/crawler/main.go b/src/server/crawler/main.go
@@ -55,7 +55,7 @@ func autocrawl(needAll bool, whichSite string) {
 		go logic.DefaultReddit.Parse("")
 
 		// 抓取 www.oschina.net/project
-		go logic.DefaultProject.ParseProjectList("http://www.oschina.net/project/lang/358/go?tag=0&os=0&sort=time")
+		go logic.DefaultProject.ParseProjectList(config.ConfigFile.MustValue("crawl", "project_url"))
 
 		// 抓取 article
 		go logic.DefaultAutoCrawl.DoCrawl(false)
