抓取重构

polaris1119 · polaris1119 · commit 8b806bd163ad · 2014-10-08T19:17:34.000+08:00
diff --git a/websites/code/studygolang/src/config/config.go b/websites/code/studygolang/src/config/config.go
@@ -8,11 +8,14 @@ package config
 
 import (
 	"encoding/json"
+	"errors"
 	"io/ioutil"
 	"path"
-	"process"
+	"reflect"
 	"strconv"
 	"strings"
+
+	"process"
 )
 
 // 项目根目录
@@ -44,14 +47,23 @@ const Gt = ">"
 
 type Conf map[string]interface{}
 
-func ParseConfig(filename string) (Conf, error) {
+func ParseConfig(filename string, store interface{}) (Conf, error) {
 	content, err := ioutil.ReadFile(ROOT + filename)
 	if err != nil {
 		return nil, err
 	}
 
 	var conf Conf
-	err = json.Unmarshal(content, &conf)
+	if store == nil {
+		store = &conf
+	} else {
+		storeType := reflect.TypeOf(store)
+		if storeType.Kind() != reflect.Ptr {
+			return nil, errors.New("store must be pointer or nil")
+		}
+	}
+
+	err = json.Unmarshal(content, store)
 	if err != nil {
 		return nil, err
 	}
diff --git a/websites/code/studygolang/src/server/crawlarticle/autocrawl.go b/websites/code/studygolang/src/server/crawlarticle/autocrawl.go
@@ -8,6 +8,7 @@ package main
 
 import (
 	"fmt"
+	"log"
 	"regexp"
 	"strings"
 
@@ -19,87 +20,42 @@ import (
 	"util"
 )
 
-var websites = map[string]map[string]string{
-	"cnblogs": {
-		"all_url":        "http://zzk.cnblogs.com/s?t=b&w=",
-		"incr_url":       "http://zzk.cnblogs.com/s?t=b&dateRange=One-Week&w=",
-		"keywords":       "golang,go语言",
-		"listselector":   "#searchResult .searchItem", // 搜索结果一项的选择器
-		"resultselector": "h3 a",
-		"page_field":     "p",
-		"max_page":       "30",
-	},
-	"csdn": {
-		"all_url":        "http://so.csdn.net/so/search/s.do?t=blog&q=",
-		"incr_url":       "http://so.csdn.net/so/search/s.do?t=blog&q=",
-		"keywords":       "go,golang,golang语言,go语言",
-		"listselector":   ".search-list",
-		"resultselector": "dt a",
-		"page_field":     "p",
-		"max_page":       "13",
-	},
-	"oschina": {
-		"all_url":        "http://www.oschina.net/search?scope=blog&q=",
-		"incr_url":       "http://www.oschina.net/search?scope=blog&sort_by_time=1&q=",
-		"keywords":       "go,golang",
-		"listselector":   "#results li",
-		"resultselector": "h3 a",
-		"page_field":     "p",
-		"max_page":       "50",
-	},
-	"oschina_translate": {
-		"all_url":        "http://www.oschina.net/search?scope=translate&q=",
-		"incr_url":       "http://www.oschina.net/search?scope=translate&sort_by_time=1&q=",
-		"keywords":       "go,golang",
-		"listselector":   "#results li",
-		"resultselector": "h3 a",
-		"page_field":     "p",
-		"max_page":       "50",
-	},
-	"iteye": {
-		"all_url":        "http://www.iteye.com/search?type=blog&query=",
-		"incr_url":       "http://www.iteye.com/search?type=blog&sort=created_at&query=",
-		"keywords":       "go语言,golang",
-		"listselector":   "#search_result .topic",
-		"resultselector": ".content h4 a",
-		"page_field":     "page",
-		"max_page":       "20",
-	},
-	"iteye_news": {
-		"all_url":        "http://www.iteye.com/search?type=news&query=",
-		"incr_url":       "http://www.iteye.com/search?type=news&sort=created_at&query=",
-		"keywords":       "go语言,golang",
-		"listselector":   "#search_result .topic",
-		"resultselector": ".content h4 a",
-		"page_field":     "page",
-		"max_page":       "20",
-	},
-}
+var websites = make(map[string]map[string]string)
 
 const pattern = "go|golang|goroutine|channel/i"
 
-func autocrawl(needAll bool) {
+func autocrawl(needAll bool, crawlConfFile string, whichSite string) {
+
+	_, err := config.ParseConfig(crawlConfFile, &websites)
+	if err != nil {
+		log.Fatalln("parse crawl config error:", err)
+	}
 
 	if needAll {
 		// 全量
 		for website, wbconf := range websites {
+			if whichSite != "" && whichSite != website {
+				continue
+			}
+
 			logger.Infoln("all crawl", website)
 			go doCrawl(wbconf, true)
 		}
 	}
 
 	// 定时增量
 	c := cron.New()
-	c.AddFunc(config.Config["crawl_spec"], startCrawl)
-	c.Start()
-}
-
-func startCrawl() {
+	c.AddFunc(config.Config["crawl_spec"], func() {
+		for website, wbconf := range websites {
+			if whichSite != "" && whichSite != website {
+				continue
+			}
 
-	for website, wbconf := range websites {
-		logger.Infoln("do crawl", website)
-		go doCrawl(wbconf, false)
-	}
+			logger.Infoln("do crawl", website)
+			go doCrawl(wbconf, false)
+		}
+	})
+	c.Start()
 }
 
 func doCrawl(wbconf map[string]string, isAll bool) {
diff --git a/websites/code/studygolang/src/server/crawlarticle/main.go b/websites/code/studygolang/src/server/crawlarticle/main.go
@@ -28,11 +28,17 @@ func init() {
 }
 
 func main() {
-	var needAll bool
+	var (
+		needAll          bool
+		crawConfFilename string
+		whichSite        string
+	)
 	flag.BoolVar(&needAll, "all", false, "是否需要全量抓取，默认否")
+	flag.StringVar(&crawConfFilename, "config", "conf/auto_crawl_conf.json", "自动抓取配置文件")
+	flag.StringVar(&whichSite, "site", "", "抓取配置中哪个站点（空表示所有配置站点）")
 	flag.Parse()
 
-	go autocrawl(needAll)
+	go autocrawl(needAll, crawConfFilename, whichSite)
 
 	router := initRouter()
 	http.Handle("/", router)
diff --git a/websites/code/studygolang/src/service/article.go b/websites/code/studygolang/src/service/article.go
@@ -76,15 +76,21 @@ func ParseArticle(articleUrl string, auto bool) (*model.Article, error) {
 		author = urlPaths[index]
 		authorTxt = author
 	} else {
-		authorSelection := doc.Find(rule.Author)
-		author, err = authorSelection.Html()
-		if err != nil {
-			logger.Errorln("goquery parse author error:", err)
-			return nil, err
+		if strings.HasPrefix(rule.Author, ".") || strings.HasPrefix(rule.Author, "#") {
+			authorSelection := doc.Find(rule.Author)
+			author, err = authorSelection.Html()
+			if err != nil {
+				logger.Errorln("goquery parse author error:", err)
+				return nil, err
+			}
+
+			author = strings.TrimSpace(author)
+			authorTxt = strings.TrimSpace(authorSelection.Text())
+		} else {
+			// 某些个人博客，页面中没有作者的信息，因此，规则中 author 即为 作者
+			author = rule.Author
+			authorTxt = rule.Author
 		}
-
-		author = strings.TrimSpace(author)
-		authorTxt = strings.TrimSpace(authorSelection.Text())
 	}
 
 	title := ""
