自动抓取文章，url去除 多余参数

zhima1234 · zhima1234 · commit 29fd64bdd559 · 2017-02-10T16:20:13.000+08:00
diff --git a/src/logic/article.go b/src/logic/article.go
@@ -42,6 +42,8 @@ func (self ArticleLogic) ParseArticle(ctx context.Context, articleUrl string, au
 		articleUrl = "http://" + articleUrl
 	}
 
+	articleUrl = self.cleanUrl(articleUrl, auto)
+
 	tmpArticle := &model.Article{}
 	_, err := MasterDB.Where("url=?", articleUrl).Get(tmpArticle)
 	if err != nil || tmpArticle.Id != 0 {
@@ -210,6 +212,22 @@ func (self ArticleLogic) ParseArticle(ctx context.Context, articleUrl string, au
 	return article, nil
 }
 
+func (ArticleLogic) cleanUrl(articleUrl string, auto bool) string {
+	pos := strings.LastIndex(articleUrl, "#")
+	if pos > 0 {
+		articleUrl = articleUrl[:pos]
+	}
+	// 过滤多余的参数，避免加一个参数就是一个新文章，但实际上是同一篇
+	if auto {
+		pos = strings.Index(articleUrl, "?")
+		if pos > 0 {
+			articleUrl = articleUrl[:pos]
+		}
+	}
+
+	return articleUrl
+}
+
 func (ArticleLogic) convertByExt(extMap map[string]string, article *model.Article) error {
 	var err error
 	if css, ok := extMap["css"]; ok {
