Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 9082878

Browse files
zhima1234zhima1234
committed
增量更新
1 parent 792f5e0 commit 9082878

File tree

3 files changed

+164
-129
lines changed

3 files changed

+164
-129
lines changed

‎config/db.sql

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -321,7 +321,7 @@ CREATE TABLE IF NOT EXISTS `articles` (
321321
KEY (`top`),
322322
KEY (`author_txt`),
323323
KEY (`domain`),
324-
KEY (`ctime`)
324+
KEY (`mtime`)
325325
)ENGINE=InnoDB DEFAULT CHARSET=utf8 COMMENT '网络文章聚合表';
326326

327327
CREATE TABLE IF NOT EXISTS `crawl_rule` (

‎src/logic/searcher.go

Lines changed: 159 additions & 128 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"net/http"
1414
"net/url"
1515
"strconv"
16+
"time"
1617
"util"
1718

1819
. "db"
@@ -51,44 +52,51 @@ func (self SearcherLogic) IndexingArticle(isAll bool) {
5152
err error
5253
)
5354

54-
if isAll {
55-
id := 0
56-
for {
57-
articleList = make([]*model.Article, 0)
55+
id := 0
56+
for {
57+
articleList = make([]*model.Article, 0)
58+
if isAll {
5859
err = MasterDB.Where("id>?", id).Limit(self.maxRows).OrderBy("id ASC").Find(&articleList)
59-
if err != nil {
60-
logger.Errorln("IndexingArticle error:", err)
61-
break
62-
}
60+
} else {
61+
timeAgo := time.Now().Add(-5 * time.Minute).Format("2006-01-02 15:04:05")
62+
err = MasterDB.Where("mtime>?", timeAgo).Find(&articleList)
63+
}
64+
if err != nil {
65+
logger.Errorln("IndexingArticle error:", err)
66+
break
67+
}
6368

64-
if len(articleList) == 0 {
65-
break
66-
}
69+
if len(articleList) == 0 {
70+
break
71+
}
6772

68-
for _, article := range articleList {
69-
logger.Infoln("deal article_id:", article.Id)
73+
for _, article := range articleList {
74+
logger.Infoln("deal article_id:", article.Id)
7075

71-
if id < article.Id {
72-
id = article.Id
73-
}
76+
if id < article.Id {
77+
id = article.Id
78+
}
7479

75-
if article.Tags == "" {
76-
// 自动生成
77-
article.Tags = model.AutoTag(article.Title, article.Txt, 4)
78-
if article.Tags != "" {
79-
MasterDB.Id(article.Id).Cols("tags").Update(article)
80-
}
80+
if article.Tags == "" {
81+
// 自动生成
82+
article.Tags = model.AutoTag(article.Title, article.Txt, 4)
83+
if article.Tags != "" {
84+
MasterDB.Id(article.Id).Cols("tags").Update(article)
8185
}
86+
}
8287

83-
document := model.NewDocument(article, nil)
84-
if article.Status != model.ArticleStatusOffline {
85-
solrClient.PushAdd(model.NewDefaultArgsAddCommand(document))
86-
} else {
87-
solrClient.PushDel(model.NewDelCommand(document))
88-
}
88+
document := model.NewDocument(article, nil)
89+
if article.Status != model.ArticleStatusOffline {
90+
solrClient.PushAdd(model.NewDefaultArgsAddCommand(document))
91+
} else {
92+
solrClient.PushDel(model.NewDelCommand(document))
8993
}
94+
}
9095

91-
solrClient.Post()
96+
solrClient.Post()
97+
98+
if !isAll {
99+
break
92100
}
93101
}
94102
}
@@ -104,54 +112,61 @@ func (self SearcherLogic) IndexingTopic(isAll bool) {
104112
err error
105113
)
106114

107-
if isAll {
108-
id := 0
109-
for {
110-
topicList = make([]*model.Topic, 0)
111-
topicExList = make(map[int]*model.TopicUpEx)
115+
id := 0
116+
for {
117+
topicList = make([]*model.Topic, 0)
118+
topicExList = make(map[int]*model.TopicUpEx)
112119

120+
if isAll {
113121
err = MasterDB.Where("tid>?", id).OrderBy("tid ASC").Limit(self.maxRows).Find(&topicList)
114-
if err != nil {
115-
logger.Errorln("IndexingTopic error:", err)
116-
break
117-
}
122+
} else {
123+
timeAgo := time.Now().Add(-5 * time.Minute).Format("2006-01-02 15:04:05")
124+
err = MasterDB.Where("mtime>?", timeAgo).Find(&topicList)
125+
}
126+
if err != nil {
127+
logger.Errorln("IndexingTopic error:", err)
128+
break
129+
}
118130

119-
if len(topicList) == 0 {
120-
break
121-
}
131+
if len(topicList) == 0 {
132+
break
133+
}
122134

123-
tids := util.Models2Intslice(topicList, "Tid")
135+
tids := util.Models2Intslice(topicList, "Tid")
124136

125-
err = MasterDB.In("tid", tids).Find(&topicExList)
126-
if err != nil {
127-
logger.Errorln("IndexingTopic error:", err)
128-
break
129-
}
137+
err = MasterDB.In("tid", tids).Find(&topicExList)
138+
if err != nil {
139+
logger.Errorln("IndexingTopic error:", err)
140+
break
141+
}
130142

131-
for _, topic := range topicList {
132-
logger.Infoln("deal topic_id:", topic.Tid)
143+
for _, topic := range topicList {
144+
logger.Infoln("deal topic_id:", topic.Tid)
133145

134-
if id < topic.Tid {
135-
id = topic.Tid
136-
}
146+
if id < topic.Tid {
147+
id = topic.Tid
148+
}
137149

138-
if topic.Tags == "" {
139-
// 自动生成
140-
topic.Tags = model.AutoTag(topic.Title, topic.Content, 4)
141-
if topic.Tags != "" {
142-
MasterDB.Id(topic.Tid).Cols("tags").Update(topic)
143-
}
150+
if topic.Tags == "" {
151+
// 自动生成
152+
topic.Tags = model.AutoTag(topic.Title, topic.Content, 4)
153+
if topic.Tags != "" {
154+
MasterDB.Id(topic.Tid).Cols("tags").Update(topic)
144155
}
156+
}
145157

146-
topicEx := topicExList[topic.Tid]
158+
topicEx := topicExList[topic.Tid]
147159

148-
document := model.NewDocument(topic, topicEx)
149-
addCommand := model.NewDefaultArgsAddCommand(document)
160+
document := model.NewDocument(topic, topicEx)
161+
addCommand := model.NewDefaultArgsAddCommand(document)
150162

151-
solrClient.PushAdd(addCommand)
152-
}
163+
solrClient.PushAdd(addCommand)
164+
}
153165

154-
solrClient.Post()
166+
solrClient.Post()
167+
168+
if !isAll {
169+
break
155170
}
156171
}
157172
}
@@ -166,54 +181,61 @@ func (self SearcherLogic) IndexingResource(isAll bool) {
166181
err error
167182
)
168183

169-
if isAll {
170-
id := 0
171-
for {
172-
resourceList = make([]*model.Resource, 0)
173-
resourceExList = make(map[int]*model.ResourceEx)
184+
id := 0
185+
for {
186+
resourceList = make([]*model.Resource, 0)
187+
resourceExList = make(map[int]*model.ResourceEx)
174188

189+
if isAll {
175190
err = MasterDB.Where("id>?", id).OrderBy("id ASC").Limit(self.maxRows).Find(&resourceList)
176-
if err != nil {
177-
logger.Errorln("IndexingResource error:", err)
178-
break
179-
}
191+
} else {
192+
timeAgo := time.Now().Add(-5 * time.Minute).Format("2006-01-02 15:04:05")
193+
err = MasterDB.Where("mtime>?", timeAgo).Find(&resourceList)
194+
}
195+
if err != nil {
196+
logger.Errorln("IndexingResource error:", err)
197+
break
198+
}
180199

181-
if len(resourceList) == 0 {
182-
break
183-
}
200+
if len(resourceList) == 0 {
201+
break
202+
}
184203

185-
ids := util.Models2Intslice(resourceList, "Id")
204+
ids := util.Models2Intslice(resourceList, "Id")
186205

187-
err = MasterDB.In("id", ids).Find(&resourceExList)
188-
if err != nil {
189-
logger.Errorln("IndexingResource error:", err)
190-
break
191-
}
206+
err = MasterDB.In("id", ids).Find(&resourceExList)
207+
if err != nil {
208+
logger.Errorln("IndexingResource error:", err)
209+
break
210+
}
192211

193-
for _, resource := range resourceList {
194-
logger.Infoln("deal resource_id:", resource.Id)
212+
for _, resource := range resourceList {
213+
logger.Infoln("deal resource_id:", resource.Id)
195214

196-
if id < resource.Id {
197-
id = resource.Id
198-
}
215+
if id < resource.Id {
216+
id = resource.Id
217+
}
199218

200-
if resource.Tags == "" {
201-
// 自动生成
202-
resource.Tags = model.AutoTag(resource.Title+resource.CatName, resource.Content, 4)
203-
if resource.Tags != "" {
204-
MasterDB.Id(resource.Id).Cols("tags").Update(resource)
205-
}
219+
if resource.Tags == "" {
220+
// 自动生成
221+
resource.Tags = model.AutoTag(resource.Title+resource.CatName, resource.Content, 4)
222+
if resource.Tags != "" {
223+
MasterDB.Id(resource.Id).Cols("tags").Update(resource)
206224
}
225+
}
207226

208-
resourceEx := resourceExList[resource.Id]
227+
resourceEx := resourceExList[resource.Id]
209228

210-
document := model.NewDocument(resource, resourceEx)
211-
addCommand := model.NewDefaultArgsAddCommand(document)
229+
document := model.NewDocument(resource, resourceEx)
230+
addCommand := model.NewDefaultArgsAddCommand(document)
212231

213-
solrClient.PushAdd(addCommand)
214-
}
232+
solrClient.PushAdd(addCommand)
233+
}
234+
235+
solrClient.Post()
215236

216-
solrClient.Post()
237+
if !isAll {
238+
break
217239
}
218240
}
219241
}
@@ -227,46 +249,55 @@ func (self SearcherLogic) IndexingOpenProject(isAll bool) {
227249
err error
228250
)
229251

230-
if isAll {
231-
id := 0
232-
for {
233-
projectList = make([]*model.OpenProject, 0)
252+
id := 0
253+
for {
254+
projectList = make([]*model.OpenProject, 0)
255+
256+
if isAll {
234257
err = MasterDB.Where("id>?", id).OrderBy("id ASC").Limit(self.maxRows).Find(&projectList)
235-
if err != nil {
236-
logger.Errorln("IndexingArticle error:", err)
237-
break
238-
}
258+
} else {
259+
timeAgo := time.Now().Add(-5 * time.Minute).Format("2006-01-02 15:04:05")
260+
err = MasterDB.Where("mtime>?", timeAgo).Find(&projectList)
261+
}
262+
if err != nil {
263+
logger.Errorln("IndexingArticle error:", err)
264+
break
265+
}
239266

240-
if len(projectList) == 0 {
241-
break
242-
}
267+
if len(projectList) == 0 {
268+
break
269+
}
243270

244-
for _, project := range projectList {
245-
logger.Infoln("deal project_id:", project.Id)
271+
for _, project := range projectList {
272+
logger.Infoln("deal project_id:", project.Id)
246273

247-
if id < project.Id {
248-
id = project.Id
249-
}
274+
if id < project.Id {
275+
id = project.Id
276+
}
250277

251-
if project.Tags == "" {
252-
// 自动生成
253-
project.Tags = model.AutoTag(project.Name+project.Category, project.Desc, 4)
254-
if project.Tags != "" {
255-
MasterDB.Id(project.Id).Cols("tags").Update(project)
256-
}
278+
if project.Tags == "" {
279+
// 自动生成
280+
project.Tags = model.AutoTag(project.Name+project.Category, project.Desc, 4)
281+
if project.Tags != "" {
282+
MasterDB.Id(project.Id).Cols("tags").Update(project)
257283
}
284+
}
258285

259-
document := model.NewDocument(project, nil)
260-
if project.Status != model.ProjectStatusOffline {
261-
solrClient.PushAdd(model.NewDefaultArgsAddCommand(document))
262-
} else {
263-
solrClient.PushDel(model.NewDelCommand(document))
264-
}
286+
document := model.NewDocument(project, nil)
287+
if project.Status != model.ProjectStatusOffline {
288+
solrClient.PushAdd(model.NewDefaultArgsAddCommand(document))
289+
} else {
290+
solrClient.PushDel(model.NewDelCommand(document))
265291
}
292+
}
266293

267-
solrClient.Post()
294+
solrClient.Post()
295+
296+
if !isAll {
297+
break
268298
}
269299
}
300+
270301
}
271302

272303
const searchContentLen = 350

‎src/server/server.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,10 @@ func IndexingServer() {
3636

3737
c := cron.New()
3838
// 构建 solr 需要的索引数据
39+
// 1 分钟一次增量
40+
c.AddFunc("@every 1m", func() {
41+
indexing(false)
42+
})
3943
// 一天一次全量
4044
c.AddFunc("@daily", func() {
4145
indexing(true)

0 commit comments

Comments
 (0)