Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit b7c9337

Browse files
polaris1119polaris1119
committed
solr 索引程序
1 parent d4dd2df commit b7c9337

File tree

5 files changed

+285
-150
lines changed

5 files changed

+285
-150
lines changed

‎websites/code2/studygolang/src/logic/searcher.go

Lines changed: 203 additions & 146 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"net/http"
1313
"net/url"
1414
"strconv"
15+
"util"
1516

1617
. "db"
1718

@@ -40,156 +41,187 @@ func (self SearcherLogic) Indexing(isAll bool) {
4041

4142
// IndexingArticle 索引博文
4243
func (self SearcherLogic) IndexingArticle(isAll bool) {
43-
// solrClient := NewSolrClient()
44-
45-
// articleObj := model.NewArticle()
46-
47-
// limit := strconv.Itoa(self.maxRows)
48-
// if isAll {
49-
// id := 0
50-
// for {
51-
// articleList, err := articleObj.Where("id>? AND status!=?", id, model.StatusOffline).Limit(limit).FindAll()
52-
// if err != nil {
53-
// logger.Errorln("IndexingArticle error:", err)
54-
// break
55-
// }
56-
57-
// if len(articleList) == 0 {
58-
// break
59-
// }
60-
61-
// for _, article := range articleList {
62-
// if id < article.Id {
63-
// id = article.Id
64-
// }
65-
66-
// document := model.NewDocument(article, nil)
67-
// addCommand := model.NewDefaultArgsAddCommand(document)
68-
69-
// solrClient.Push(addCommand)
70-
// }
71-
72-
// solrClient.Post()
73-
// }
74-
// }
44+
solrClient := NewSolrClient()
45+
46+
var (
47+
articleList []*model.Article
48+
err error
49+
)
50+
51+
if isAll {
52+
id := 0
53+
for {
54+
articleList = make([]*model.Article, 0)
55+
err = MasterDB.Where("id>?", id).Limit(self.maxRows).Find(&articleList)
56+
if err != nil {
57+
logger.Errorln("IndexingArticle error:", err)
58+
break
59+
}
60+
61+
if len(articleList) == 0 {
62+
break
63+
}
64+
65+
for _, article := range articleList {
66+
if id < article.Id {
67+
id = article.Id
68+
}
69+
70+
document := model.NewDocument(article, nil)
71+
if article.Status != model.ArticleStatusOffline {
72+
solrClient.PushAdd(model.NewDefaultArgsAddCommand(document))
73+
} else {
74+
solrClient.PushDel(model.NewDelCommand(document))
75+
}
76+
}
77+
78+
solrClient.Post()
79+
}
80+
}
7581
}
7682

77-
// 索引帖子
78-
func (SearcherLogic) IndexingTopic(isAll bool) {
79-
// solrClient := NewSolrClient()
80-
81-
// topicObj := model.NewTopic()
82-
// topicExObj := model.NewTopicEx()
83-
84-
// limit := strconv.Itoa(MaxRows)
85-
// if isAll {
86-
// id := 0
87-
// for {
88-
// topicList, err := topicObj.Where("tid>?", id).Limit(limit).FindAll()
89-
// if err != nil {
90-
// logger.Errorln("IndexingTopic error:", err)
91-
// break
92-
// }
93-
94-
// if len(topicList) == 0 {
95-
// break
96-
// }
97-
98-
// tids := util.Models2Intslice(topicList, "Tid")
99-
100-
// tmpStr := strings.Repeat("?,", len(tids))
101-
// query := "tid in(" + tmpStr[:len(tmpStr)-1] + ")"
102-
// args := make([]interface{}, len(tids))
103-
// for i, tid := range tids {
104-
// args[i] = tid
105-
// }
106-
107-
// topicExList, err := topicExObj.Where(query, args...).FindAll()
108-
// if err != nil {
109-
// logger.Errorln("IndexingTopic error:", err)
110-
// break
111-
// }
112-
113-
// topicExMap := make(map[int]*model.TopicEx, len(topicExList))
114-
// for _, topicEx := range topicExList {
115-
// topicExMap[topicEx.Tid] = topicEx
116-
// }
117-
118-
// for _, topic := range topicList {
119-
// if id < topic.Tid {
120-
// id = topic.Tid
121-
// }
122-
123-
// topicEx, _ := topicExMap[topic.Tid]
124-
125-
// document := model.NewDocument(topic, topicEx)
126-
// addCommand := model.NewDefaultArgsAddCommand(document)
127-
128-
// solrClient.Push(addCommand)
129-
// }
130-
131-
// solrClient.Post()
132-
// }
133-
// }
83+
// 索引主题
84+
func (self SearcherLogic) IndexingTopic(isAll bool) {
85+
solrClient := NewSolrClient()
86+
87+
var (
88+
topicList []*model.Topic
89+
topicExList map[int]*model.TopicEx
90+
91+
err error
92+
)
93+
94+
if isAll {
95+
id := 0
96+
for {
97+
topicList = make([]*model.Topic, 0)
98+
topicExList = make(map[int]*model.TopicEx)
99+
100+
err = MasterDB.Where("tid>?", id).Limit(self.maxRows).Find(&topicList)
101+
if err != nil {
102+
logger.Errorln("IndexingTopic error:", err)
103+
break
104+
}
105+
106+
if len(topicList) == 0 {
107+
break
108+
}
109+
110+
tids := util.Models2Intslice(topicList, "Tid")
111+
112+
err = MasterDB.In("tid", tids).Find(&topicExList)
113+
if err != nil {
114+
logger.Errorln("IndexingTopic error:", err)
115+
break
116+
}
117+
118+
for _, topic := range topicList {
119+
if id < topic.Tid {
120+
id = topic.Tid
121+
}
122+
123+
topicEx := topicExList[topic.Tid]
124+
125+
document := model.NewDocument(topic, topicEx)
126+
addCommand := model.NewDefaultArgsAddCommand(document)
127+
128+
solrClient.PushAdd(addCommand)
129+
}
130+
131+
solrClient.Post()
132+
}
133+
}
134134
}
135135

136136
// 索引资源
137-
func (SearcherLogic) IndexingResource(isAll bool) {
138-
// solrClient := NewSolrClient()
139-
140-
// resourceObj := model.NewResource()
141-
// resourceExObj := model.NewResourceEx()
142-
143-
// limit := strconv.Itoa(MaxRows)
144-
// if isAll {
145-
// id := 0
146-
// for {
147-
// resourceList, err := resourceObj.Where("id>?", id).Limit(limit).FindAll()
148-
// if err != nil {
149-
// logger.Errorln("IndexingResource error:", err)
150-
// break
151-
// }
152-
153-
// if len(resourceList) == 0 {
154-
// break
155-
// }
156-
157-
// ids := util.Models2Intslice(resourceList, "Id")
158-
159-
// tmpStr := strings.Repeat("?,", len(ids))
160-
// query := "id in(" + tmpStr[:len(tmpStr)-1] + ")"
161-
// args := make([]interface{}, len(ids))
162-
// for i, rid := range ids {
163-
// args[i] = rid
164-
// }
165-
166-
// resourceExList, err := resourceExObj.Where(query, args...).FindAll()
167-
// if err != nil {
168-
// logger.Errorln("IndexingResource error:", err)
169-
// break
170-
// }
171-
172-
// resourceExMap := make(map[int]*model.ResourceEx, len(resourceExList))
173-
// for _, resourceEx := range resourceExList {
174-
// resourceExMap[resourceEx.Id] = resourceEx
175-
// }
176-
177-
// for _, resource := range resourceList {
178-
// if id < resource.Id {
179-
// id = resource.Id
180-
// }
181-
182-
// resourceEx, _ := resourceExMap[resource.Id]
183-
184-
// document := model.NewDocument(resource, resourceEx)
185-
// addCommand := model.NewDefaultArgsAddCommand(document)
186-
187-
// solrClient.Push(addCommand)
188-
// }
189-
190-
// solrClient.Post()
191-
// }
192-
// }
137+
func (self SearcherLogic) IndexingResource(isAll bool) {
138+
solrClient := NewSolrClient()
139+
140+
var (
141+
resourceList []*model.Resource
142+
resourceExList map[int]*model.ResourceEx
143+
err error
144+
)
145+
146+
if isAll {
147+
id := 0
148+
for {
149+
resourceList = make([]*model.Resource, 0)
150+
err = MasterDB.Where("id>?", id).Limit(self.maxRows).Find(&resourceList)
151+
if err != nil {
152+
logger.Errorln("IndexingResource error:", err)
153+
break
154+
}
155+
156+
if len(resourceList) == 0 {
157+
break
158+
}
159+
160+
ids := util.Models2Intslice(resourceList, "Id")
161+
162+
err = MasterDB.In("id", ids).Find(&resourceExList)
163+
if err != nil {
164+
logger.Errorln("IndexingResource error:", err)
165+
break
166+
}
167+
168+
for _, resource := range resourceList {
169+
if id < resource.Id {
170+
id = resource.Id
171+
}
172+
173+
resourceEx := resourceExList[resource.Id]
174+
175+
document := model.NewDocument(resource, resourceEx)
176+
addCommand := model.NewDefaultArgsAddCommand(document)
177+
178+
solrClient.PushAdd(addCommand)
179+
}
180+
181+
solrClient.Post()
182+
}
183+
}
184+
}
185+
186+
// IndexingOpenProject 索引博文
187+
func (self SearcherLogic) IndexingOpenProject(isAll bool) {
188+
solrClient := NewSolrClient()
189+
190+
var (
191+
projectList []*model.OpenProject
192+
err error
193+
)
194+
195+
if isAll {
196+
id := 0
197+
for {
198+
projectList = make([]*model.OpenProject, 0)
199+
err = MasterDB.Where("id>?", id).Limit(self.maxRows).Find(&projectList)
200+
if err != nil {
201+
logger.Errorln("IndexingArticle error:", err)
202+
break
203+
}
204+
205+
if len(projectList) == 0 {
206+
break
207+
}
208+
209+
for _, project := range projectList {
210+
if id < project.Id {
211+
id = project.Id
212+
}
213+
214+
document := model.NewDocument(project, nil)
215+
if project.Status != model.ProjectStatusOffline {
216+
solrClient.PushAdd(model.NewDefaultArgsAddCommand(document))
217+
} else {
218+
solrClient.PushDel(model.NewDelCommand(document))
219+
}
220+
}
221+
222+
solrClient.Post()
223+
}
224+
}
193225
}
194226

195227
const searchContentLen = 350
@@ -304,18 +336,24 @@ func (self SearcherLogic) DoSearch(q, field string, start, rows int) (*model.Res
304336

305337
type SolrClient struct {
306338
addCommands []*model.AddCommand
339+
delCommands []*model.DelCommand
307340
}
308341

309342
func NewSolrClient() *SolrClient {
310343
return &SolrClient{
311344
addCommands: make([]*model.AddCommand, 0, 100),
345+
delCommands: make([]*model.DelCommand, 0, 100),
312346
}
313347
}
314348

315-
func (this *SolrClient) Push(addCommand *model.AddCommand) {
349+
func (this *SolrClient) PushAdd(addCommand *model.AddCommand) {
316350
this.addCommands = append(this.addCommands, addCommand)
317351
}
318352

353+
func (this *SolrClient) PushDel(delCommand *model.DelCommand) {
354+
this.delCommands = append(this.delCommands, delCommand)
355+
}
356+
319357
func (this *SolrClient) Post() error {
320358
stringBuilder := goutils.NewBuffer().Append("{")
321359

@@ -339,6 +377,25 @@ func (this *SolrClient) Post() error {
339377
stringBuilder.Append(`"add":`).Append(commandJson)
340378
}
341379

380+
for _, delCommand := range this.delCommands {
381+
commandJson, err := json.Marshal(delCommand)
382+
if err != nil {
383+
continue
384+
}
385+
386+
if stringBuilder.Len() == 1 {
387+
needComma = false
388+
} else {
389+
needComma = true
390+
}
391+
392+
if needComma {
393+
stringBuilder.Append(",")
394+
}
395+
396+
stringBuilder.Append(`"del":`).Append(commandJson)
397+
}
398+
342399
if stringBuilder.Len() == 1 {
343400
logger.Errorln("post docs:no right addcommand")
344401
return errors.New("no right addcommand")

0 commit comments

Comments
 (0)