Thanks to visit codestin.com
Credit goes to github.com

Skip to content

基于百度接口提取标签 #74

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

基于百度接口提取标签 #74

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8aeade1
百度文章标签
May 21, 2018
380cc3a
基于百度接口提取标签
May 21, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 97 additions & 1 deletion src/model/auto_tag.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,111 @@
package model

import (
"bytes"
"encoding/json"
"io/ioutil"
"net/http"
"strings"

"github.com/axgle/mahonia"
"github.com/polaris1119/keyword"
"github.com/polaris1119/nosql"
)

type item struct {
Score float32
Tag string
}
type resData struct {
Log_id int
Items []item
}

type resTokenData struct {
Access_token string
Scope string
Session_key string
Refresh_token string
Session_secret string
Expires_in int
}

// AutoTag 自动生成 tag
func AutoTag(title, content string, num int) string {
defer func() {
recover()
}()
return strings.Join(keyword.ExtractWithTitle(title, content, num), ",")
key := "baidu_access_token"
client_id := ""
client_secret := ""

// 取百度token
redisClient := nosql.NewRedisClient()
defer redisClient.Close()
token := redisClient.GET(key)
if token == "" {
resp, err := http.Get("https://aip.baidubce.com/oauth/2.0/token?grant_type=client_credentials&client_id=" + client_id + "client_secret=" + client_secret)
if err != nil {
return strings.Join(keyword.ExtractWithTitle(title, content, num), ",")
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return strings.Join(keyword.ExtractWithTitle(title, content, num), ",")
}
var data resTokenData
err = json.Unmarshal(body, &data)
if err != nil {
return strings.Join(keyword.ExtractWithTitle(title, content, num), ",")
}
token = data.Access_token
err = redisClient.SET(key, token, data.Expires_in)
if err != nil {
return strings.Join(keyword.ExtractWithTitle(title, content, num), ",")
}
}

// 转成GBK
titleGBK := mahonia.NewEncoder("gbk").ConvertString(title)
contentGBK := mahonia.NewEncoder("gbk").ConvertString(content)
post := "{\"title\":\"" + string(titleGBK) + "\",\"content\":\"" + string(contentGBK) + "\"}"
url := "https://aip.baidubce.com/rpc/2.0/nlp/v1/keyword?access_token=" + token
jsonStr := []byte(post)
req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonStr))
req.Header.Set("Content-Type", "application/json")

client := &http.Client{}
// 请求百度接口,提取标签
resp, err := client.Do(req)
if err != nil {
return strings.Join(keyword.ExtractWithTitle(title, content, num), ",")
}
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
// 解析返回值
var data resData
err = json.Unmarshal(ConvertToByte(string(body), "gbk", "utf8"), &data)
if err != nil {
return strings.Join(keyword.ExtractWithTitle(title, content, num), ",")
}
var words []string
var length int
if len(data.Items) > num {
length = num
} else {
length = len(data.Items)
}
for i := 0; i < length; i++ {
word := data.Items[i].Tag
words = append(words, string(word))
}
return strings.Join(words, ",")
}

func ConvertToByte(src string, srcCode string, targetCode string) []byte {
srcCoder := mahonia.NewDecoder(srcCode)
srcResult := srcCoder.ConvertString(src)
tagCoder := mahonia.NewDecoder(targetCode)
_, cdata, _ := tagCoder.Translate([]byte(srcResult), true)
return cdata
}