支持抓取 go 官方下载包信息

polaris1119 · polaris1119 · commit e2f8c49eed49 · 2019-02-17T23:45:19.000+08:00
diff --git a/src/http/controller/download.go b/src/http/controller/download.go
@@ -27,6 +27,7 @@ type DownloadController struct{}
 func (self DownloadController) RegisterRoute(g *echo.Group) {
 	g.Get("/dl", self.GoDl)
 	g.Get("/dl/golang/:filename", self.FetchGoInstallPackage)
+	g.Get("/dl/add_new_version", self.AddNewDownload)
 }
 
 // GoDl Go 语言安装包下载
@@ -117,6 +118,26 @@ func (self DownloadController) FetchGoInstallPackage(ctx echo.Context) error {
 	return ctx.Redirect(http.StatusSeeOther, "/static/"+filePath)
 }
 
+func (DownloadController) AddNewDownload(ctx echo.Context) error {
+	version := ctx.QueryParam("version")
+	selector := ctx.QueryParam("selector")
+
+	if version == "" {
+		return fail(ctx, 1, "version is empty")
+	}
+
+	if selector == "" {
+		selector = "toggleVisible"
+	}
+
+	err := logic.DefaultDownload.AddNewDownload(ctx, version, selector)
+	if err != nil {
+		return fail(ctx, 1, err.Error())
+	}
+
+	return success(ctx, nil)
+}
+
 func (DownloadController) headWithTimeout(dlUrl string) (*http.Response, error) {
 	client := http.Client{
 		Timeout: 2 * time.Second,
diff --git a/src/logic/download.go b/src/logic/download.go
@@ -7,12 +7,17 @@
 package logic
 
 import (
+	"net/http"
+	"strings"
+
 	"model"
 
 	"golang.org/x/net/context"
 
 	. "db"
 
+	"github.com/PuerkitoBio/goquery"
+	"github.com/polaris1119/goutils"
 	"github.com/polaris1119/logger"
 )
 
@@ -35,3 +40,71 @@ func (DownloadLogic) RecordDLTimes(ctx context.Context, filename string) error {
 
 	return nil
 }
+
+func (DownloadLogic) AddNewDownload(ctx context.Context, version, selector string) error {
+	objLog := GetLogger(ctx)
+
+	resp, err := http.Get("https://golang.google.cn/dl/")
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	doc, err := goquery.NewDocumentFromResponse(resp)
+	if err != nil {
+		return err
+	}
+
+	doc.Find(selector).Each(func(i int, versionSel *goquery.Selection) {
+		idVal, exists := versionSel.Attr("id")
+		if !exists {
+			return
+		}
+
+		if idVal != version {
+			return
+		}
+
+		versionSel.Find("table tbody tr").Each(func(j int, dlSel *goquery.Selection) {
+			download := &model.Download{
+				Version: version,
+			}
+
+			if dlSel.HasClass("highlight") {
+				download.IsRecommend = true
+			}
+
+			dlSel.Find("td").Each(func(k int, fieldSel *goquery.Selection) {
+				val := fieldSel.Text()
+				switch k {
+				case 0:
+					download.Filename = val
+				case 1:
+					download.Kind = val
+				case 2:
+					download.OS = val
+				case 3:
+					download.Arch = val
+				case 4:
+					download.Size = goutils.MustInt(strings.TrimRight(val, "MB"))
+				case 5:
+					download.Checksum = val
+				}
+			})
+
+			has, err := MasterDB.Where("filename=?", download.Filename).Exist(new(model.Download))
+			if err != nil || has {
+				return
+			}
+
+			_, err = MasterDB.Insert(download)
+			if err != nil {
+				objLog.Errorln("insert download error:", err, "version:", version)
+			}
+		})
+
+		MasterDB.Exec("UPDATE download SET seq=id WHERE seq=0")
+	})
+
+	return nil
+}
