fogfish · fogfish · Mar 3, 2024 · Feb 29, 2024 · Mar 1, 2024 · Mar 3, 2024
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -0,0 +1,48 @@
+##
+## Build the main branch
+##
+name: build
+on:
+  push:
+    branches:
+      - main
+      - /refs/heads/main
+
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+    steps:
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version: "1.21"
+
+      - uses: actions/[email protected]
+
+      - name: go build
+        run: |
+          go build ./...
+
+      - name: go test
+        run: |
+          go test -v -coverprofile=profile.cov $(go list ./... | grep -v /examples/)
+
+      - uses: shogo82148/actions-goveralls@v1
+        continue-on-error: true
+        with:
+          path-to-profile: profile.cov
+
+      - uses: reecetech/[email protected]
+        id: version
+        with:
+          scheme: semver
+          increment: patch
+
+      - name: publish
+        run: |
+          git config user.name "GitHub Actions"
+          git config user.email "[email protected]"
+          git tag ${{ steps.version.outputs.v-version }}
+          git push origin -u ${{ steps.version.outputs.v-version }}
+
diff --git a/.github/workflows/check-code.yml b/.github/workflows/check-code.yml
@@ -0,0 +1,25 @@
+##
+## Quality checks
+##
+name: check
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+
+jobs:
+
+  code:
+    runs-on: ubuntu-latest
+    steps:
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version: "1.21"
+
+      - uses: actions/[email protected]
+
+      - uses: dominikh/[email protected]
+        with:
+          install-go: false
diff --git a/.github/workflows/check-test.yml b/.github/workflows/check-test.yml
@@ -0,0 +1,34 @@
+##
+## Unit Tests
+##
+name: test
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+
+jobs:
+
+  unit:
+    runs-on: ubuntu-latest
+    steps:
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version: "1.21"
+
+      - uses: actions/[email protected]
+
+      - name: go build
+        run: |
+          go build ./...
+
+      - name: go test
+        run: |
+          go test -v -coverprofile=profile.cov $(go list ./... | grep -v /examples/)
+
+      - uses: shogo82148/actions-goveralls@v1
+        continue-on-error: true
+        with:
+          path-to-profile: profile.cov
diff --git a/.gitignore b/.gitignore
@@ -1,5 +1,9 @@
 go.work
 go.work.sum
 cmd/sift*
+
 *.html
+*.fvecs
+*.ivecs
+*.bvecs
 
diff --git a/cmd/go.mod b/cmd/go.mod
@@ -3,17 +3,21 @@ module github.com/fogfish/hnsw/cmd
 go 1.21.3
 
 require (
+	github.com/bits-and-blooms/bitset v1.13.0
 	github.com/fogfish/hnsw v0.0.0-00010101000000-000000000000
 	github.com/go-echarts/go-echarts/v2 v2.3.3
+	github.com/kshard/fvecs v0.0.1
+	github.com/kshard/vector v0.0.3
 	github.com/spf13/cobra v1.8.0
 )
 
 require github.com/fogfish/golem/pure v0.10.1 // indirect
 
 require (
+	github.com/chewxy/math32 v1.10.1 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/willf/bitset v1.1.11
+	golang.org/x/sys v0.17.0 // indirect
 )
 
 replace github.com/fogfish/hnsw => ../
diff --git a/cmd/go.sum b/cmd/go.sum
@@ -1,3 +1,7 @@
+github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE=
+github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
+github.com/chewxy/math32 v1.10.1 h1:LFpeY0SLJXeaiej/eIp2L40VYfscTvKh/FSEZ68uMkU=
+github.com/chewxy/math32 v1.10.1/go.mod h1:dOB2rcuFrCn6UHrze36WSLVPKtzPMRAQvBvUwkSsLqs=
 github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -10,6 +14,10 @@ github.com/go-echarts/go-echarts/v2 v2.3.3 h1:uImZAk6qLkC6F9ju6mZ5SPBqTyK8xjZKwS
 github.com/go-echarts/go-echarts/v2 v2.3.3/go.mod h1:56YlvzhW/a+du15f3S2qUGNDfKnFOeJSThBIrVFHDtI=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/kshard/fvecs v0.0.1 h1:4FIjuJaiWWv1Q2y20w/1l13WhNlErWXs4yYVLmotNGo=
+github.com/kshard/fvecs v0.0.1/go.mod h1:cehO9AfnF3Tb2vOwhOWmoaNUfYqmm4WQrUMyrPGqN6Q=
+github.com/kshard/vector v0.0.3 h1:8/4t3uuv4N6gXOxgnN4RPfIel4Uq/BBSwnOjxmMWEXc=
+github.com/kshard/vector v0.0.3/go.mod h1:l5c902GqrnE4/LvJAmSCbtGorXeMe9si5SHtwckr8jc=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
@@ -19,8 +27,8 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/testify v1.6.0 h1:jlIyCplCJFULU/01vCkhKuTyc3OorI3bJFuw6obfgho=
 github.com/stretchr/testify v1.6.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/willf/bitset v1.1.11 h1:N7Z7E9UvjW+sGsEl7k/SJrvY2reP1A07MrGuCjIOjRE=
-github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI=
+golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
diff --git a/cmd/opt/create.go b/cmd/opt/create.go
@@ -0,0 +1,55 @@
+//
+// Copyright (C) 2024 Dmitry Kolesnikov
+//
+// This file may be modified and distributed under the terms
+// of the MIT license.  See the LICENSE file for details.
+// https://github.com/fogfish/hnsw
+//
+
+package opt
+
+import (
+	"fmt"
+	"runtime"
+
+	"github.com/fogfish/hnsw/cmd/try"
+	"github.com/fogfish/hnsw/vector"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	rootCmd.AddCommand(createCmd)
+	createCmd.Flags().StringVarP(&createDataset, "dataset", "d", "", "path to input *.fvecs files")
+	createCmd.Flags().StringVarP(&createOutput, "output", "o", "test", "output")
+	createCmd.Flags().IntVarP(&createVecSize, "vector", "v", 128, "vector size")
+}
+
+var (
+	createDataset string
+	createOutput  string
+	createVecSize int
+)
+
+var createCmd = &cobra.Command{
+	Use:   "create",
+	Short: "create the dataset",
+	Long: `
+Creates the dataset from *.fvecs, making it reusable for other tests
+`,
+	SilenceUsage: true,
+	RunE:         create,
+}
+
+func create(cmd *cobra.Command, args []string) error {
+	h := try.New(createVecSize)
+	if err := try.Insert(h, runtime.NumCPU(), createDataset); err != nil {
+		return err
+	}
+
+	fmt.Printf("==> writing %s\n", createOutput)
+	if err := vector.Write(h, createOutput); err != nil {
+		return err
+	}
+
+	return nil
+}
diff --git a/cmd/opt/draw.go b/cmd/opt/draw.go
@@ -14,13 +14,14 @@ import (
 	"os"
 	"strconv"
 
+	"github.com/bits-and-blooms/bitset"
 	"github.com/fogfish/hnsw"
 	"github.com/fogfish/hnsw/cmd/try"
+	kv "github.com/fogfish/hnsw/vector"
 	"github.com/go-echarts/go-echarts/v2/charts"
 	"github.com/go-echarts/go-echarts/v2/components"
 	"github.com/go-echarts/go-echarts/v2/opts"
 	"github.com/spf13/cobra"
-	"github.com/willf/bitset"
 )
 
 func init() {
@@ -51,9 +52,9 @@ It is required to obtain the dataset(s) into local environment:
 }
 
 func draw(cmd *cobra.Command, args []string) error {
-	h := try.New()
+	h := try.New(128)
 
-	if err := try.Create(h, drawDataset); err != nil {
+	if err := try.Insert(h, 8, drawDataset); err != nil {
 		return err
 	}
 
@@ -68,7 +69,7 @@ func draw(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
-func drawLevel(h *hnsw.HNSW[try.Node], level int) error {
+func drawLevel(h *hnsw.HNSW[kv.VF32], level int) error {
 	nodes, links, kinds := cutLevel(h, level)
 	if len(nodes) == 0 || len(links) == 0 {
 		return nil
@@ -80,12 +81,11 @@ func drawLevel(h *hnsw.HNSW[try.Node], level int) error {
 		SetSeriesOptions(
 			charts.WithGraphChartOpts(opts.GraphChart{
 				Layout:             "force",
-				Draggable:          true,
 				Roam:               true,
 				FocusNodeAdjacency: true,
 				Force: &opts.GraphForce{
-					Repulsion:  200.0, //800.0,
-					Gravity:    0.05,  //0.01,
+					Repulsion:  800.0,
+					Gravity:    0.05,
 					EdgeLength: 60.0,
 				},
 
@@ -100,6 +100,7 @@ func drawLevel(h *hnsw.HNSW[try.Node], level int) error {
 			}),
 			charts.WithLineStyleOpts(opts.LineStyle{
 				Curveness: 0.3,
+				Color:     "source",
 			}),
 		)
 
@@ -117,35 +118,37 @@ func drawLevel(h *hnsw.HNSW[try.Node], level int) error {
 	return page.Render(io.MultiWriter(f))
 }
 
-func cutLevel(h *hnsw.HNSW[try.Node], level int) ([]opts.GraphNode, []opts.GraphLink, []*opts.GraphCategory) {
+func cutLevel(h *hnsw.HNSW[kv.VF32], level int) ([]opts.GraphNode, []opts.GraphLink, []*opts.GraphCategory) {
 	var visited bitset.BitSet
+
 	mrank := level
 	nodes := []opts.GraphNode{}
 	links := []opts.GraphLink{}
 	kinds := []*opts.GraphCategory{}
 
-	h.FMap(level, func(rank int, vector try.Node, vertex []try.Node) error {
-		if visited.Test(uint(vector.ID)) {
+	h.FMap(level, func(rank int, vector kv.VF32, vertex []kv.VF32) error {
+		if visited.Test(uint(vector.Key)) {
 			return nil
 		}
-		visited.Set(uint(vector.ID))
+		visited.Set(uint(vector.Key))
 
 		if rank > mrank {
 			mrank = rank
 		}
 
 		nodes = append(nodes,
 			opts.GraphNode{
-				Name:     strconv.Itoa(vector.ID),
+				Name:     strconv.Itoa(int(vector.Key)),
 				Category: rank - level - 1,
 			},
 		)
 
 		for _, v := range vertex {
 			links = append(links,
 				opts.GraphLink{
-					Source: strconv.Itoa(vector.ID),
-					Target: strconv.Itoa(v.ID),
+					Source: strconv.Itoa(int(vector.Key)),
+					Target: strconv.Itoa(int(v.Key)),
+					Value:  200.0 * h.Distance(vector, v),
 				},
 			)
 		}