Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Support index Read/Write to Key-Value storage #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
fogfish merged 1 commit into from
Apr 7, 2024
Merged

Support index Read/Write to Key-Value storage #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions cmd/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ module github.com/fogfish/hnsw/cmd
go 1.21.3

require (
github.com/akrylysov/pogreb v0.10.2
github.com/bits-and-blooms/bitset v1.13.0
github.com/fogfish/hnsw v0.0.0-00010101000000-000000000000
github.com/go-echarts/go-echarts/v2 v2.3.3
Expand All @@ -15,7 +16,9 @@ require github.com/fogfish/golem/pure v0.10.1 // indirect

require (
github.com/chewxy/math32 v1.10.1 // indirect
github.com/fogfish/faults v0.2.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/kelindar/binary v1.0.19 // indirect
github.com/spf13/pflag v1.0.5 // indirect
golang.org/x/sys v0.17.0 // indirect
)
Expand Down
6 changes: 6 additions & 0 deletions cmd/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,14 @@
github.com/akrylysov/pogreb v0.10.2 h1:e6PxmeyEhWyi2AKOBIJzAEi4HkiC+lKyCocRGlnDi78=
github.com/akrylysov/pogreb v0.10.2/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI=
github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE=
github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
github.com/chewxy/math32 v1.10.1 h1:LFpeY0SLJXeaiej/eIp2L40VYfscTvKh/FSEZ68uMkU=
github.com/chewxy/math32 v1.10.1/go.mod h1:dOB2rcuFrCn6UHrze36WSLVPKtzPMRAQvBvUwkSsLqs=
github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/fogfish/faults v0.2.0 h1:3KHvZN3cgv2omAGw0MCVH/AbrqxfNag+TFGpgUp6m1w=
github.com/fogfish/faults v0.2.0/go.mod h1:PtvzLt9TP4IF/hRkwRp4dZub42oaMrLbxdS6vmSCJOs=
github.com/fogfish/golem/pure v0.10.1 h1:0+cnvdaV9zF+0NN8SZMgR5bgFM6yNfBHU4rynYSDfmE=
github.com/fogfish/golem/pure v0.10.1/go.mod h1:kLPfgu5uKP0CrwVap7jejisRwV7vo1q8Eyqnc/Z0qyw=
github.com/fogfish/it v1.0.0 h1:kiwFHZcrkRLUydZoIoY0gTuMfj38trwvLo0YRyIkeG8=
Expand All @@ -14,6 +18,8 @@ github.com/go-echarts/go-echarts/v2 v2.3.3 h1:uImZAk6qLkC6F9ju6mZ5SPBqTyK8xjZKwS
github.com/go-echarts/go-echarts/v2 v2.3.3/go.mod h1:56YlvzhW/a+du15f3S2qUGNDfKnFOeJSThBIrVFHDtI=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/kelindar/binary v1.0.19 h1:DNyQCtKjkLhBh9pnP49OWREddLB0Mho+1U/AOt/Qzxw=
github.com/kelindar/binary v1.0.19/go.mod h1:/twdz8gRLNMffx0U4UOgqm1LywPs6nd9YK2TX52MDh8=
github.com/kshard/fvecs v0.0.1 h1:4FIjuJaiWWv1Q2y20w/1l13WhNlErWXs4yYVLmotNGo=
github.com/kshard/fvecs v0.0.1/go.mod h1:cehO9AfnF3Tb2vOwhOWmoaNUfYqmm4WQrUMyrPGqN6Q=
github.com/kshard/vector v0.0.3 h1:8/4t3uuv4N6gXOxgnN4RPfIel4Uq/BBSwnOjxmMWEXc=
Expand Down
17 changes: 15 additions & 2 deletions cmd/opt/create.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ import (
"fmt"
"runtime"

"github.com/akrylysov/pogreb"
"github.com/fogfish/hnsw/cmd/try"
"github.com/fogfish/hnsw/vector"
"github.com/spf13/cobra"
)

Expand Down Expand Up @@ -41,13 +41,26 @@ Creates the dataset from *.fvecs, making it reusable for other tests
}

func create(cmd *cobra.Command, args []string) error {
db, err := pogreb.Open(createOutput, nil)
if err != nil {
panic(err)
}

h := try.New(createVecSize)
if err := try.Insert(h, runtime.NumCPU(), createDataset); err != nil {
return err
}

fmt.Printf("==> writing %s\n", createOutput)
if err := vector.Write(h, createOutput); err != nil {
if err := h.Write(db); err != nil {
return err
}

if err := db.Sync(); err != nil {
return err
}

if err := db.Close(); err != nil {
return err
}

Expand Down
10 changes: 8 additions & 2 deletions cmd/opt/query.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@ import (
"os"
"time"

"github.com/akrylysov/pogreb"
"github.com/fogfish/hnsw/cmd/try"
"github.com/fogfish/hnsw/vector"
kv "github.com/fogfish/hnsw/vector"
"github.com/kshard/fvecs"
"github.com/spf13/cobra"
Expand Down Expand Up @@ -47,9 +47,14 @@ var queryCmd = &cobra.Command{
}

func query(cmd *cobra.Command, args []string) error {
db, err := pogreb.Open(queryDataset, nil)
if err != nil {
panic(err)
}

h := try.New(queryVecSize)

if err := vector.Read(h, queryDataset); err != nil {
if err := h.Read(db); err != nil {
return err
}

Expand Down Expand Up @@ -88,6 +93,7 @@ func query(cmd *cobra.Command, args []string) error {
os.Stderr.WriteString(
fmt.Sprintf("==> query %9d vectors in %s (%d ns/op)\n", c, time.Since(t), int(time.Since(t).Nanoseconds())/c),
)
db.Close()
return nil
default:
return err
Expand Down
52 changes: 0 additions & 52 deletions cmd/opt/test.go
View file
Open in desktop

This file was deleted.

4 changes: 1 addition & 3 deletions cmd/try/try.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ func Insert(h *hnsw.HNSW[kv.VF32], threads int, dataset string) error {
switch {
case err == nil:
c++
// fmt.Printf("%v\n", vec)

w <- kv.VF32{Key: c, Vector: vec}
case errors.Is(err, io.EOF):
progress()
Expand All @@ -78,8 +78,6 @@ func Insert(h *hnsw.HNSW[kv.VF32], threads int, dataset string) error {
func Query(h *hnsw.HNSW[kv.VF32], k int, query []float32, truth []uint32) (int, float64) {
result := h.Search(kv.VF32{Vector: query}, k, 100)

//fmt.Printf("%v\n", result)

errors := 0
weight := 0.0
for i, vector := range result {
Expand Down
Loading