Thanks to visit codestin.com
Credit goes to github.com

Skip to content

fix(metadata)!: remove CreatedAt from Metadata #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
terwey merged 1 commit into from
Oct 20, 2025
Merged

fix(metadata)!: remove CreatedAt from Metadata #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion engine/engine.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,6 @@ func (e *Engine) processDeal(ctx context.Context, wi WorkKey, deal *tc.Deal, eve
md := metadata.Metadata{
BotID: wi.BotID,
DealID: wi.DealID,
CreatedAt: *event.CreatedAt,
BotEventID: event.FingerprintAsID(),
}
// we want to store all incoming as a log
Expand Down
7 changes: 1 addition & 6 deletions internal/api/handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -229,17 +229,12 @@ func (h *ApiHandler) orderRecordFromItem(ctx context.Context, row OrderItem, inc
}
}

createdAt := row.Metadata.CreatedAt
if row.BotEvent != nil && row.BotEvent.CreatedAt != nil {
createdAt = *row.BotEvent.CreatedAt
}

record := OrderRecord{
MetadataHex: row.Metadata.Hex(),
BotId: int64(row.Metadata.BotID),
DealId: int64(row.Metadata.DealID),
BotEventId: int64(row.Metadata.BotEventID),
CreatedAt: createdAt,
CreatedAt: *row.BotEvent.CreatedAt,
ObservedAt: row.ObservedAt,
BotEventPayload: eventPayload,
LatestSubmission: latestSubmission,
Expand Down
1 change: 0 additions & 1 deletion internal/testutil/testutil.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@ func NewBotEvent(t *testing.T, base time.Time, botID uint32,
md := metadata.Metadata{
BotID: uint32(botID),
DealID: uint32(dealID),
CreatedAt: *evt.CreatedAt,
BotEventID: evt.FingerprintAsID(),
}
return evt, md
Expand Down
43 changes: 8 additions & 35 deletions metadata/md.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,23 +4,19 @@ import (
"encoding/binary"
"encoding/hex"
"fmt"
"hash/crc32"
"log"
"strings"
"time"

"github.com/howeyc/crc16"
)

// Metadata is to track an order across different states
type Metadata struct {
CreatedAt time.Time
BotID uint32
DealID uint32
BotEventID uint32
}

func (md *Metadata) String() string {
// return fmt.Sprintf("%s::%d::%d::%s", md.CreatedAt.Format("2006-01-02"), md.BotID, md.DealID, md.OrderID)
// return fmt.Sprintf("%s::%s", md.CreatedAt.Format("2006-01-02"), md.Hex())
return md.Hex()
}

Expand All @@ -35,39 +31,19 @@ func (md *Metadata) Hex() string {

// AsHex returns a 16 byte representation of the metadata
// All are BigEndian encoded
// 2 bytes are the days since epoch uint16
// 4 bytes are the BotID uint32
// 4 bytes are the DealID uint32
// 4 bytes are the OrderID uint32
// 2 bytes are a CRC16 of the preceding bytes
// 4 bytes are a CRC32 of the preceding bytes
// The time is stored with UTC
func (md *Metadata) AsHex() []byte {
out := make([]byte, 0, 16)

// unix returns number of seconds since epoch
// we can divide this by the amount of seconds in a day
// this should fit in an uint16
d := md.CreatedAt.UTC().Unix() / 86400
// log.Printf("days since epoch: %d", d)
out = binary.BigEndian.AppendUint16(out, uint16(d))

out = binary.BigEndian.AppendUint32(out, uint32(md.BotID))
out = binary.BigEndian.AppendUint32(out, uint32(md.DealID))

// var oid uint64
// var err error
// if md.OrderID != "" {
// // we know OrderID is actually an uint32 too
// oid, err = strconv.ParseUint(md.OrderID, 10, 32)
// if err != nil {
// panic("orderid cannot be parsed as uint32")
// }
// }

out = binary.BigEndian.AppendUint32(out, uint32(md.BotEventID))
// out = binary.BigEndian.AppendUint32(out, uint32(oid))

out = binary.BigEndian.AppendUint16(out, crc16.Checksum(out, crc16.IBMTable))
out = binary.BigEndian.AppendUint32(out, crc32.Checksum(out, crc32.IEEETable))

return out
}
Expand All @@ -83,17 +59,14 @@ func FromHex(v []byte) (*Metadata, error) {
return nil, ErrHexTooShort
}

if crc16.Checksum(v[0:14], crc16.IBMTable) != binary.BigEndian.Uint16(v[14:16]) {
if crc32.Checksum(v[0:12], crc32.IEEETable) != binary.BigEndian.Uint32(v[12:16]) {
return nil, ErrIncorrectChecksum
}

md := &Metadata{}
days := binary.BigEndian.Uint16(v[0:2])
md.CreatedAt = time.Unix(int64(days)*86400, 0).UTC()

md.BotID = uint32(binary.BigEndian.Uint32(v[2:6]))
md.DealID = uint32(binary.BigEndian.Uint32(v[6:10]))
md.BotEventID = uint32(binary.BigEndian.Uint32(v[10:14]))
md.BotID = uint32(binary.BigEndian.Uint32(v[0:4]))
md.DealID = uint32(binary.BigEndian.Uint32(v[4:8]))
md.BotEventID = uint32(binary.BigEndian.Uint32(v[8:12]))

return md, nil
}
Expand Down
48 changes: 12 additions & 36 deletions metadata/md_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"encoding/hex"
"strings"
"testing"
"time"

"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/require"
Expand Down Expand Up @@ -40,8 +39,10 @@ func TestAsHex_(t *testing.T) {
if tc.wantPanic {
return
}
if !bytes.Equal(mustHex(tc.hex), got) {
t.Fatalf("not equal\nwant: % X\ngot: % X", tc.hex, got)

want := mustHex(tc.hex)
if !bytes.Equal(want, got) {
t.Fatalf("not equal\nwant: % X\ngot: % X", want, got)
}
})
}
Expand Down Expand Up @@ -113,63 +114,38 @@ type hexData struct {
}

func getTests() []hexData {
epoch := time.Date(1970, 1, 1, 0, 0, 0, 0, time.UTC)
return []hexData{
{
name: "Empty",
md: Metadata{},
hex: "06 C6 00 00 00 00 00 00 00 00 00 00 00 00 3F 5A",
},
{
name: "epoch day 0, all zeros",
md: Metadata{
CreatedAt: epoch,
BotID: 0,
DealID: 0,
BotEventID: 0,
},
hex: "0000 00000000 00000000 00000000 FE 54",
hex: "00000000 00000000 00000000 7B D5 C6 6F",
},
{
name: "small values sample",
md: Metadata{
CreatedAt: epoch.Add(24 * time.Hour), // 1970-01-02 -> day=1
BotID: 1,
DealID: 2,
BotEventID: 3,
},
hex: "0001 00000001 00000002 00000003 2E 62",
hex: "00000001 00000002 00000003 8F 67 D0 F6",
},
{
name: "max uint16 day boundary",
name: "boundary",
md: Metadata{
CreatedAt: epoch.AddDate(0, 0, 65535), // day=65535 -> 0xFFFF
BotID: 0x01020304,
DealID: 0xAABBCCDD,
BotEventID: 3735928559, // 0xDEADBEEF within uint32
},
hex: "FFFF 01020304 AABBCCDD DEADBEEF 7D 2C",
hex: "01020304 AABBCCDD DEADBEEF 5D AF 20 B1",
},
// {
// name: "invalid order id panics",
// md: Metadata{
// CreatedAt: epoch,
// BotID: 123,
// DealID: 456,
// OrderID: "not-a-number",
// },
// wantPanic: true,
// wantErrorFromHex: HexTooShort,
// },
{
name: "received back from API",
md: Metadata{
CreatedAt: time.Date(2025, 8, 11, 0, 0, 0, 0, time.UTC),
BotID: 1,
DealID: 1,
BotEventID: 1,
BotID: 16541235,
DealID: 2381631392,
BotEventID: 568275668,
},
hex: "0x4f57000000010000000100000001f620",
hex: "0x00fc66338df4cfa021df32d4094afe38",
},
}
}
21 changes: 7 additions & 14 deletions storage/latestHlSafetyStatus_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,9 @@ func TestStorageListLatestHyperliquidSafetyStatuses(t *testing.T) {
volume int
}

insertSafetyEvent := func(t *testing.T, store *Storage, md metadata.Metadata, o order) metadata.Metadata {
insertSafetyEvent := func(t *testing.T, store *Storage, md metadata.Metadata, created time.Time, o order) metadata.Metadata {
t.Helper()

created := md.CreatedAt
botevent := tc.BotEvent{
CreatedAt: &created,
Action: tc.BotEventActionExecute,
Expand Down Expand Up @@ -79,24 +78,22 @@ func TestStorageListLatestHyperliquidSafetyStatuses(t *testing.T) {
botID := uint32(42)

md1 := metadata.Metadata{
CreatedAt: base,
BotID: botID,
DealID: 9001,
BotEventID: 1,
}
md2 := metadata.Metadata{
CreatedAt: base.Add(24 * time.Hour),
BotID: botID,
DealID: 9001,
BotEventID: 2,
}

normalized1 := insertSafetyEvent(t, store, md1, order{
normalized1 := insertSafetyEvent(t, store, md1, base, order{
position: 1,
ordersize: 2,
volume: 2,
})
normalized2 := insertSafetyEvent(t, store, md2, order{
normalized2 := insertSafetyEvent(t, store, md2, base.Add(24*time.Hour), order{
position: 2,
ordersize: 2,
volume: 1,
Expand Down Expand Up @@ -178,24 +175,22 @@ func TestStorageListLatestHyperliquidSafetyStatuses(t *testing.T) {
botID := uint32(43)

md1 := metadata.Metadata{
CreatedAt: base,
BotID: botID,
DealID: 9002,
BotEventID: 3,
}
md2 := metadata.Metadata{
CreatedAt: base.Add(24 * time.Hour),
BotID: botID,
DealID: 9002,
BotEventID: 4,
}

normalized1 := insertSafetyEvent(t, store, md1, order{
normalized1 := insertSafetyEvent(t, store, md1, base, order{
position: 1,
ordersize: 2,
volume: 3,
})
normalized2 := insertSafetyEvent(t, store, md2, order{
normalized2 := insertSafetyEvent(t, store, md2, base.Add(24*time.Hour), order{
position: 2,
ordersize: 2,
volume: 4,
Expand Down Expand Up @@ -273,12 +268,11 @@ func TestStorageListLatestHyperliquidSafetyStatuses(t *testing.T) {

otherDeal := uint32(9100)
otherMD := metadata.Metadata{
CreatedAt: base,
BotID: 50,
DealID: otherDeal,
BotEventID: 1,
}
insertSafetyEvent(t, store, otherMD, order{
insertSafetyEvent(t, store, otherMD, base, order{
position: 1,
ordersize: 1,
volume: 1,
Expand All @@ -299,12 +293,11 @@ func TestStorageListLatestHyperliquidSafetyStatuses(t *testing.T) {
})

mdTakeProfit := metadata.Metadata{
CreatedAt: base,
BotID: 51,
DealID: 9003,
BotEventID: 2,
}
created := mdTakeProfit.CreatedAt
created := base
takeProfit := tc.BotEvent{
CreatedAt: &created,
Action: tc.BotEventActionExecute,
Expand Down
8 changes: 0 additions & 8 deletions storage/sqlite_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@ func TestStorageThreeCommasRoundTrip(t *testing.T) {
{
name: "basic-roundtrip",
md: metadata.Metadata{
CreatedAt: base,
BotID: 42,
DealID: 7,
BotEventID: 3,
Expand Down Expand Up @@ -79,7 +78,6 @@ func TestStorageThreeCommasRoundTrip(t *testing.T) {
{
name: "different-metadata",
md: metadata.Metadata{
CreatedAt: base.Add(24 * time.Hour),
BotID: 99,
DealID: 1001,
BotEventID: 222,
Expand Down Expand Up @@ -160,7 +158,6 @@ func TestStorageHyperliquidRoundTrip(t *testing.T) {
store := newTestStorage(t)

md := metadata.Metadata{
CreatedAt: time.Date(2024, time.March, 3, 0, 0, 0, 0, time.UTC),
BotID: 7,
DealID: 77,
BotEventID: 777,
Expand Down Expand Up @@ -776,7 +773,6 @@ func TestStorageListEventsForOrder(t *testing.T) {
mk := func(offset time.Duration, price float64, label string) (metadata.Metadata, tc.BotEvent) {
ts := base.Add(offset)
return metadata.Metadata{
CreatedAt: ts,
BotID: botID,
DealID: dealID,
BotEventID: botEventID,
Expand Down Expand Up @@ -824,7 +820,6 @@ func TestStorageListEventsForOrder(t *testing.T) {
}

otherMD := metadata.Metadata{
CreatedAt: base,
BotID: botID,
DealID: dealID + 1,
BotEventID: botEventID,
Expand Down Expand Up @@ -856,7 +851,6 @@ func TestRecordThreeCommasBotEventDuplicateReturnsPreviousInsertID(t *testing.T)

base := time.Date(2024, time.February, 2, 9, 30, 0, 0, time.UTC)
md := metadata.Metadata{
CreatedAt: base,
BotID: 123,
DealID: 456,
BotEventID: 789,
Expand Down Expand Up @@ -902,7 +896,6 @@ func TestLoadTakeProfitForDeal(t *testing.T) {
t.Run("returns latest take profit event", func(t *testing.T) {
base := time.Date(2024, time.January, 15, 8, 0, 0, 0, time.UTC)
md := metadata.Metadata{
CreatedAt: base,
BotID: 321,
DealID: 654,
BotEventID: 987,
Expand All @@ -929,7 +922,6 @@ func TestLoadTakeProfitForDeal(t *testing.T) {
require.Equal(t, md.BotID, gotMD.BotID)
require.Equal(t, md.DealID, gotMD.DealID)
require.Equal(t, md.BotEventID, gotMD.BotEventID)
require.Equal(t, md.CreatedAt.Truncate(24*time.Hour), gotMD.CreatedAt)
require.Equal(t, evt.Coin, gotEvent.Coin)
require.Equal(t, evt.Action, gotEvent.Action)
require.Equal(t, tc.MarketOrderDealOrderTypeTakeProfit, gotEvent.OrderType)
Expand Down