Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit f126219

Browse files
wlamewlame
committed
Latency compensation feature.
1 parent 4fbeabe commit f126219

8 files changed

Lines changed: 280 additions & 30 deletions

File tree

‎README.md‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,7 @@ Every successful request returns JSON with the **full resolved configuration**
180180
}
181181
```
182182

183-
Optional fields appear only when non-zero: `burst_size` (token_bucket), `window_seconds` (sliding_window), `queue_timeout`, `dynamic`.
183+
Optional fields appear only when non-zero: `burst_size` (token_bucket), `window_seconds` (sliding_window), `queue_timeout`, `latency_compensation`, `network_latency_ms`, `dynamic`.
184184

185185
| Field | Description |
186186
|-----------------|-------------|
@@ -190,6 +190,8 @@ Optional fields appear only when non-zero: `burst_size` (token_bucket), `window_
190190
| `max_queue_size`| Maximum queue capacity |
191191
| `overflow` | What happens when queue is full (`reject` or `block`) |
192192
| `dynamic` | `true` if this endpoint was auto-created from an unconfigured path |
193+
| `latency_compensation` | Configured latency compensation in ms |
194+
| `network_latency_ms` | One-way network latency computed from `X-Sent-At` header (present only when header is sent) |
193195

194196
When the queue is full (`overflow: reject`) or the estimated wait exceeds `queue_timeout`, rls returns HTTP 429:
195197

@@ -244,6 +246,33 @@ Set `queue_timeout` (seconds) to reject requests upfront when the predicted wait
244246
245247
Clients can override per-request with the `?timeout=N` query parameter (e.g. `?timeout=999` to effectively disable). A value of `0` (default) disables the check entirely. The timeout prediction is skipped for `lifo` and `random` schedulers where wait time is unpredictable.
246248

249+
### Latency compensation
250+
251+
When a client calls rls and then the target API, the total delay includes the network round-trip to rls. Set `latency_compensation` (ms) to release tickets early, so the actual API call hits the target closer to the ideal rate interval:
252+
253+
```yaml
254+
defaults:
255+
latency_compensation: 20 # compensate for 20ms one-way network latency
256+
257+
endpoints:
258+
- path: "/api"
259+
rate: 10
260+
latency_compensation: 15 # per-endpoint override
261+
```
262+
263+
Formula: `effective_interval = max(1ms, 1/rate - compensation_ms/1000)`. At 10 RPS (100ms interval) with 20ms compensation, the effective interval becomes 80ms (12.5 effective RPS). Defaults to `0` (no compensation, identical behavior to before).
264+
265+
### `X-Sent-At` header
266+
267+
Clients can send `X-Sent-At: <unix_milliseconds>` to measure one-way network latency. The server computes `network_latency_ms = now - sent_at` and includes it in the response for observability:
268+
269+
```bash
270+
curl -H "X-Sent-At: $(date +%s%3N)" http://localhost:8080/
271+
# Response: {..., "network_latency_ms": 23}
272+
```
273+
274+
If the header is missing, unparseable, or the timestamp is in the future (clock skew), the field is omitted or clamped to 0.
275+
247276
## Client example (Python)
248277

249278
```python

‎config/config.go‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ type Defaults struct {
2121
MaxQueueSize int `yaml:"max_queue_size" json:"max_queue_size"`
2222
Overflow string `yaml:"overflow" json:"overflow"`
2323
QueueTimeout float64 `yaml:"queue_timeout" json:"queue_timeout"`
24+
LatencyCompensation float64 `yaml:"latency_compensation" json:"latency_compensation"`
2425
MaxDynamicEndpoints int `yaml:"max_dynamic_endpoints" json:"max_dynamic_endpoints"`
2526
}
2627

@@ -35,8 +36,9 @@ type EndpointConfig struct {
3536
Overflow string `yaml:"overflow" json:"overflow"`
3637
BurstSize int `yaml:"burst_size" json:"burst_size"`
3738
WindowSeconds int `yaml:"window_seconds" json:"window_seconds"`
38-
QueueTimeout float64 `yaml:"queue_timeout" json:"queue_timeout"`
39-
Dynamic bool `yaml:"-" json:"-"`
39+
QueueTimeout float64 `yaml:"queue_timeout" json:"queue_timeout"`
40+
LatencyCompensation float64 `yaml:"latency_compensation" json:"latency_compensation"`
41+
Dynamic bool `yaml:"-" json:"-"`
4042
}
4143

4244
// Config is the top-level configuration.
@@ -132,6 +134,9 @@ func ApplyDefaults(cfg *Config) {
132134
if ep.Overflow == "" {
133135
ep.Overflow = d.Overflow
134136
}
137+
if ep.LatencyCompensation == 0 {
138+
ep.LatencyCompensation = d.LatencyCompensation
139+
}
135140
}
136141
}
137142

@@ -165,6 +170,9 @@ func InheritFrom(child, parent EndpointConfig) EndpointConfig {
165170
if child.QueueTimeout == 0 {
166171
child.QueueTimeout = parent.QueueTimeout
167172
}
173+
if child.LatencyCompensation == 0 {
174+
child.LatencyCompensation = parent.LatencyCompensation
175+
}
168176
return child
169177
}
170178

‎config/config_test.go‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -401,6 +401,43 @@ func TestInheritFrom_DynamicAndPathPreserved(t *testing.T) {
401401
}
402402
}
403403

404+
func TestInheritFrom_LatencyCompensation(t *testing.T) {
405+
parent := EndpointConfig{Path: "/", Rate: 10, LatencyCompensation: 20}
406+
child := EndpointConfig{Path: "/child"}
407+
got := InheritFrom(child, parent)
408+
if got.LatencyCompensation != 20 {
409+
t.Errorf("latency_compensation: got %f, want 20", got.LatencyCompensation)
410+
}
411+
412+
// Child with own value keeps it.
413+
child2 := EndpointConfig{Path: "/child2", LatencyCompensation: 5}
414+
got2 := InheritFrom(child2, parent)
415+
if got2.LatencyCompensation != 5 {
416+
t.Errorf("latency_compensation: got %f, want 5", got2.LatencyCompensation)
417+
}
418+
}
419+
420+
func TestApplyDefaults_LatencyCompensation(t *testing.T) {
421+
cfg := &Config{
422+
Defaults: Defaults{LatencyCompensation: 15},
423+
Endpoints: []EndpointConfig{
424+
{Path: "/", Rate: 1},
425+
{Path: "/api", Rate: 2, LatencyCompensation: 10},
426+
},
427+
}
428+
ApplyDefaults(cfg)
429+
430+
// Root should inherit from defaults.
431+
for _, ep := range cfg.Endpoints {
432+
if ep.Path == "/" && ep.LatencyCompensation != 15 {
433+
t.Errorf("/: latency_compensation: got %f, want 15", ep.LatencyCompensation)
434+
}
435+
if ep.Path == "/api" && ep.LatencyCompensation != 10 {
436+
t.Errorf("/api: latency_compensation: got %f, want 10 (own value)", ep.LatencyCompensation)
437+
}
438+
}
439+
}
440+
404441
func TestMergeOverrides_Empty(t *testing.T) {
405442
cfg := &Config{Server: ServerConfig{Host: "1.2.3.4", Port: 1234}}
406443
if err := MergeOverrides(cfg, map[string]string{}); err != nil {

‎endpoint/endpoint.go‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,9 @@ func New(cfg config.EndpointConfig, opts ...Option) (*Endpoint, error) {
3434
}
3535

3636
l, err := limiter.New(cfg.Algorithm, cfg.Rate, cfg.Unit, limiter.LimiterOptions{
37-
BurstSize: cfg.BurstSize,
38-
WindowSeconds: cfg.WindowSeconds,
37+
BurstSize: cfg.BurstSize,
38+
WindowSeconds: cfg.WindowSeconds,
39+
CompensationMs: cfg.LatencyCompensation,
3940
})
4041
if err != nil {
4142
return nil, err
@@ -155,7 +156,18 @@ func (e *Endpoint) Handle(w http.ResponseWriter, r *http.Request) {
155156
return
156157
}
157158

158-
resp := buildResponse(e.cfg, e.queue.Len(), ticket.EnqueuedAt)
159+
var networkLatencyMs *int64
160+
if sentAt := r.Header.Get("X-Sent-At"); sentAt != "" {
161+
if ms, err := strconv.ParseInt(sentAt, 10, 64); err == nil {
162+
latency := time.Now().UnixMilli() - ms
163+
if latency < 0 {
164+
latency = 0
165+
}
166+
networkLatencyMs = &latency
167+
}
168+
}
169+
170+
resp := buildResponse(e.cfg, e.queue.Len(), ticket.EnqueuedAt, networkLatencyMs)
159171
e.emit(Event{Kind: EventServed, Path: e.cfg.Path, WaitedMs: resp.QueuedForMs, QueueDepth: resp.QueueDepth})
160172
req := r.URL.RawQuery
161173
if req == "" {

‎endpoint/endpoint_test.go‎

Lines changed: 89 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"net/http"
77
"net/http/httptest"
88
"sort"
9+
"strconv"
910
"strings"
1011
"sync"
1112
"testing"
@@ -147,7 +148,7 @@ func TestEndpoint_PriorityHeader_Invalid(t *testing.T) {
147148
func TestBuildResponse_Fields(t *testing.T) {
148149
cfg := baseConfig("/test", 5)
149150
now := time.Now().Add(-50 * time.Millisecond) // simulated 50ms wait
150-
resp := buildResponse(cfg, 3, now)
151+
resp := buildResponse(cfg, 3, now, nil)
151152

152153
if !resp.OK {
153154
t.Error("ok: want true")
@@ -189,7 +190,7 @@ func TestBuildResponse_AllConfigFields(t *testing.T) {
189190
QueueTimeout: 5.5,
190191
Dynamic: false,
191192
}
192-
resp := buildResponse(cfg, 0, time.Now())
193+
resp := buildResponse(cfg, 0, time.Now(), nil)
193194

194195
if resp.Algorithm != "token_bucket" {
195196
t.Errorf("algorithm: got %q", resp.Algorithm)
@@ -222,7 +223,7 @@ func TestBuildResponse_DynamicEndpoint(t *testing.T) {
222223
Overflow: "reject",
223224
Dynamic: true,
224225
}
225-
resp := buildResponse(cfg, 2, time.Now().Add(-100*time.Millisecond))
226+
resp := buildResponse(cfg, 2, time.Now().Add(-100*time.Millisecond), nil)
226227

227228
if !resp.Dynamic {
228229
t.Error("dynamic: want true for dynamic endpoint")
@@ -244,7 +245,7 @@ func TestBuildResponse_JSONContainsAllFields(t *testing.T) {
244245
Algorithm: "token_bucket", MaxQueueSize: 500, Overflow: "reject",
245246
BurstSize: 20, WindowSeconds: 60, QueueTimeout: 3, Dynamic: true,
246247
}
247-
resp := buildResponse(cfg, 5, time.Now().Add(-200*time.Millisecond))
248+
resp := buildResponse(cfg, 5, time.Now().Add(-200*time.Millisecond), nil)
248249

249250
data, err := json.Marshal(resp)
250251
if err != nil {
@@ -275,7 +276,7 @@ func TestBuildResponse_OmitsZeroOptionalFields(t *testing.T) {
275276
Overflow: "reject",
276277
// BurstSize, WindowSeconds, QueueTimeout all zero
277278
}
278-
resp := buildResponse(cfg, 0, time.Now())
279+
resp := buildResponse(cfg, 0, time.Now(), nil)
279280

280281
if resp.BurstSize != 0 {
281282
t.Errorf("burst_size: got %d, want 0", resp.BurstSize)
@@ -926,6 +927,89 @@ func TestEndpoint_ClientDisconnect_HandlerReturns(t *testing.T) {
926927
wg.Wait()
927928
}
928929

930+
func TestEndpoint_XSentAt_ReturnsNetworkLatency(t *testing.T) {
931+
ep, err := New(baseConfig("/", 100))
932+
if err != nil {
933+
t.Fatal(err)
934+
}
935+
defer ep.Stop()
936+
937+
sentAt := time.Now().Add(-50 * time.Millisecond).UnixMilli()
938+
req := httptest.NewRequest(http.MethodGet, "/", nil)
939+
req.Header.Set("X-Sent-At", strconv.FormatInt(sentAt, 10))
940+
rr := httptest.NewRecorder()
941+
ep.Handle(rr, req)
942+
943+
var resp Response
944+
if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil {
945+
t.Fatal(err)
946+
}
947+
if resp.NetworkLatencyMs == nil {
948+
t.Fatal("expected network_latency_ms in response")
949+
}
950+
if *resp.NetworkLatencyMs < 40 || *resp.NetworkLatencyMs > 200 {
951+
t.Errorf("network_latency_ms: got %d, want ~50", *resp.NetworkLatencyMs)
952+
}
953+
}
954+
955+
func TestEndpoint_XSentAt_MissingHeader_OmitsField(t *testing.T) {
956+
ep, err := New(baseConfig("/", 100))
957+
if err != nil {
958+
t.Fatal(err)
959+
}
960+
defer ep.Stop()
961+
962+
req := httptest.NewRequest(http.MethodGet, "/", nil)
963+
rr := httptest.NewRecorder()
964+
ep.Handle(rr, req)
965+
966+
if strings.Contains(rr.Body.String(), "network_latency_ms") {
967+
t.Error("network_latency_ms should be omitted when X-Sent-At is absent")
968+
}
969+
}
970+
971+
func TestEndpoint_XSentAt_FutureTimestamp_ClampsToZero(t *testing.T) {
972+
ep, err := New(baseConfig("/", 100))
973+
if err != nil {
974+
t.Fatal(err)
975+
}
976+
defer ep.Stop()
977+
978+
sentAt := time.Now().Add(10 * time.Second).UnixMilli()
979+
req := httptest.NewRequest(http.MethodGet, "/", nil)
980+
req.Header.Set("X-Sent-At", strconv.FormatInt(sentAt, 10))
981+
rr := httptest.NewRecorder()
982+
ep.Handle(rr, req)
983+
984+
var resp Response
985+
if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil {
986+
t.Fatal(err)
987+
}
988+
if resp.NetworkLatencyMs == nil {
989+
t.Fatal("expected network_latency_ms in response")
990+
}
991+
if *resp.NetworkLatencyMs != 0 {
992+
t.Errorf("network_latency_ms: got %d, want 0 (clamped)", *resp.NetworkLatencyMs)
993+
}
994+
}
995+
996+
func TestEndpoint_XSentAt_InvalidValue_OmitsField(t *testing.T) {
997+
ep, err := New(baseConfig("/", 100))
998+
if err != nil {
999+
t.Fatal(err)
1000+
}
1001+
defer ep.Stop()
1002+
1003+
req := httptest.NewRequest(http.MethodGet, "/", nil)
1004+
req.Header.Set("X-Sent-At", "not-a-number")
1005+
rr := httptest.NewRecorder()
1006+
ep.Handle(rr, req)
1007+
1008+
if strings.Contains(rr.Body.String(), "network_latency_ms") {
1009+
t.Error("network_latency_ms should be omitted when X-Sent-At is invalid")
1010+
}
1011+
}
1012+
9291013
func TestRegistry_LongestPrefixWins(t *testing.T) {
9301014
cfgs := []config.EndpointConfig{
9311015
baseConfig("/api", 10),

‎endpoint/response.go‎

Lines changed: 21 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -21,28 +21,32 @@ type Response struct {
2121
Overflow string `json:"overflow"`
2222
BurstSize int `json:"burst_size,omitempty"`
2323
WindowSeconds int `json:"window_seconds,omitempty"`
24-
QueueTimeout float64 `json:"queue_timeout,omitempty"`
25-
Dynamic bool `json:"dynamic,omitempty"`
24+
QueueTimeout float64 `json:"queue_timeout,omitempty"`
25+
LatencyCompensation float64 `json:"latency_compensation,omitempty"`
26+
NetworkLatencyMs *int64 `json:"network_latency_ms,omitempty"`
27+
Dynamic bool `json:"dynamic,omitempty"`
2628
}
2729

2830
// buildResponse constructs a Response from the endpoint config, current queue depth,
2931
// and the time the ticket was enqueued. All config fields are included — for dynamic
3032
// endpoints this reflects the fully resolved inherited values.
31-
func buildResponse(cfg config.EndpointConfig, queueDepth int, enqueuedAt time.Time) Response {
33+
func buildResponse(cfg config.EndpointConfig, queueDepth int, enqueuedAt time.Time, networkLatencyMs *int64) Response {
3234
return Response{
33-
OK: true,
34-
Endpoint: cfg.Path,
35-
QueuedForMs: time.Since(enqueuedAt).Milliseconds(),
36-
QueueDepth: queueDepth,
37-
Rate: cfg.Rate,
38-
Unit: cfg.Unit,
39-
Scheduler: cfg.Scheduler,
40-
Algorithm: cfg.Algorithm,
41-
MaxQueueSize: cfg.MaxQueueSize,
42-
Overflow: cfg.Overflow,
43-
BurstSize: cfg.BurstSize,
44-
WindowSeconds: cfg.WindowSeconds,
45-
QueueTimeout: cfg.QueueTimeout,
46-
Dynamic: cfg.Dynamic,
35+
OK: true,
36+
Endpoint: cfg.Path,
37+
QueuedForMs: time.Since(enqueuedAt).Milliseconds(),
38+
QueueDepth: queueDepth,
39+
Rate: cfg.Rate,
40+
Unit: cfg.Unit,
41+
Scheduler: cfg.Scheduler,
42+
Algorithm: cfg.Algorithm,
43+
MaxQueueSize: cfg.MaxQueueSize,
44+
Overflow: cfg.Overflow,
45+
BurstSize: cfg.BurstSize,
46+
WindowSeconds: cfg.WindowSeconds,
47+
QueueTimeout: cfg.QueueTimeout,
48+
LatencyCompensation: cfg.LatencyCompensation,
49+
NetworkLatencyMs: networkLatencyMs,
50+
Dynamic: cfg.Dynamic,
4751
}
4852
}

‎limiter/limiter.go‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package limiter
33
import (
44
"context"
55
"fmt"
6+
"math"
67
)
78

89
// Limiter controls when rate-limited slots are made available.
@@ -21,8 +22,9 @@ type BurstQuerier interface {
2122

2223
// LimiterOptions carries algorithm-specific configuration.
2324
type LimiterOptions struct {
24-
BurstSize int // token_bucket: max accumulated tokens
25-
WindowSeconds int // sliding_window: observation window length
25+
BurstSize int // token_bucket: max accumulated tokens
26+
WindowSeconds int // sliding_window: observation window length
27+
CompensationMs float64 // latency compensation: release tickets early by this many ms
2628
}
2729

2830
// New creates a Limiter for the given algorithm, rate, and unit.
@@ -34,6 +36,11 @@ func New(algorithm string, rate float64, unit string, opts LimiterOptions) (Limi
3436
return nil, fmt.Errorf("rate must be > 0, got %f %s", rate, unit)
3537
}
3638

39+
if opts.CompensationMs > 0 {
40+
interval := 1.0/rps - opts.CompensationMs/1000.0
41+
rps = 1.0 / math.Max(0.001, interval)
42+
}
43+
3744
switch algorithm {
3845
case "strict":
3946
return NewStrict(rps), nil

0 commit comments

Comments
 (0)