Shopify · Aguasvivas22 · Oct 23, 2025 · Oct 23, 2025
diff --git a/experiments/oscillating_errors.png b/experiments/oscillating_errors.png
diff --git a/experiments/oscillating_errors_adaptive.png b/experiments/oscillating_errors_adaptive.png
diff --git a/experiments/test_oscillating_errors.rb b/experiments/test_oscillating_errors.rb
@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift(File.expand_path("../lib", __dir__))
+
+require "semian"
+require_relative "mock_service"
+require_relative "experimental_resource"
+
+puts "Creating mock service..."
+# Create a single shared mock service instance
+service = Semian::Experiments::MockService.new(
+  endpoints_count: 50,
+  min_latency: 0.01,
+  max_latency: 0.3,
+  distribution: {
+    type: :log_normal,
+    mean: 0.15,
+    std_dev: 0.05,
+  },
+  error_rate: 0.01,
+  timeout: 5, # 5 seconds timeout
+)
+
+# Semian configuration
+semian_config = {
+  success_threshold: 2,
+  error_threshold: 3,
+  error_threshold_timeout: 20,
+  error_timeout: 15,
+  bulkhead: false,
+}
+
+# Initialize Semian resource before threading to avoid race conditions
+puts "Initializing Semian resource..."
+begin
+  init_resource = Semian::Experiments::ExperimentalResource.new(
+    name: "protected_service_oscillating",
+    service: service,
+    semian: semian_config,
+  )
+  init_resource.request(0) # Make one request to trigger registration
+rescue
+  # Ignore any error, we just needed to trigger registration
+end
+puts "Resource initialized successfully.\n"
+
+outcomes = {}
+done = false
+outcomes_mutex = Mutex.new
+
+num_threads = 60
+puts "Starting #{num_threads} concurrent request threads (50 requests/second each = 3000 rps total)..."
+puts "Each thread will have its own adapter instance connected to the shared service...\n"
+
+request_threads = []
+num_threads.times do |_|
+  request_threads << Thread.new do
+    # Each thread creates its own adapter instance that wraps the shared service
+    # They share the same Semian circuit breaker via the name
+    thread_resource = Semian::Experiments::ExperimentalResource.new(
+      name: "protected_service_oscillating",
+      service: service,
+      semian: semian_config,
+    )
+    until done
+      sleep(0.02) # Each thread: 50 requests per second
+
+      begin
+        thread_resource.request(rand(service.endpoints_count))
+
+        outcomes_mutex.synchronize do
+          current_sec = outcomes[Time.now.to_i] ||= {
+            success: 0,
+            circuit_open: 0,
+            error: 0,
+          }
+          print("✓")
+          current_sec[:success] += 1
+        end
+      rescue Semian::Experiments::ExperimentalResource::CircuitOpenError
+        outcomes_mutex.synchronize do
+          current_sec = outcomes[Time.now.to_i] ||= {
+            success: 0,
+            circuit_open: 0,
+            error: 0,
+          }
+          print("⚡")
+          current_sec[:circuit_open] += 1
+        end
+      rescue Semian::Experiments::ExperimentalResource::RequestError, Semian::Experiments::ExperimentalResource::TimeoutError
+        outcomes_mutex.synchronize do
+          current_sec = outcomes[Time.now.to_i] ||= {
+            success: 0,
+            circuit_open: 0,
+            error: 0,
+          }
+          print("✗")
+          current_sec[:error] += 1
+        end
+      end
+    end
+  end
+end
+
+error_phases = [0.01, 0.01] + [0.02, 0.06] * 9 + [0.01, 0.01]
+phase_duration = 10
+test_duration = error_phases.length * phase_duration
+
+puts "\n=== Oscillating Error Test (CLASSIC) ==="
+puts "Error rate: #{error_phases.map { |r| "#{(r * 100).round(1)}%" }.join(" -> ")}"
+puts "Phase duration: #{phase_duration} seconds (#{(phase_duration / 60.0).round(1)} minutes) per phase"
+puts "Duration: #{test_duration} seconds (#{(test_duration / 60.0).round(1)} minutes)"
+puts "Starting test...\n"
+
+start_time = Time.now
+
+# Execute each phase
+error_phases.each_with_index do |error_rate, idx|
+  if idx > 0 # Skip transition message for first phase (already at 1%)
+    puts "\n=== Transitioning to #{(error_rate * 100).round(1)}% error rate ==="
+    # Update error rate on the shared service
+    service.set_error_rate(error_rate)
+  end
+
+  sleep phase_duration
+end
+
+done = true
+puts "\nWaiting for all request threads to finish..."
+request_threads.each(&:join)
+end_time = Time.now
+
+puts "\n\n=== Test Complete ==="
+puts "Actual duration: #{(end_time - start_time).round(2)} seconds"
+puts "\nGenerating analysis..."
+
+# Calculate summary statistics
+total_success = outcomes.values.sum { |data| data[:success] }
+total_circuit_open = outcomes.values.sum { |data| data[:circuit_open] }
+total_error = outcomes.values.sum { |data| data[:error] }
+total_requests = total_success + total_circuit_open + total_error
+
+puts "\n=== Summary Statistics ==="
+puts "Total Requests: #{total_requests}"
+puts "  Successes: #{total_success} (#{(total_success.to_f / total_requests * 100).round(2)}%)"
+puts "  Circuit Open: #{total_circuit_open} (#{(total_circuit_open.to_f / total_requests * 100).round(2)}%)"
+puts "  Errors: #{total_error} (#{(total_error.to_f / total_requests * 100).round(2)}%)"
+
+# Time-based analysis (phase_duration buckets - one per phase)
+bucket_size = phase_duration
+num_buckets = (test_duration / bucket_size.to_f).ceil
+
+puts "\n=== Time-Based Analysis (#{bucket_size}-second buckets) ==="
+(0...num_buckets).each do |bucket_idx|
+  bucket_start = outcomes.keys[0] + (bucket_idx * bucket_size)
+  bucket_data = outcomes.select { |time, _| time >= bucket_start && time < bucket_start + bucket_size }
+
+  bucket_success = bucket_data.values.sum { |d| d[:success] }
+  bucket_errors = bucket_data.values.sum { |d| d[:error] }
+  bucket_circuit = bucket_data.values.sum { |d| d[:circuit_open] }
+  bucket_total = bucket_success + bucket_errors + bucket_circuit
+
+  bucket_time_range = "#{bucket_idx * bucket_size}-#{(bucket_idx + 1) * bucket_size}s"
+  circuit_pct = bucket_total > 0 ? ((bucket_circuit.to_f / bucket_total) * 100).round(2) : 0
+  error_pct = bucket_total > 0 ? ((bucket_errors.to_f / bucket_total) * 100).round(2) : 0
+  status = bucket_circuit > 0 ? "⚡" : "✓"
+
+  phase_error_rate = error_phases[bucket_idx] || error_phases.last
+  phase_label = "[Target: #{(phase_error_rate * 100).round(1)}%]"
+
+  puts "#{status} #{bucket_time_range} #{phase_label}: #{bucket_total} requests | Success: #{bucket_success} | Errors: #{bucket_errors} (#{error_pct}%) | Circuit Open: #{bucket_circuit} (#{circuit_pct}%)"
+end
+
+puts "\nGenerating visualization..."
+
+require "gruff"
+
+# Create line graph showing requests per 10-second bucket
+graph = Gruff::Line.new(1400)
+graph.title = "Classic Circuit Breaker: Oscillating Errors (2% <-> 6%)"
+graph.x_axis_label = "Time (10-second intervals)"
+graph.y_axis_label = "Requests per Interval"
+
+graph.hide_dots = false
+graph.line_width = 3
+
+# Aggregate data into 10-second buckets for detailed visualization
+small_bucket_size = 10
+num_small_buckets = (test_duration / small_bucket_size.to_f).ceil
+
+bucketed_data = []
+(0...num_small_buckets).each do |bucket_idx|
+  bucket_start = outcomes.keys[0] + (bucket_idx * small_bucket_size)
+  bucket_data = outcomes.select { |time, _| time >= bucket_start && time < bucket_start + small_bucket_size }
+
+  bucketed_data << {
+    success: bucket_data.values.sum { |d| d[:success] },
+    circuit_open: bucket_data.values.sum { |d| d[:circuit_open] },
+    error: bucket_data.values.sum { |d| d[:error] },
+  }
+end
+
+# Set x-axis labels (show every 20 seconds for clarity)
+labels = {}
+(0...num_small_buckets).each do |i|
+  time_sec = i * small_bucket_size
+  labels[i] = "#{time_sec}s" if time_sec % 20 == 0
+end
+graph.labels = labels
+
+graph.data("Success", bucketed_data.map { |d| d[:success] })
+graph.data("Circuit Open", bucketed_data.map { |d| d[:circuit_open] })
+graph.data("Error", bucketed_data.map { |d| d[:error] })
+
+graph.write("oscillating_errors.png")
+
+puts "Graph saved to oscillating_errors.png"