package controller

import (

"encoding/json"

"errors"

"fmt"

"net"

"os"

"os/exec"

"path/filepath"

"runtime"

"strings"

"sync"

"sync/atomic"

"time"

"github.com/canopy-network/canopy/bft"

"github.com/canopy-network/canopy/fsm"

"github.com/canopy-network/canopy/lib"

"github.com/canopy-network/canopy/lib/crypto"

"github.com/canopy-network/canopy/p2p"

)

/* This file contains the 'Controller' implementation which acts as a bus between the bft, p2p, fsm, and store modules to create the node */

var _ bft.Controller = new(Controller)

// Controller acts as the 'manager' of the modules of the application

type Controller struct {

Address []byte // self address

PublicKey []byte // self public key

PrivateKey crypto.PrivateKeyI // self private key

Config lib.Config // node configuration

Metrics *lib.Metrics // telemetry

LastValidatorSet map[uint64]map[uint64]*lib.ValidatorSet // cache [height][chainID] -> set

FSM *fsm.StateMachine // the core protocol component responsible for maintaining and updating the state of the blockchain

Mempool *Mempool // the in memory list of pending transactions

Consensus *bft.BFT // the async consensus process between the committee members for the chain

P2P *p2p.P2P // the P2P module the node uses to connect to the network

RCManager lib.RCManagerI // the data manager for the 'root chain'

Plugin *lib.Plugin // extensible plugin for FSM

checkpoints map[uint64]map[uint64]lib.HexBytes // cached checkpoints loaded from file

isSyncing *atomic.Bool // is the chain currently being downloaded from peers

log lib.LoggerI // object for logging

*sync.Mutex // mutex for thread safety

}

// New() creates a new instance of a Controller, this is the entry point when initializing an instance of a Canopy application

func New(fsm *fsm.StateMachine, c lib.Config, valKey crypto.PrivateKeyI, metrics *lib.Metrics, l lib.LoggerI) (controller *Controller, err lib.ErrorI) {

address := valKey.PublicKey().Address()

// load the maximum validators param to set limits on P2P

maxMembersPerCommittee, err := fsm.GetMaxValidators()

// if an error occurred when retrieving the max validators

if err != nil {

// exit with error

return

}

// initialize the mempool using the FSM copy and the mempool config

mempool, err := NewMempool(fsm, address, c.MempoolConfig, metrics, l)

// if an error occurred when creating a new mempool

if err != nil {

// exit with error

return

}

// create the controller

controller = &Controller{

Address: address.Bytes(),

PublicKey: valKey.PublicKey().Bytes(),

PrivateKey: valKey,

Config: c,

Metrics: metrics,

FSM: fsm,

Mempool: mempool,

Consensus: nil,

P2P: p2p.New(valKey, maxMembersPerCommittee, metrics, c, l),

isSyncing: &atomic.Bool{},

log: l,

Mutex: &sync.Mutex{},

}

// load checkpoints from file (if provided)

controller.loadCheckpointsFile()

// setup plugin if enabled

if c.Plugin != "" {

if err = controller.PluginExecute(c.Plugin); err != nil {

return nil, err

}

controller.PluginConnectSync()

}

// initialize the consensus in the controller, passing a reference to itself

controller.Consensus, err = bft.New(c, valKey, fsm.Height(), fsm.Height()-1, controller, c.RunVDF, metrics, l)

// initialize the mempool controller

mempool.controller = controller

// if an error occurred initializing the bft module

if err != nil {

// exit with error

return

}

// exit

return

}

// Start() begins the Controller service

func (c *Controller) Start() {

rootChainId, err := c.FSM.GetRootChainId()

if err != nil {

c.log.Fatal(err.Error())

}

// in a non-blocking sub-function

go func() {

// start the P2P module

c.P2P.Start()

// log the beginning of the root-chain API connection

c.log.Warnf("Attempting to connect to the root-chain: %d", rootChainId)

// set a timer to go off once per second

t := time.NewTicker(time.Second)

// once function completes, stop the timer

defer t.Stop()

// each time the timer fires

for range t.C {

// get the root chain info from the rpc

rootChainInfo, e := c.RCManager.GetRootChainInfo(rootChainId, c.Config.ChainId)

if e != nil {

c.log.Error(e.Error()) // log error but continue

} else if rootChainInfo != nil && rootChainInfo.Height != 0 {

c.log.Infof("Received root chain info with %d validators", len(rootChainInfo.ValidatorSet.GetValidatorSet()))

// call mempool check

c.Mempool.CheckMempool()

// update the peer 'must connect'

c.UpdateP2PMustConnect(rootChainInfo.ValidatorSet)

// exit the loop

break

}

c.log.Warnf("Empty root chain info")

}

// start mempool service

go c.CheckMempool()

// start internal Controller listeners for P2P

c.StartListeners()

// Wait until peers reaches minimum count

c.P2P.WaitForMinimumPeers()

// start the syncing process (if not synced to top)

go c.Sync()

// allow sleep and wake up using config

wakeDate := time.Unix(int64(c.Config.SleepUntil), 0)

if time.Now().Before(wakeDate) {

untilTime := time.Until(wakeDate)

c.log.Infof("Sleeping until %s", untilTime.String())

time.Sleep(untilTime)

}

// start the bft consensus (if synced to top)

go c.Consensus.Start()

}()

}

// StartListeners() runs all listeners on separate threads

func (c *Controller) StartListeners() {

c.log.Debug("Listening for inbound txs, block requests, and consensus messages")

// listen for syncing peers

go c.ListenForBlockRequests()

// listen for inbound consensus messages

go c.ListenForConsensus()

// listen for inbound

go c.ListenForTx()

// ListenForBlock() is called once syncing finished

}

// Stop() terminates the Controller service

func (c *Controller) Stop() {

// lock the controller

c.Lock()

// unlock when the function completes

defer c.Unlock()

// close the controller mempool store

c.Mempool.FSM.Discard()

// stop the store module

if err := c.FSM.Store().(lib.StoreI).Close(); err != nil {

c.log.Error(err.Error())

}

// stop the p2p module

c.P2P.Stop()

// stop the plugin process if configured

if c.Config.Plugin != "" {

if err := c.PluginStop(c.Config.Plugin); err != nil {

c.log.Error(err.Error())

}

}

}

// ROOT CHAIN CALLS BELOW

// UpdateRootChainInfo() receives updates from the root-chain thread

func (c *Controller) UpdateRootChainInfo(info *lib.RootChainInfo) {

c.log.Debugf("Updating root chain info")

// ensure this root chain is active

activeRootChainId, _ := c.FSM.GetRootChainId()

// if inactive

if activeRootChainId != info.RootChainId {

c.log.Debugf("Detected inactive root-chain update at rootChainId=%d", info.RootChainId)

return

}

// set timestamp if included

var timestamp time.Time

// if timestamp is not 0

if info.Timestamp != 0 {

timestamp = time.UnixMicro(int64(info.Timestamp))

}

// if the last validator set is empty

if info.LastValidatorSet == nil || len(info.LastValidatorSet.ValidatorSet) == 0 {

// signal to reset consensus and start a new height

c.Consensus.ResetBFT <- bft.ResetBFT{IsRootChainUpdate: false, StartTime: timestamp}

} else {

// signal to reset consensus

c.Consensus.ResetBFT <- bft.ResetBFT{IsRootChainUpdate: true, StartTime: timestamp}

}

// update the peer 'must connect'

c.UpdateP2PMustConnect(info.ValidatorSet)

}

// LoadCommittee() gets the ValidatorSet that is authorized to come to Consensus agreement on the Proposal for a specific height/chainId

func (c *Controller) LoadCommittee(rootChainId, rootHeight uint64) (lib.ValidatorSet, lib.ErrorI) {

return c.RCManager.GetValidatorSet(rootChainId, c.Config.ChainId, rootHeight)

}

// LoadRootChainOrderBook() gets the order book from the root-chain

func (c *Controller) LoadRootChainOrderBook(rootChainId, rootHeight uint64) (*lib.OrderBook, lib.ErrorI) {

return c.RCManager.GetOrders(rootChainId, rootHeight, c.Config.ChainId)

}

// GetRootChainLotteryWinner() gets the pseudorandomly selected delegate to reward and their cut

func (c *Controller) GetRootChainLotteryWinner(fsm *fsm.StateMachine, rootHeight uint64) (winner *lib.LotteryWinner, err lib.ErrorI) {

// get the root chain id from the state machine

rootChainId, err := fsm.LoadRootChainId(c.ChainHeight())

// if an error occurred retrieving the id

if err != nil {

// exit with error

return nil, err

}

// execute the remote call

return c.RCManager.GetLotteryWinner(rootChainId, rootHeight, c.Config.ChainId)

}

// IsValidDoubleSigner() checks if the double signer is valid at a certain double sign height

func (c *Controller) IsValidDoubleSigner(rootChainId, rootHeight uint64, address []byte) bool {

// do a remote call to the root chain to see if the double signer is valid

isValidDoubleSigner, err := c.RCManager.IsValidDoubleSigner(rootChainId, rootHeight, lib.BytesToString(address))

// if an error occurred during the remote call

if err != nil {

// log the error

c.log.Errorf("IsValidDoubleSigner failed with error: %s", err.Error())

// return is not a valid double signer for safety

return false

}

// return the result from the remote call

return *isValidDoubleSigner

}

// PLUGIN CALLS BELOW

const socketDir = "/tmp/plugin"

const socketFile = "plugin.sock"

// runPluginCtl() executes a plugin control script action and returns the command output

func (c *Controller) runPluginCtl(plugin, action string) ([]byte, lib.ErrorI) {

if plugin == "" || strings.Contains(plugin, "..") || strings.ContainsRune(plugin, os.PathSeparator) {

return nil, lib.NewError(lib.NoCode, lib.MainModule, fmt.Sprintf("invalid plugin name %q", plugin))

}

// resolve the control script path

cmdPath, err := resolvePluginCtlPath(plugin)

if err != nil {

return nil, lib.NewError(lib.NoCode, lib.MainModule, err.Error())

}

// create the command using the requested action

cmd := exec.Command(cmdPath, action)

// execute the command and capture output

output, err := cmd.CombinedOutput()

if err != nil {

return nil, lib.NewError(lib.NoCode, lib.MainModule, fmt.Sprintf("failed to execute plugin %s (%s): %v, output: %s", plugin, action, err, string(output)))

}

return output, nil

}

// PluginExecute() executes the plugin control script to start the plugin process

func (c *Controller) PluginExecute(plugin string) lib.ErrorI {

output, err := c.runPluginCtl(plugin, "start")

if err != nil {

return err

}

c.log.Infof("Plugin %s started: %s", plugin, string(output))

return nil

}

// PluginStop() executes the plugin control script to stop the plugin process

func (c *Controller) PluginStop(plugin string) lib.ErrorI {

output, err := c.runPluginCtl(plugin, "stop")

if err != nil {

return err

}

c.log.Infof("Plugin %s stopped: %s", plugin, string(output))

return nil

}

// PluginConnectSync() blocking: enables a unix socket file where plugins can interact with the Canopy FSM

func (c *Controller) PluginConnectSync() {

sockPath := filepath.Join(socketDir, socketFile)

// make the path

if err := os.MkdirAll(socketDir, 0777); err != nil {

c.log.Fatalf("Failed to make the plugin socket path %s: %v", sockPath, err)

}

// clean old socket

if err := os.RemoveAll(sockPath); err != nil {

c.log.Fatalf("Failed to remove plugin socket %s: %v", sockPath, err)

}

// create a unix listener

l, err := net.Listen("unix", sockPath)

if err != nil {

c.log.Fatalf("Failed to listen on socket: %v", err)

}

defer l.Close()

// log the listener

c.log.Infof("Plugin service listening on socket: %s", sockPath)

// wait for a connection

conn, e := l.Accept()

if e != nil {

c.log.Fatalf("Failed to accept plugin connection: %v", e)

}

// create plugin object

c.Plugin = lib.NewPlugin(conn, c.log, time.Duration(c.Config.PluginTimeoutMS)*time.Millisecond)

// register the detached, read-only query provider so plugins can serve custom RPC endpoints

c.Plugin.SetQueryProvider(&pluginQueryProvider{controller: c})

// set plugin in FSM and mempool FSM

c.FSM.Plugin, c.Mempool.FSM.Plugin = c.Plugin, c.Plugin

}

// pluginQueryProvider serves detached, read-only state queries from the plugin by backing them

// with Canopy's historical read-only snapshots (TimeMachine). It is the live-node-owned adapter

// that lets plugin builders implement custom RPC endpoints without a tx/block in flight.

type pluginQueryProvider struct {

controller *Controller

}

// QueryState() executes a read-only state read against a TimeMachine snapshot at the given height (0 = latest committed)

func (p *pluginQueryProvider) QueryState(height uint64, request *lib.PluginStateReadRequest) (lib.PluginStateReadResponse, lib.ErrorI) {

// guard: a nil read request would nil-deref in StateRead()

if request == nil {

return lib.PluginStateReadResponse{}, lib.ErrNilPluginQueryRead()

}

// create a read-only state snapshot at the requested height

sm, err := p.controller.FSM.TimeMachine(height)

if err != nil {

return lib.PluginStateReadResponse{}, lib.ErrTimeMachine(err)

}

// at height 0 (fresh node, pre-first-commit) TimeMachine returns the LIVE FSM, not a snapshot;

// never read from — nor Discard() — the live consensus store

if sm == p.controller.FSM {

return lib.PluginStateReadResponse{}, lib.ErrNoCommittedState()

}

// ensure proper cleanup of the snapshot

defer sm.Discard()

// execute the read against the read-only state

return sm.StateRead(request)

}

// resolvePluginCtlPath() locates the plugin control script from common startup locations

func resolvePluginCtlPath(plugin string) (string, error) {

// construct the relative path for the plugin control script

relPath := filepath.Join("plugin", plugin, "pluginctl.sh")

// try the current working directory first

candidates := []string{

relPath,

}

// add paths relative to the running executable if available

if exePath, err := os.Executable(); err == nil {

exeDir := filepath.Dir(exePath)

candidates = append(candidates,

filepath.Join(exeDir, relPath),

filepath.Join(filepath.Dir(exeDir), relPath),

)

}

// add a path relative to the source tree for local development

if _, sourceFile, _, ok := runtime.Caller(0); ok {

repoRoot := filepath.Dir(filepath.Dir(sourceFile))

candidates = append(candidates, filepath.Join(repoRoot, relPath))

}

// return the first existing file path

for _, candidate := range candidates {

info, err := os.Stat(candidate)

if err == nil && !info.IsDir() {

return candidate, nil

}

}

// exit with a descriptive error containing all attempted paths

return "", fmt.Errorf("plugin launcher not found for %q; checked: %s", plugin, strings.Join(candidates, ", "))

}

// INTERNAL CALLS BELOW

// LoadIsOwnRoot() returns if this chain is its own root (base)

func (c *Controller) LoadIsOwnRoot() (isOwnRoot bool) {

// use the state machine to check if this chain is the root chain

isOwnRoot, err := c.FSM.LoadIsOwnRoot()

// if an error occurred

if err != nil {

// log the error

c.log.Error(err.Error())

}

// exit

return

}

// RootChainId() returns the root chain id according to the FSM

func (c *Controller) LoadRootChainId(height uint64) (rootChainId uint64) {

// use the state machine to get the root chain id

rootChainId, err := c.FSM.LoadRootChainId(height)

// if an error occurred

if err != nil {

// log the error

c.log.Error(err.Error())

}

// exit

return

}

// LoadCertificate() gets the certificate for from the indexer at a specific height

func (c *Controller) LoadCertificate(height uint64) (*lib.QuorumCertificate, lib.ErrorI) {

return c.FSM.LoadCertificate(height)

}

// LoadMinimumEvidenceHeight() gets the minimum evidence height from the finite state machine

func (c *Controller) LoadMinimumEvidenceHeight(rootChainId, rootHeight uint64) (*uint64, lib.ErrorI) {

return c.RCManager.GetMinimumEvidenceHeight(rootChainId, rootHeight)

}

// LoadMaxBlockSize() gets the max block size from the state

func (c *Controller) LoadMaxBlockSize() int {

// load the maximum block size from the nested chain FSM

params, _ := c.FSM.GetParamsCons()

// if the parameters are empty

if params == nil {

// return 0 as the 'max'

return 0

}

// return the max block size as set by the governance param

return int(params.BlockSize)

}

// LoadLastCommitTime() gets a timestamp from the most recent Quorum Block

func (c *Controller) LoadLastCommitTime(height uint64) time.Time {

// load the certificate (and block) from the indexer

cert, err := c.FSM.LoadCertificate(height)

if err != nil {

c.log.Error(err.Error())

return time.Time{}

}

// create a new object reference (to ensure a non-nil result)

block := new(lib.Block)

// populate the object reference with bytes

if err = lib.Unmarshal(cert.Block, block); err != nil {

// log the error

c.log.Error(err.Error())

// exit with empty time

return time.Time{}

}

// ensure the block isn't nil

if block.BlockHeader == nil {

// log the error

c.log.Error("Last block synced is nil")

// exit with empty time

return time.Time{}

}

// return the last block time

return time.UnixMicro(int64(block.BlockHeader.Time))

}

// LoadProposerKeys() gets the last root-chainId proposer keys

func (c *Controller) LoadLastProposers(height uint64) (*lib.Proposers, lib.ErrorI) {

// load the last proposers as determined by the last 5 quorum certificates

return c.FSM.LoadLastProposers(height)

}

// LoadCommitteeData() returns the state metadata for the 'self chain'

func (c *Controller) LoadCommitteeData() (data *lib.CommitteeData, err lib.ErrorI) {

// get the committee data from the FSM

return c.FSM.GetCommitteeData(c.Config.ChainId)

}

// Syncing() returns if any of the supported chains are currently syncing

func (c *Controller) Syncing() *atomic.Bool { return c.isSyncing }

// ResetFSM() resets the underlying state machine to last valid state

func (c *Controller) ResetFSM() { c.FSM.Reset() }

// RootChainHeight() returns the height of the canopy root-chain

func (c *Controller) RootChainHeight() uint64 {

chainId, _ := c.FSM.GetRootChainId()

return c.RCManager.GetHeight(chainId)

}

// ChainHeight() returns the height of this target chain

func (c *Controller) ChainHeight() uint64 { return c.FSM.Height() }

// emptyInbox() discards all unread messages for a specific topic

func (c *Controller) emptyInbox(topic lib.Topic) {

// for each message in the inbox

for len(c.P2P.Inbox(topic)) > 0 {

// discard the message

<-c.P2P.Inbox(topic)

}

}

// getDexRootBatch() is a helper to retrieve the dex batch directly from the root chain of the node

// for the current committee

func (c *Controller) getDexRootBatch(rcBuildHeight uint64) (*lib.DexBatch, lib.ErrorI) {

rcID, err := c.FSM.GetRootChainId()

if err != nil {

return nil, err

}

return c.RCManager.GetDexBatch(rcID, rcBuildHeight, c.Config.ChainId, false)

}

const checkpointsFileName = "checkpoints.json"

// loadCheckpointsFile reads checkpoints.json (if present) into the controller cache.

func (c *Controller) loadCheckpointsFile() {

path := filepath.Join(c.Config.DataDirPath, checkpointsFileName)

fileBytes, err := os.ReadFile(path)

if err != nil {

if !errors.Is(err, os.ErrNotExist) {

c.log.Warnf("failed to read checkpoints file: %s", err)

}

return

}

checkpoints := make(map[uint64]map[uint64]lib.HexBytes)

if err = json.Unmarshal(fileBytes, &checkpoints); err != nil {

c.log.Warnf("failed to parse checkpoints file: %s", err)

return

}

c.checkpoints = checkpoints

}

// checkpointFromFile returns a cached checkpoint for a given chain and height, or nil if not found.

func (c *Controller) checkpointFromFile(height, chainId uint64) lib.HexBytes {

if c.checkpoints == nil {

return nil

}

if chainCheckpoints, ok := c.checkpoints[chainId]; ok {

if checkpoint, ok := chainCheckpoints[height]; ok {

return checkpoint

}

}

return nil

}

// convenience aliases that reference the library package

const (

BlockRequest = lib.Topic_BLOCK_REQUEST

Block = lib.Topic_BLOCK

Tx = lib.Topic_TX

Cons = lib.Topic_CONSENSUS

)