diff --git a/peer/brontide.go b/peer/brontide.go index d59f90ddf4..4c11fccc0d 100644 --- a/peer/brontide.go +++ b/peer/brontide.go @@ -4997,22 +4997,24 @@ func newRPCShutdownInit(req *htlcswitch.ChanClose) shutdownInit { // waitUntilRbfCoastClear waits for the shutdown handshake and any local // offer in progress to finish before attempting another fee bump. +// It returns the local state the close was in when the coast was clear. func waitUntilRbfCoastClear(ctx context.Context, - rbfCloser *chancloser.RbfChanCloser) error { + rbfCloser *chancloser.RbfChanCloser) (chancloser.AsymmetricPeerState, + error) { coopCloseStates := rbfCloser.RegisterStateEvents() defer rbfCloser.RemoveStateSub(coopCloseStates) for { if err := ctx.Err(); err != nil { - return err + return nil, err } // Subscribe before querying, and query again after each event: // a queued notification can already describe an older state. state, err := rbfCloser.CurrentState() if err != nil { - return err + return nil, err } switch state := state.(type) { case *chancloser.ShutdownPending, *chancloser.ChannelFlushing: @@ -5024,28 +5026,29 @@ func waitUntilRbfCoastClear(ctx context.Context, case *chancloser.LocalCloseStart, *chancloser.ClosePending, *chancloser.CloseErr: - return nil + return local, nil case *chancloser.LocalOfferSent: // Wait for the peer's signature for this offer. default: - return fmt.Errorf("cannot bump cooperative "+ - "close in %T", local) + return nil, fmt.Errorf("cannot bump "+ + "cooperative close in %T", local) } case *chancloser.CloseFin: - return fmt.Errorf("cooperative close already confirmed") + return nil, fmt.Errorf("cooperative close already " + + "confirmed") default: - return fmt.Errorf("cannot bump cooperative close in "+ - "%T", state) + return nil, fmt.Errorf("cannot bump cooperative "+ + "close in %T", state) } select { case <-coopCloseStates.NewItemCreated.ChanOut(): case <-ctx.Done(): - return ctx.Err() + return nil, ctx.Err() case <-rbfCloser.Done(): - return protofsm.ErrStateMachineShutdown + return nil, protofsm.ErrStateMachineShutdown } } } @@ -5122,7 +5125,7 @@ func (p *Brontide) startRbfChanCloser(shutdown shutdownInit, ctx, cancel := p.cg.Create(closeReqCtx(rpcReq)) defer cancel() - startErr = p.sendRbfCloseEvent( + _, startErr = p.sendRbfCloseEvent( ctx, rbfCloser, chanID, rpcReq, false, &chancloser.SendShutdown{ IdealFeeRate: feeRate, @@ -5219,6 +5222,10 @@ func (p *Brontide) bumpRbfCloseWhenClear(rbfCloser *chancloser.RbfChanCloser, // waiting on the same close are released by the same state, and the second // offer would then reach a close that is waiting for the peer's signature of // the first, which stops the closer. +// +// A bump keeps its turn until the close has taken its offer. Sending only +// queues the offer, so the next bump could otherwise still read the state +// from before it. func (p *Brontide) sendRbfFeeBump(ctx context.Context, rbfCloser *chancloser.RbfChanCloser, chanID lnwire.ChannelID, req *htlcswitch.ChanClose, feeRate chainfee.SatPerVByte) error { @@ -5229,45 +5236,92 @@ func (p *Brontide) sendRbfFeeBump(ctx context.Context, } defer release() - if err := waitUntilRbfCoastClear(ctx, rbfCloser); err != nil { + local, err := waitUntilRbfCoastClear(ctx, rbfCloser) + if err != nil { return err } - return p.sendRbfCloseEvent( + // Subscribe before sending, so that the transition the offer causes + // is not missed. + states := rbfCloser.RegisterStateEvents() + defer rbfCloser.RemoveStateSub(states) + + sent, err := p.sendRbfCloseEvent( ctx, rbfCloser, chanID, req, true, &chancloser.SendOfferEvent{TargetFeeRate: feeRate}, ) + if err != nil || !sent { + return err + } + + waitRbfOfferTaken(ctx, rbfCloser, states, local) + + return nil +} + +// waitRbfOfferTaken waits until the close has taken an offer that was sent +// while its local state was before. Only a local offer replaces the local +// state: an event for the remote party keeps it, and the fee bumps of a +// channel take turns, so a different local state means this offer was taken. +// The wait also ends when the caller or the peer goes away, when the closer +// stops, or when the close leaves negotiation. +func waitRbfOfferTaken(ctx context.Context, + rbfCloser *chancloser.RbfChanCloser, states chancloser.RbfStateSub, + before chancloser.AsymmetricPeerState) { + + for { + select { + case state := <-states.NewItemCreated.ChanOut(): + closing, ok := state.(*chancloser.ClosingNegotiation) + if !ok { + return + } + + local := closing.PeerState.GetForParty(lntypes.Local) + if local != before { + return + } + + case <-ctx.Done(): + return + + case <-rbfCloser.Done(): + return + } + } } // sendRbfCloseEvent sends an event that starts or bumps a cooperative close. // For a caller, it first starts the observer that reports how the close ends; -// recorded says whether the caller is already in activeCloseReqs. An error -// means the event was not sent and the caller has not been answered. +// recorded says whether the caller is already in activeCloseReqs. It reports +// whether the event was sent. An error means the event was not sent and the +// caller has not been answered. func (p *Brontide) sendRbfCloseEvent(ctx context.Context, rbfCloser *chancloser.RbfChanCloser, chanID lnwire.ChannelID, req *htlcswitch.ChanClose, recorded bool, - event chancloser.ProtocolEvent) error { + event chancloser.ProtocolEvent) (bool, error) { if err := ctx.Err(); err != nil { - return err + return false, err } coopCloseStates := rbfCloser.RegisterStateEvents() current, err := rbfCloser.CurrentState() if err != nil { rbfCloser.RemoveStateSub(coopCloseStates) - return err + return false, err } if _, confirmed := current.(*chancloser.CloseFin); confirmed { rbfCloser.RemoveStateSub(coopCloseStates) - return fmt.Errorf("cooperative close already confirmed") + return false, fmt.Errorf("cooperative close already " + + "confirmed") } if req == nil { defer rbfCloser.RemoveStateSub(coopCloseStates) rbfCloser.SendEvent(ctx, event) - return nil + return true, nil } // Record the caller against the channel so that retiring the closer @@ -5287,7 +5341,7 @@ func (p *Brontide) sendRbfCloseEvent(ctx context.Context, rbfCloser.RemoveStateSub(coopCloseStates) p.claimAndAnswerRetiredCaller(chanID, req) - return nil + return false, nil } // Before we send the event, we'll launch a goroutine to watch for the @@ -5301,7 +5355,7 @@ func (p *Brontide) sendRbfCloseEvent(ctx context.Context, rbfCloser.SendEvent(ctx, event) - return nil + return true, nil } // handleLocalCloseReq kicks-off the workflow to execute a cooperative or diff --git a/peer/rbf_bump_turn_test.go b/peer/rbf_bump_turn_test.go new file mode 100644 index 0000000000..5e57bf629f --- /dev/null +++ b/peer/rbf_bump_turn_test.go @@ -0,0 +1,202 @@ +package peer + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/btcsuite/btcd/btcec/v2" + "github.com/btcsuite/btcd/btcec/v2/ecdsa" + "github.com/btcsuite/btcd/btcutil" + "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/btcsuite/btcd/wire" + "github.com/lightningnetwork/lnd/chainntnfs" + "github.com/lightningnetwork/lnd/channeldb" + "github.com/lightningnetwork/lnd/input" + "github.com/lightningnetwork/lnd/lntypes" + "github.com/lightningnetwork/lnd/lnwallet" + "github.com/lightningnetwork/lnd/lnwallet/chainfee" + "github.com/lightningnetwork/lnd/lnwallet/chancloser" + "github.com/lightningnetwork/lnd/lnwire" + "github.com/lightningnetwork/lnd/protofsm" + "github.com/stretchr/testify/require" +) + +// turnFeeEstimator makes every offer payable. +type turnFeeEstimator struct{} + +func (turnFeeEstimator) EstimateFee(channeldb.ChannelType, *wire.TxOut, + *wire.TxOut, []*wire.TxOut, chainfee.SatPerKWeight) btcutil.Amount { + + return 1000 +} + +// turnCloseSigner signs every offer with a throwaway key. It reports each +// offer it is asked to sign, then waits for the gate before it signs. +type turnCloseSigner struct { + key *btcec.PrivateKey + signing chan struct{} + gate chan struct{} +} + +func (s *turnCloseSigner) CreateCloseProposal(btcutil.Amount, []byte, + []byte, ...lnwallet.ChanCloseOpt) (input.Signature, *wire.MsgTx, + btcutil.Amount, error) { + + s.signing <- struct{}{} + <-s.gate + + sig := ecdsa.Sign(s.key, chainhash.HashB([]byte("close"))) + + return sig, wire.NewMsgTx(2), 9000, nil +} + +func (s *turnCloseSigner) CompleteCooperativeClose(input.Signature, + input.Signature, []byte, []byte, btcutil.Amount, + ...lnwallet.ChanCloseOpt) (*wire.MsgTx, btcutil.Amount, error) { + + return wire.NewMsgTx(2), 9000, nil +} + +// turnDaemon accepts the closer's messages and does nothing else. +type turnDaemon struct{} + +func (turnDaemon) SendMessages(btcec.PublicKey, []lnwire.Message) error { + return nil +} + +func (turnDaemon) BroadcastTransaction(*wire.MsgTx, string) error { + return nil +} + +func (turnDaemon) RegisterConfirmationsNtfn(*chainhash.Hash, []byte, uint32, + uint32, ...chainntnfs.NotifierOption) (*chainntnfs.ConfirmationEvent, + error) { + + return &chainntnfs.ConfirmationEvent{}, nil +} + +func (turnDaemon) RegisterSpendNtfn(*wire.OutPoint, []byte, + uint32) (*chainntnfs.SpendEvent, error) { + + return &chainntnfs.SpendEvent{}, nil +} + +// turnErrors records the error that stops the closer. +type turnErrors struct { + errs chan error +} + +func (e *turnErrors) ReportError(err error) { + e.errs <- err +} + +// TestRbfFeeBumpTurnHeldUntilOfferTaken checks that a fee bump keeps its turn +// until the close has taken its offer. The signer holds the close in the +// middle of the offer: the offer is sent, but the state does not show it yet, +// so another bump must not get the turn. +func TestRbfFeeBumpTurnHeldUntilOfferTaken(t *testing.T) { + key, err := btcec.NewPrivateKey() + require.NoError(t, err) + + h, err := createTestPeerWithChannel(t, noUpdate) + require.NoError(t, err) + p := h.peer + point := h.channel.ChannelPoint() + chanID := lnwire.NewChanIDFromOutPoint(point) + + terms := &chancloser.CloseChannelTerms{ + ShutdownScripts: chancloser.ShutdownScripts{ + LocalDeliveryScript: genScript(t, p2wshAddress), + RemoteDeliveryScript: genScript(t, p2wshAddress), + }, + ShutdownBalances: chancloser.ShutdownBalances{ + LocalBalance: 10_000_000, + RemoteBalance: 10_000_000, + }, + } + signer := &turnCloseSigner{ + key: key, + signing: make(chan struct{}, 1), + gate: make(chan struct{}), + } + reporter := &turnErrors{errs: make(chan error, 1)} + machine := protofsm.NewStateMachine(chancloser.RbfChanCloserCfg{ + ErrorReporter: reporter, + Daemon: turnDaemon{}, + InitialState: &chancloser.ClosingNegotiation{ + PeerState: lntypes.Dual[chancloser.AsymmetricPeerState]{ + Local: &chancloser.LocalCloseStart{ + CloseChannelTerms: terms, + }, + Remote: &chancloser.RemoteCloseStart{ + CloseChannelTerms: terms, + }, + }, + CloseChannelTerms: terms, + }, + Env: &chancloser.Environment{ + ChainParams: p.cfg.Wallet.Cfg.NetParams, + ChanPeer: *key.PubKey(), + ChanPoint: point, + ChanID: chanID, + FeeEstimator: turnFeeEstimator{}, + CloseSigner: signer, + }, + }) + machine.Start(t.Context()) + t.Cleanup(machine.Stop) + p.activeChanCloses.Store(chanID, makeRbfCloser(&machine)) + + // Let the signer go before the close is stopped, also when the test + // fails early: a close held in the signer cannot stop. + var openGate sync.Once + releaseSigner := func() { + openGate.Do(func() { close(signer.gate) }) + } + t.Cleanup(releaseSigner) + + req := newFeeBumpRequest(t.Context(), &point, 5000) + startFeeBump(t, p, point, req) + + // The close is in the middle of the offer. + select { + case <-signer.signing: + case <-time.After(timeout): + t.Fatalf("the offer did not reach the signer") + } + + // The bump still has the turn. + ctx, cancel := context.WithTimeout(t.Context(), shortTimeout) + defer cancel() + _, err = p.rbfFeeBumps.acquire(ctx, chanID) + require.ErrorIs( + t, err, context.DeadlineExceeded, + "turn given up before the close took the offer", + ) + + // Once the close has taken the offer, the turn is free again. + releaseSigner() + ctx, cancel = context.WithTimeout(t.Context(), timeout) + defer cancel() + release, err := p.rbfFeeBumps.acquire(ctx, chanID) + require.NoError(t, err) + release() + + state, err := machine.CurrentState() + require.NoError(t, err) + negotiation, ok := state.(*chancloser.ClosingNegotiation) + require.True(t, ok) + require.IsType( + t, &chancloser.LocalOfferSent{}, + negotiation.PeerState.GetForParty(lntypes.Local), + ) + require.Empty(t, reporter.errs) + require.Eventually(t, func() bool { + p.rbfFeeBumps.mu.Lock() + defer p.rbfFeeBumps.mu.Unlock() + + return len(p.rbfFeeBumps.slots) == 0 + }, timeout, 10*time.Millisecond, "slot kept after the last bump") +} diff --git a/peer/rbf_fee_bump_slots.go b/peer/rbf_fee_bump_slots.go index 0ac3b8be93..3e4ab6da32 100644 --- a/peer/rbf_fee_bump_slots.go +++ b/peer/rbf_fee_bump_slots.go @@ -12,10 +12,20 @@ import ( // // A slot is a channel with room for one token: holding the token is holding // the slot. A bump that waits for the slot can still be cancelled, which a -// mutex would not allow. +// mutex would not allow. A channel's slot is kept only while a bump holds it +// or waits for it. type rbfFeeBumpSlots struct { mu sync.Mutex - slots map[lnwire.ChannelID]chan struct{} + slots map[lnwire.ChannelID]*rbfFeeBumpSlot +} + +// rbfFeeBumpSlot is the slot of one channel. +type rbfFeeBumpSlot struct { + // token holds one value while a bump holds the slot. + token chan struct{} + + // users counts the bumps that hold the slot or wait for it. + users int } // acquire waits until the channel's slot is free and takes it, or returns the @@ -26,20 +36,37 @@ func (s *rbfFeeBumpSlots) acquire(ctx context.Context, s.mu.Lock() if s.slots == nil { - s.slots = make(map[lnwire.ChannelID]chan struct{}) + s.slots = make(map[lnwire.ChannelID]*rbfFeeBumpSlot) } slot, ok := s.slots[chanID] if !ok { - slot = make(chan struct{}, 1) + slot = &rbfFeeBumpSlot{token: make(chan struct{}, 1)} s.slots[chanID] = slot } + slot.users++ s.mu.Unlock() + // leave drops the channel's slot once no bump uses it. + leave := func() { + s.mu.Lock() + defer s.mu.Unlock() + + slot.users-- + if slot.users == 0 { + delete(s.slots, chanID) + } + } + select { - case slot <- struct{}{}: - return func() { <-slot }, nil + case slot.token <- struct{}{}: + return func() { + <-slot.token + leave() + }, nil case <-ctx.Done(): + leave() + return nil, ctx.Err() } } diff --git a/peer/rbf_reconnect_test.go b/peer/rbf_reconnect_test.go index 50b5633508..5e40315ad4 100644 --- a/peer/rbf_reconnect_test.go +++ b/peer/rbf_reconnect_test.go @@ -488,4 +488,9 @@ func TestRbfFeeBumpSlots(t *testing.T) { err, recvErr := fn.RecvOrTimeout(got, timeout) require.NoError(t, recvErr) require.NoError(t, err) + + // A channel's slot is dropped once no bump uses it. + slots.mu.Lock() + defer slots.mu.Unlock() + require.Empty(t, slots.slots) }