Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions docs/release-notes/unreleased/11540.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
# Bug Fixes

* A reconnect while a channel funding transaction is still unconfirmed now
sends `channel_reestablish` for that channel. The channel was remembered,
but no reestablish was sent until it confirmed, so a peer waiting on the
message disconnected and took every other channel to that peer offline.
The message is sent once on that connection.
[Link](https://github.com/lightningnetwork/lnd/pull/11540)
50 changes: 36 additions & 14 deletions htlcswitch/link.go
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,13 @@ type ChannelLinkConfig struct {
// clients have been restarted, or remote peer have been reconnected.
SyncStates bool

// ChanSyncMsgSent is set when this connection already queued
// channel_reestablish for the channel, before the link existed. The
// link still waits for the remote channel_reestablish, but it must
// not send a second copy. A second copy is not a reconnection, and
// peers fail the channel.
ChanSyncMsgSent bool

// BatchTicker is the ticker that determines the interval that we'll
// use to check the batch to see if there're any updates we should
// flush out. By batching updates into a single commit, we attempt to
Expand Down Expand Up @@ -296,7 +303,7 @@ type ChannelLinkConfig struct {
// time of the link being started, persisted shutdown info was found for
// the channel. This value being set means that we previously sent a
// Shutdown message to our peer, and so we should do so again on
// re-establish and should not allow anymore HTLC adds on the outgoing
// re-establish and should not allow any more HTLC adds in either
// direction of the link.
PreviouslySentShutdown fn.Option[lnwire.Shutdown]

Expand Down Expand Up @@ -983,9 +990,19 @@ func (l *channelLink) syncChanStates(ctx context.Context) error {
return fmt.Errorf("unable to generate chan sync message for "+
"ChannelPoint(%v)", l.channel.ChannelPoint())
}
if err := l.cfg.Peer.SendMessage(true, localChanSyncMsg); err != nil {
return fmt.Errorf("unable to send chan sync message for "+
"ChannelPoint(%v): %v", l.channel.ChannelPoint(), err)

// The peer already queued this message when the channel was pending.
// Transmit it once per connection.
if l.cfg.ChanSyncMsgSent {
l.log.Debugf("channel_reestablish already queued on this " +
"connection")
} else {
err := l.cfg.Peer.SendMessage(true, localChanSyncMsg)
if err != nil {
return fmt.Errorf("unable to send chan sync "+
"message for ChannelPoint(%v): %v",
l.channel.ChannelPoint(), err)
}
}

var msgsToReSend []lnwire.Message
Expand Down Expand Up @@ -4073,20 +4090,25 @@ func (l *channelLink) resumeLink(ctx context.Context) error {
}
}

// If a shutdown message has previously been sent on this link, then we
// need to make sure that we have disabled any HTLC adds on the outgoing
// direction of the link and that we re-resend the same shutdown message
// that we previously sent.
shutdownInfo, err := l.channel.State().ShutdownInfo()
if err != nil && !errors.Is(err, channeldb.ErrNoShutdownInfo) {
return fmt.Errorf("unable to fetch shutdown info: %w", err)
}
if shutdownInfo.IsSome() || l.cfg.PreviouslySentShutdown.IsSome() {
if !l.DisableAdds(Incoming) {
l.log.Warnf("Incoming link adds already disabled")
}
if !l.DisableAdds(Outgoing) {
l.log.Warnf("Outgoing link adds already disabled")
}
}

// Re-send the same shutdown message if one was previously sent.
//
// TODO(yy): we should either move this to chanCloser, or move all
// shutdown handling logic to be managed by the link, but not a mixed of
// partial management by two subsystems.
l.cfg.PreviouslySentShutdown.WhenSome(func(shutdown lnwire.Shutdown) {
// Immediately disallow any new outgoing HTLCs.
if !l.DisableAdds(Outgoing) {
l.log.Warnf("Outgoing link adds already disabled")
}

// Re-send the shutdown message the peer. Since syncChanStates
// would have sent any outstanding CommitSig, it is fine for us
// to immediately queue the shutdown message now.
Expand All @@ -4105,7 +4127,7 @@ func (l *channelLink) resumeLink(ctx context.Context) error {
// ensure that all acknowledgments that occur during channel
// resynchronization have taken affect, causing us only to pull unacked
// packets after starting to read from the downstream mailbox.
err := l.mailBox.ResetPackets()
err = l.mailBox.ResetPackets()
if err != nil {
l.log.Errorf("failed to reset packets: %v", err)
}
Expand Down
36 changes: 36 additions & 0 deletions peer/brontide.go
Original file line number Diff line number Diff line change
Expand Up @@ -742,6 +742,13 @@ type Brontide struct {
// peer's chansync message with its own over and over again.
resentChanSyncMsg map[lnwire.ChannelID]struct{}

// chanSyncSent records channels for which this connection has already
// queued channel_reestablish. A pending channel has no link at
// reconnect, so loadActiveChannels queues the message itself. The link
// created after confirmation must not send a second copy: that message
// is not a reconnection, and peers fail the channel.
chanSyncSent *lnutils.SyncMap[lnwire.ChannelID, struct{}]

// channelEventClient is the channel event subscription client that's
// used to assist retry enabling the channels. This client is only
// created when the reenableTimeout is no greater than 1 minute. Once
Expand Down Expand Up @@ -790,6 +797,8 @@ func NewBrontide(cfg Config) *Brontide {
msgmux.NewMultiMsgRouter(),
))

chanSyncSent := &lnutils.SyncMap[lnwire.ChannelID, struct{}]{}

p := &Brontide{
cfg: cfg,
activeSignal: make(chan struct{}),
Expand All @@ -815,6 +824,7 @@ func NewBrontide(cfg Config) *Brontide {
chanCloseMsgs: make(chan *closeMsg),
chanCloseFlushed: make(chan lnwire.ChannelID),
resentChanSyncMsg: make(map[lnwire.ChannelID]struct{}),
chanSyncSent: chanSyncSent,
startReady: make(chan struct{}),
log: peerLog.WithPrefix(logPrefix),
msgRouter: msgRouter,
Expand Down Expand Up @@ -1347,6 +1357,7 @@ func (p *Brontide) loadActiveChannels(chans []*chanstate.OpenChannel) (
p.log.Errorf("Unable to create channel "+
"reestablish message for channel %v: "+
"%v", chanPoint, err)

continue
}

Expand Down Expand Up @@ -1438,9 +1449,25 @@ func (p *Brontide) loadActiveChannels(chans []*chanstate.OpenChannel) (
// the funding manager's responsibility to spin up pending
// channels. Adding them here would just be extra work as we'll
// tear them down when creating + adding the final link.
//
// BOLT 2 requires channel_reestablish on reconnection for each
// remembered channel. Confirmation is not a condition. Start
// writes this message before the write handler. The link
// created after confirmation must not send a second copy.
if lnChan.IsPending() {
p.markPendingChannel(chanID)

chanSync, err := dbChan.ChanSyncMsg()
if err != nil {
p.log.Errorf("Unable to create channel "+
"reestablish message for channel %v: "+
"%v", chanPoint, err)
continue
}

p.chanSyncSent.Store(chanID, struct{}{})
msgs = append(msgs, chanSync)

continue
}

Expand Down Expand Up @@ -1660,6 +1687,15 @@ func (p *Brontide) addLink(chanPoint *wire.OutPoint,
// down to ensure that the mailboxes are only ever under the control of
// one link.
chanID := lnwire.NewChanIDFromOutPoint(*chanPoint)

// A pending channel already queued channel_reestablish at reconnect.
// The link still syncs, but it must not transmit a second copy.
if p.chanSyncSent != nil {
if _, sent := p.chanSyncSent.Load(chanID); sent {
linkCfg.ChanSyncMsgSent = true
}
}

p.cfg.Switch.RemoveLink(chanID)

// With the channel link created, we'll now notify the htlc switch so
Expand Down
85 changes: 85 additions & 0 deletions peer/brontide_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"github.com/lightningnetwork/lnd/chanstate"
"github.com/lightningnetwork/lnd/contractcourt"
"github.com/lightningnetwork/lnd/fn/v2"
graphdb "github.com/lightningnetwork/lnd/graph/db"
"github.com/lightningnetwork/lnd/htlcswitch"
"github.com/lightningnetwork/lnd/input"
"github.com/lightningnetwork/lnd/lnpeer"
Expand Down Expand Up @@ -4321,6 +4322,90 @@ func TestRbfCloseCancelReleasesOnlyItsCaller(t *testing.T) {
}
}

// TestPendingChannelReestablishOnReconnect verifies that a remembered pending
// channel participates in state synchronization after a reconnect.
//
// BOLT 2, Message Retransmission, does not mention confirmation:
//
// A funding node, upon disconnection, if it has broadcast the funding
// transaction: MUST remember the channel for reconnection.
//
// A node, upon reconnection (unless the channel is in an error state):
// MUST transmit channel_reestablish for each channel.
//
// The fixture is that funding node. loadActiveChannels must return the
// channel_reestablish that Start writes and record the handoff to the link
// created after confirmation.
func TestPendingChannelReestablishOnReconnect(t *testing.T) {
t.Parallel()

harness, err := createTestPeerWithChannel(
t, func(alice, bob *chanstate.OpenChannel) {
// SyncPending writes the caller's pending bit. The
// funding transaction is already set on the initiator,
// which is the "has broadcast" case above.
alice.IsPending = true
bob.IsPending = true
},
)
require.NoError(t, err)

peer := harness.peer

signer := input.NewMockSigner(
[]*btcec.PrivateKey{harness.privKey}, nil,
)
sigPool := lnwallet.NewSigPool(1, signer)
require.NoError(t, sigPool.Start())
t.Cleanup(func() {
require.NoError(t, sigPool.Stop())
})

peer.cfg.Signer = signer
peer.cfg.SigPool = sigPool
peer.cfg.ChannelGraph = graphdb.MakeTestGraph(t)

// Same query Start uses before loadActiveChannels.
channels, err := peer.cfg.ChannelDB.FetchOpenChannels(
peer.cfg.Addr.IdentityKey,
)
require.NoError(t, err)
require.Len(t, channels, 1)

// These three fields are the BOLT 2 precondition, not a local
// policy. Confirmation is not one of them.
dbChan := channels[0]
require.True(t, dbChan.IsInitiator, "funding node")
require.NotNil(t, dbChan.FundingTxn, "funding tx was broadcast")
require.True(t, dbChan.IsPending, "funding tx is unconfirmed")

// Build the message expected from the pending channel.
want, err := dbChan.ChanSyncMsg()
require.NoError(t, err)

msgs, err := peer.loadActiveChannels(channels)
require.NoError(t, err)

var found bool
for _, msg := range msgs {
got, ok := msg.(*lnwire.ChannelReestablish)
if !ok || got.ChanID != want.ChanID {
continue
}

found = true
_, sent := peer.chanSyncSent.Load(want.ChanID)
require.True(t, sent, "later link must not send a second "+
"channel_reestablish on this connection")

break
}

require.True(t, found, "funding node remembered pending channel %v "+
"but did not transmit channel_reestablish (BOLT 2, Message "+
"Retransmission)", want.ChanID)
}

// newTestCloseRequest allocates the channels used by a close caller.
func newTestCloseRequest(point *wire.OutPoint,
buffer int) *htlcswitch.ChanClose {
Expand Down
Loading