Improve anchor healing [#3379]

.dockerignore

@@ -4,4 +4,4 @@
 *.exe
 .git
 */Dockerfile
-.test
+.test

.vscode/launch.json

@@ -539,11 +539,14 @@
       "type": "go",
       "request": "launch",
       "mode": "auto",
-      "program": "${workspaceFolder}/tools/cmd/resend-anchor",
+      "program": "${workspaceFolder}/tools/cmd/debug",
       "cwd": "${workspaceFolder}",
       "args": [
         "heal",
-        "https://mainnet.accumulatenetwork.io/v3",
+        "anchor",
+        "MainNet",
+        "--cached-scan=${env:HOME}/.accumulate/cache/mainnet.json",
+        "Directory→Apollo", "70911"
       ]
     },
     {
@@ -557,9 +560,27 @@
       "program": "${workspaceFolder}/tools/cmd/debug",
       "cwd": "${workspaceFolder}",
       "args": [
-        "heal-synth",
+        "heal",
+        "synth",
         "mainnet",
+        "--cached-scan=${env:HOME}/.accumulate/cache/mainnet.json",
+      ]
+    },
+    {
+      "name": "Heal single anchor",
+      "presentation": { "group": "99-Miscellaneous", },
+      "type": "go",
+      "request": "launch",
+      "mode": "auto",
+      "program": "${workspaceFolder}/tools/cmd/debug",
+      "cwd": "${workspaceFolder}",
+      "args": [
+        "heal",
+        "anchor",
         "mainnet",
+        "Chandrayaan → Apollo",
+        // "acc://f18b9ddd70654849ce063581b7bcbc6947d4b763d7a2fa4668e6fd9129da0e23@dn.acme/anchors",
+        "--cached-scan=${env:HOME}/.accumulate/cache/mainnet.json",
       ]
     },
     {

exp/apiutil/mainnet.go

@@ -0,0 +1,58 @@
+// Copyright 2023 The Accumulate Authors
+//
+// Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file or at
+// https://opensource.org/licenses/MIT.
+
+package apiutil
+
+import "github.com/multiformats/go-multiaddr"
+
+var MainnetAddrs = func() []multiaddr.Multiaddr {
+	s := []string{
+		"/dns/apollo-mainnet.accumulate.defidevs.io/tcp/16593/p2p/12D3KooWAgrBYpWEXRViTnToNmpCoC3dvHdmR6m1FmyKjDn1NYpj",
+		"/dns/yutu-mainnet.accumulate.defidevs.io/tcp/16593/p2p/12D3KooWDqFDwjHEog1bNbxai2dKSaR1aFvq2LAZ2jivSohgoSc7",
+		"/dns/chandrayaan-mainnet.accumulate.defidevs.io/tcp/16593/p2p/12D3KooWHzjkoeAqe7L55tAaepCbMbhvNu9v52ayZNVQobdEE1RL",
+		"/ip4/116.202.214.38/tcp/16593/p2p/12D3KooWBkJQiuvotpMemWBYfAe4ctsVHi7fLvT8RT83oXJ5dsgV",
+		"/ip4/83.97.19.82/tcp/16593/p2p/12D3KooWHSbqS6K52d4ReauHAg4n8MFbAKkdEAae2fZXnzRYi9ce",
+		"/ip4/206.189.97.165/tcp/16593/p2p/12D3KooWHyA7zgAVqGvCBBJejgvKzv7DQZ3LabJMWqmCQ9wFbT3o",
+		"/ip4/144.76.105.23/tcp/16593/p2p/12D3KooWS2Adojqun5RV1Xy4k6vKXWpRQ3VdzXnW8SbW7ERzqKie",
+		"/ip4/18.190.77.236/tcp/16593/p2p/12D3KooWP1d9vUJCzqX5bTv13tCHmVssJrgK3EnJCC2C5Ep2SXbS",
+		"/ip4/3.28.207.55/tcp/16593/p2p/12D3KooWEzhg3CRvC3xdrUBFsWETF1nG3gyYfEjx4oEJer95y1Rk",
+		"/ip4/38.135.195.81/tcp/16593/p2p/12D3KooWDWCHGAyeUWdP8yuuSYvMoUfaPoGu4p3gJb51diqNQz6j",
+		// "/ip4/50.17.246.3/tcp/16593/p2p/12D3KooWKkNsxkHJqvSje2viyqKVxtqvbTpFrbASD3q1uv6td1pW",
+		"/dns/validator-eu01.acme.sphereon.com/tcp/16593/p2p/12D3KooWKYTWKJ5jeuZmbbwiN7PoinJ2yJLoQtZyfWi2ihjBnSUR",
+		"/ip4/35.86.120.53/tcp/16593/p2p/12D3KooWKJuspMDC5GXzLYJs9nHwYfqst9QAW4m5FakXNHVMNiq7",
+		"/ip4/65.109.48.173/tcp/16593/p2p/12D3KooWHkUtGcHY96bNavZMCP2k5ps5mC7GrF1hBC1CsyGJZSPY",
+		"/dns/accumulate.detroitledger.tech/tcp/16593/p2p/12D3KooWNe1QNh5mKAa8iAEP8vFwvmWFxaCLNcAdE1sH38Bz8sc9",
+		"/ip4/3.135.9.97/tcp/16593/p2p/12D3KooWEQG3X528Ct2Kd3kxhv6WZDBqaAoEw7AKiPoK1NmWJgx1",
+		// "/ip4/3.86.85.133/tcp/16593/p2p/12D3KooWJvReA1SuLkppyXKXq6fifVPLqvNtzsvPUqagVjvYe7qe",
+		"/ip4/193.35.56.176/tcp/16593/p2p/12D3KooWJevZUFLqN7zAamDh2EEYNQZPvxGFwiFVyPXfuXZNjg1J",
+		"/ip4/35.177.70.195/tcp/16593/p2p/12D3KooWPzpRp1UCu4nvXT9h8jKvmBmCADrMnoF72DrEbUrWrB2G",
+		"/ip4/3.99.81.122/tcp/16593/p2p/12D3KooWLL5kAbD7nhv6CM9x9L1zjxSnc6hdMVKcsK9wzMGBo99X",
+		"/ip4/34.219.75.234/tcp/16593/p2p/12D3KooWKHjS5nzG9dipBXn31pYEnfa8g5UzvkSYEsuiukGHzPvt",
+		"/ip4/3.122.254.53/tcp/16593/p2p/12D3KooWRU8obVzgfw6TsUHjoy2FDD3Vd7swrPNTM7DMFs8JG4dx",
+		"/ip4/35.92.228.236/tcp/16593/p2p/12D3KooWQqMqbyJ2Zay9KHeEDgDMAxQpKD1ypiBX5ByQAA2XpsZL",
+		"/ip4/3.135.184.194/tcp/16593/p2p/12D3KooWHcxyiE3AGdPnhtj87tByfLnJZVR6mLefadWccbMByrBa",
+		"/ip4/18.133.170.113/tcp/16593/p2p/12D3KooWFbWY2NhBEWTLHUCwwPmNHm4BoJXbojnrJJfuDCVoqrFY",
+		// "/ip4/44.204.224.126/tcp/16593/p2p/12D3KooWAiJJxdgsB39up5h6fz6TSfBz4HsLKTFiBXUrbwA8o54m",
+		"/ip4/35.92.21.90/tcp/16593/p2p/12D3KooWLTV3pTN2NbKeFeseCGHyMXuAkQv68KfCeK4uqJzJMfhZ",
+		"/ip4/3.99.166.147/tcp/16593/p2p/12D3KooWGYUf93iYWsUibSvKdxsYUY1p7fC1nQotCpUcDXD1ABvR",
+		"/ip4/16.171.4.135/tcp/16593/p2p/12D3KooWEMpAxKnXJPkcEXpDmrnjrZ5iFMZvvQtimmTTxuoRGkXV",
+		"/ip4/54.237.244.42/tcp/16593/p2p/12D3KooWLoMkrgW862Gs152jLt6FiZZs4GkY24Su4QojnvMoSNaQ",
+		// "/ip4/3.238.124.43/tcp/16593/p2p/12D3KooWJ8CA8pacTnKWVgBSEav4QG1zJpyeSSME47RugpDUrZp8",
+		"/ip4/13.53.125.115/tcp/16593/p2p/12D3KooWBJk52fQExXHWhFNk692hP7JvTxNTvUMdVne8tbJ3DBf3",
+		"/ip4/13.59.241.224/tcp/16593/p2p/12D3KooWKjYKqg2TgUSLq8CZAP8G6LhjXUWTcQBd9qYL2JHug9HW",
+		"/ip4/18.168.202.86/tcp/16593/p2p/12D3KooWDiKGbUZg1rB5EufRCkRPiDCEPMjyvTfTVR9qsKVVkcuC",
+		"/ip4/35.183.112.161/tcp/16593/p2p/12D3KooWFPKeXzKMd3jtoeG6ts6ADKmVV8rVkXR9k9YkQPgpLzd6",
+	}
+	addrs := make([]multiaddr.Multiaddr, len(s))
+	for i, s := range s {
+		addr, err := multiaddr.NewMultiaddr(s)
+		if err != nil {
+			panic(err)
+		}
+		addrs[i] = addr
+	}
+	return addrs
+}()

exp/apiutil/scan.go

@@ -0,0 +1,202 @@
+// Copyright 2023 The Accumulate Authors
+//
+// Use of this source code is governed by an MIT-style
+// license that can be found in the LICENSE file or at
+// https://opensource.org/licenses/MIT.
+
+package apiutil
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+	"sync"
+
+	"github.com/libp2p/go-libp2p/core/peer"
+	"github.com/multiformats/go-multiaddr"
+	"gitlab.com/accumulatenetwork/accumulate/internal/api/routing"
+	"gitlab.com/accumulatenetwork/accumulate/internal/core/healing"
+	"gitlab.com/accumulatenetwork/accumulate/pkg/api/v3"
+	"gitlab.com/accumulatenetwork/accumulate/pkg/api/v3/message"
+	"gitlab.com/accumulatenetwork/accumulate/pkg/errors"
+	"golang.org/x/exp/slog"
+)
+
+type NetworkScan = healing.NetworkInfo
+
+func LoadNetworkScan(file string) (*NetworkScan, error) {
+	f, err := os.Open(file)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	var net *NetworkScan
+	err = json.NewDecoder(f).Decode(&net)
+	if err != nil {
+		return nil, err
+	}
+	return net, nil
+}
+
+func NewMessageRouter(scan *NetworkScan) (message.Router, error) {
+	var err error
+	router := new(routing.MessageRouter)
+	router.Router, err = routing.NewStaticRouter(scan.Status.Routing, nil)
+	return router, err
+}
+
+type StaticDialer struct {
+	Scan   *healing.NetworkInfo
+	Dialer message.Dialer
+	Nodes  api.NodeService
+
+	mu   sync.RWMutex
+	good map[string]peer.ID
+}
+
+func (h *StaticDialer) BadDial(ctx context.Context, addr multiaddr.Multiaddr, stream message.Stream, err error) bool {
+	return true
+}
+
+func (h *StaticDialer) Dial(ctx context.Context, addr multiaddr.Multiaddr) (message.Stream, error) {
+	// Have we found a good peer?
+	s := h.dialKnownGood(ctx, addr)
+	if s != nil {
+		return s, nil
+	}
+	// Unpack the service address
+	network, peerID, service, _, err := api.UnpackAddress(addr)
+	if err != nil {
+		return nil, err
+	}
+
+	// Check for a recorded address
+	if h.Scan != nil {
+		if peerID != "" {
+			info := h.Scan.PeerByID(peerID)
+			if info != nil {
+				addr := addr
+				if peerID == "" {
+					c, err := multiaddr.NewComponent("p2p", info.ID.String())
+					if err != nil {
+						panic(err)
+					}
+					addr = c.Encapsulate(addr)
+				}
+				for _, paddr := range info.Addresses {
+					s, err := h.Dialer.Dial(ctx, paddr.Encapsulate(addr))
+					if err == nil {
+						h.markGood(addr, info.ID)
+						return s, nil
+					}
+					slog.Error("Failed to connect", "peer", info.ID, "address", paddr, "service", addr, "error", err)
+				}
+			}
+		} else if service.Argument != "" {
+			// In the future not all peers will have all services
+			part, ok := h.Scan.Peers[strings.ToLower(service.Argument)]
+			if ok {
+				tried := map[string]bool{}
+				pick := func() (*healing.PeerInfo, multiaddr.Multiaddr) {
+					for _, p := range part {
+						for _, addr := range p.Addresses {
+							if tried[addr.String()] {
+								continue
+							}
+							tried[addr.String()] = true
+							c, err := multiaddr.NewComponent("p2p", p.ID.String())
+							if err != nil {
+								panic(err)
+							}
+							addr = c.Encapsulate(addr)
+							return p, addr
+						}
+					}
+					return nil, nil
+				}
+
+				for {
+					info, paddr := pick()
+					if paddr == nil {
+						break
+					}
+					s, err := h.Dialer.Dial(ctx, paddr.Encapsulate(addr))
+					if err == nil {
+						h.markGood(addr, info.ID)
+						return s, nil
+					}
+					slog.Error("Failed to connect", "peer", info.ID, "address", paddr, "service", addr, "error", err)
+				}
+			}
+		}
+	}
+
+	// If it specifies a node, do nothing
+	if h.Nodes == nil || peerID != "" {
+		return h.Dialer.Dial(ctx, addr)
+	}
+
+	// Use the API to find a node
+	nodes, err := h.Nodes.FindService(ctx, api.FindServiceOptions{Network: network, Service: service})
+	if err != nil {
+		return nil, errors.UnknownError.WithFormat("locate nodes for %v: %w", addr, err)
+	}
+	if len(nodes) == 0 {
+		return nil, errors.NoPeer.WithFormat("cannot locate a peer for %v", addr)
+	}
+
+	// Try all the nodes
+	for _, n := range nodes {
+		s, err := h.dial(ctx, addr, n.PeerID)
+		if err == nil {
+			h.markGood(addr, n.PeerID)
+			return s, nil
+		}
+		fmt.Printf("%v failed with %v\n", n.PeerID, err)
+	}
+	return nil, errors.NoPeer.WithFormat("no peers are responding for %v", addr)
+}
+
+func (h *StaticDialer) dial(ctx context.Context, addr multiaddr.Multiaddr, peer peer.ID) (message.Stream, error) {
+	c, err := multiaddr.NewComponent("p2p", peer.String())
+	if err != nil {
+		return nil, err
+	}
+	addr = addr.Encapsulate(c)
+	return h.Dialer.Dial(ctx, addr)
+}
+
+func (h *StaticDialer) dialKnownGood(ctx context.Context, addr multiaddr.Multiaddr) message.Stream {
+	h.mu.RLock()
+	id, ok := h.good[addr.String()]
+	h.mu.RUnlock()
+	if !ok {
+		return nil
+	}
+
+	s, err := h.dial(ctx, addr, id)
+	if err == nil {
+		return s
+	}
+
+	slog.Info("Failed to dial previously good node", "id", id, "error", err)
+
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	delete(h.good, addr.String())
+
+	return nil
+}
+
+func (h *StaticDialer) markGood(addr multiaddr.Multiaddr, id peer.ID) {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+
+	if h.good == nil {
+		h.good = map[string]peer.ID{}
+	}
+	h.good[addr.String()] = id
+}

internal/api/routing/router.go

@@ -143,6 +143,9 @@ func routeMessage(routeAccount func(*url.URL) (string, error), route *string, ms
 	case *messaging.SequencedMessage:
 		r, err = routeAccount(msg.Destination)
 
+	case *messaging.SyntheticMessage:
+		return routeMessage(routeAccount, route, msg.Message)
+
 	case *messaging.BlockAnchor:
 		return routeMessage(routeAccount, route, msg.Anchor)