From 225dbe6c0340527071a162913e4b64a64df9be32 Mon Sep 17 00:00:00 2001 From: Patryk Date: Wed, 24 Jul 2024 23:42:19 +0200 Subject: [PATCH] feat: periodic version check and json config (#10438) Co-authored-by: Lucas Molas Co-authored-by: Marcin Rataj --- cmd/ipfs/kubo/daemon.go | 57 ++++++++++- config/config.go | 1 + config/version.go | 14 +++ core/commands/commands_test.go | 1 + core/commands/version.go | 181 +++++++++++++++++++++++++++++++-- docs/changelogs/v0.30.md | 17 ++++ docs/config.md | 40 ++++++++ go.mod | 1 + go.sum | 2 + test/sharness/t0026-id.sh | 11 ++ 10 files changed, 315 insertions(+), 10 deletions(-) create mode 100644 config/version.go diff --git a/cmd/ipfs/kubo/daemon.go b/cmd/ipfs/kubo/daemon.go index 1d6ff5035e4..5dec799dd20 100644 --- a/cmd/ipfs/kubo/daemon.go +++ b/cmd/ipfs/kubo/daemon.go @@ -1,9 +1,11 @@ package kubo import ( + "context" "errors" _ "expvar" "fmt" + "math" "net" "net/http" _ "net/http/pprof" @@ -438,9 +440,11 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment return fmt.Errorf("unrecognized routing option: %s", routingOption) } - agentVersionSuffixString, _ := req.Options[agentVersionSuffix].(string) - if agentVersionSuffixString != "" { - version.SetUserAgentSuffix(agentVersionSuffixString) + // Set optional agent version suffix + versionSuffixFromCli, _ := req.Options[agentVersionSuffix].(string) + versionSuffix := cfg.Version.AgentSuffix.WithDefault(versionSuffixFromCli) + if versionSuffix != "" { + version.SetUserAgentSuffix(versionSuffix) } node, err := core.NewNode(req.Context, ncfg) @@ -610,6 +614,15 @@ take effect. } if len(peers) == 0 { log.Error("failed to bootstrap (no peers found): consider updating Bootstrap or Peering section of your config") + } else { + // After 1 minute we should have enough peers + // to run informed version check + startVersionChecker( + cctx.Context(), + node, + cfg.Version.SwarmCheckEnabled.WithDefault(true), + cfg.Version.SwarmCheckPercentThreshold.WithDefault(config.DefaultSwarmCheckPercentThreshold), + ) } }) } @@ -1056,3 +1069,41 @@ func printVersion() { fmt.Printf("System version: %s\n", runtime.GOARCH+"/"+runtime.GOOS) fmt.Printf("Golang version: %s\n", runtime.Version()) } + +func startVersionChecker(ctx context.Context, nd *core.IpfsNode, enabled bool, percentThreshold int64) { + if !enabled { + return + } + ticker := time.NewTicker(time.Hour) + defer ticker.Stop() + go func() { + for { + o, err := commands.DetectNewKuboVersion(nd, percentThreshold) + if err != nil { + // The version check is best-effort, and may fail in custom + // configurations that do not run standard WAN DHT. If it + // errors here, no point in spamming logs: og once and exit. + log.Errorw("initial version check failed, will not be run again", "error", err) + return + } + if o.UpdateAvailable { + newerPercent := fmt.Sprintf("%.0f%%", math.Round(float64(o.WithGreaterVersion)/float64(o.PeersSampled)*100)) + log.Errorf(` +โš ๏ธ A NEW VERSION OF KUBO DETECTED + +This Kubo node is running an outdated version (%s). +%s of the sampled Kubo peers are running a higher version. +Visit https://github.com/ipfs/kubo/releases or https://dist.ipfs.tech/#kubo and update to version %s or later.`, + o.RunningVersion, newerPercent, o.GreatestVersion) + } + select { + case <-ctx.Done(): + return + case <-nd.Process.Closing(): + return + case <-ticker.C: + continue + } + } + }() +} diff --git a/config/config.go b/config/config.go index 046c930be93..71365eb0b5e 100644 --- a/config/config.go +++ b/config/config.go @@ -37,6 +37,7 @@ type Config struct { Plugins Plugins Pinning Pinning Import Import + Version Version Internal Internal // experimental/unstable options } diff --git a/config/version.go b/config/version.go new file mode 100644 index 00000000000..8096107bb26 --- /dev/null +++ b/config/version.go @@ -0,0 +1,14 @@ +package config + +const DefaultSwarmCheckPercentThreshold = 5 + +// Version allows controling things like custom user agent and update checks. +type Version struct { + // Optional suffix to the AgentVersion presented by `ipfs id` and exposed + // via libp2p identify protocol. + AgentSuffix *OptionalString `json:",omitempty"` + + // Detect when to warn about new version when observed via libp2p identify + SwarmCheckEnabled Flag `json:",omitempty"` + SwarmCheckPercentThreshold *OptionalInteger `json:",omitempty"` +} diff --git a/core/commands/commands_test.go b/core/commands/commands_test.go index 38172fd66d9..018b6734e79 100644 --- a/core/commands/commands_test.go +++ b/core/commands/commands_test.go @@ -199,6 +199,7 @@ func TestCommands(t *testing.T) { "/swarm/resources", "/update", "/version", + "/version/check", "/version/deps", } diff --git a/core/commands/version.go b/core/commands/version.go index e404074fe75..22172688146 100644 --- a/core/commands/version.go +++ b/core/commands/version.go @@ -5,17 +5,25 @@ import ( "fmt" "io" "runtime/debug" + "strings" - version "github.com/ipfs/kubo" - + versioncmp "github.com/hashicorp/go-version" cmds "github.com/ipfs/go-ipfs-cmds" + version "github.com/ipfs/kubo" + "github.com/ipfs/kubo/config" + "github.com/ipfs/kubo/core" + "github.com/ipfs/kubo/core/commands/cmdenv" + "github.com/libp2p/go-libp2p-kad-dht/fullrt" + peer "github.com/libp2p/go-libp2p/core/peer" + pstore "github.com/libp2p/go-libp2p/core/peerstore" ) const ( - versionNumberOptionName = "number" - versionCommitOptionName = "commit" - versionRepoOptionName = "repo" - versionAllOptionName = "all" + versionNumberOptionName = "number" + versionCommitOptionName = "commit" + versionRepoOptionName = "repo" + versionAllOptionName = "all" + versionCheckThresholdOptionName = "min-percent" ) var VersionCmd = &cmds.Command{ @@ -24,7 +32,8 @@ var VersionCmd = &cmds.Command{ ShortDescription: "Returns the current version of IPFS and exits.", }, Subcommands: map[string]*cmds.Command{ - "deps": depsVersionCommand, + "deps": depsVersionCommand, + "check": checkVersionCommand, }, Options: []cmds.Option{ @@ -130,3 +139,161 @@ Print out all dependencies and their versions.`, }), }, } + +const DefaultMinimalVersionFraction = 0.05 // 5% + +type VersionCheckOutput struct { + UpdateAvailable bool + RunningVersion string + GreatestVersion string + PeersSampled int + WithGreaterVersion int +} + +var checkVersionCommand = &cmds.Command{ + Helptext: cmds.HelpText{ + Tagline: "Checks Kubo version against connected peers.", + ShortDescription: ` +This command uses the libp2p identify protocol to check the 'AgentVersion' +of connected peers and see if the Kubo version we're running is outdated. + +Peers with an AgentVersion that doesn't start with 'kubo/' are ignored. +'UpdateAvailable' is set to true only if the 'min-fraction' criteria are met. + +The 'ipfs daemon' does the same check regularly and logs when a new version +is available. You can stop these regular checks by setting +Version.SwarmCheckEnabled:false in the config. +`, + }, + Options: []cmds.Option{ + cmds.IntOption(versionCheckThresholdOptionName, "t", "Percentage (1-100) of sampled peers with the new Kubo version needed to trigger an update warning.").WithDefault(config.DefaultSwarmCheckPercentThreshold), + }, + Type: VersionCheckOutput{}, + + Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error { + nd, err := cmdenv.GetNode(env) + if err != nil { + return err + } + + if !nd.IsOnline { + return ErrNotOnline + } + + minPercent, _ := req.Options[versionCheckThresholdOptionName].(int64) + output, err := DetectNewKuboVersion(nd, minPercent) + if err != nil { + return err + } + + if err := cmds.EmitOnce(res, output); err != nil { + return err + } + return nil + }, +} + +// DetectNewKuboVersion observers kubo version reported by other peers via +// libp2p identify protocol and notifies when threshold fraction of seen swarm +// is running updated Kubo. It is used by RPC and CLI at 'ipfs version check' +// and also periodically when 'ipfs daemon' is running. +func DetectNewKuboVersion(nd *core.IpfsNode, minPercent int64) (VersionCheckOutput, error) { + ourVersion, err := versioncmp.NewVersion(version.CurrentVersionNumber) + if err != nil { + return VersionCheckOutput{}, fmt.Errorf("could not parse our own version %q: %w", + version.CurrentVersionNumber, err) + } + // MAJOR.MINOR.PATCH without any suffix + ourVersion = ourVersion.Core() + + greatestVersionSeen := ourVersion + totalPeersSampled := 1 // Us (and to avoid division-by-zero edge case) + withGreaterVersion := 0 + + recordPeerVersion := func(agentVersion string) { + // We process the version as is it assembled in GetUserAgentVersion + segments := strings.Split(agentVersion, "/") + if len(segments) < 2 { + return + } + if segments[0] != "kubo" { + return + } + versionNumber := segments[1] // As in our CurrentVersionNumber + + peerVersion, err := versioncmp.NewVersion(versionNumber) + if err != nil { + // Do not error on invalid remote versions, just ignore + return + } + + // Ignore prerelases and development releases (-dev, -rcX) + if peerVersion.Metadata() != "" || peerVersion.Prerelease() != "" { + return + } + + // MAJOR.MINOR.PATCH without any suffix + peerVersion = peerVersion.Core() + + // Valid peer version number + totalPeersSampled += 1 + if ourVersion.LessThan(peerVersion) { + withGreaterVersion += 1 + } + if peerVersion.GreaterThan(greatestVersionSeen) { + greatestVersionSeen = peerVersion + } + } + + processPeerstoreEntry := func(id peer.ID) { + if v, err := nd.Peerstore.Get(id, "AgentVersion"); err == nil { + recordPeerVersion(v.(string)) + } else if errors.Is(err, pstore.ErrNotFound) { // ignore noop + } else { // a bug, usually. + log.Errorw("failed to get agent version from peerstore", "error", err) + } + } + + // Amino DHT client keeps information about previously seen peers + if nd.DHTClient != nd.DHT && nd.DHTClient != nil { + client, ok := nd.DHTClient.(*fullrt.FullRT) + if !ok { + return VersionCheckOutput{}, errors.New("could not perform version check due to missing or incompatible DHT configuration") + } + for _, p := range client.Stat() { + processPeerstoreEntry(p) + } + } else if nd.DHT != nil && nd.DHT.WAN != nil { + for _, pi := range nd.DHT.WAN.RoutingTable().GetPeerInfos() { + processPeerstoreEntry(pi.Id) + } + } else if nd.DHT != nil && nd.DHT.LAN != nil { + for _, pi := range nd.DHT.LAN.RoutingTable().GetPeerInfos() { + processPeerstoreEntry(pi.Id) + } + } else { + return VersionCheckOutput{}, errors.New("could not perform version check due to missing or incompatible DHT configuration") + } + + if minPercent < 1 || minPercent > 100 { + if minPercent == 0 { + minPercent = config.DefaultSwarmCheckPercentThreshold + } else { + return VersionCheckOutput{}, errors.New("Version.SwarmCheckPercentThreshold must be between 1 and 100") + } + } + + minFraction := float64(minPercent) / 100.0 + + // UpdateAvailable flag is set only if minFraction was reached + greaterFraction := float64(withGreaterVersion) / float64(totalPeersSampled) + + // Gathered metric are returned every time + return VersionCheckOutput{ + UpdateAvailable: (greaterFraction >= minFraction), + RunningVersion: ourVersion.String(), + GreatestVersion: greatestVersionSeen.String(), + PeersSampled: totalPeersSampled, + WithGreaterVersion: withGreaterVersion, + }, nil +} diff --git a/docs/changelogs/v0.30.md b/docs/changelogs/v0.30.md index e561566d90d..8457662bf84 100644 --- a/docs/changelogs/v0.30.md +++ b/docs/changelogs/v0.30.md @@ -6,6 +6,8 @@ - [Overview](#overview) - [๐Ÿ”ฆ Highlights](#-highlights) + - [Automated `ipfs version check`](#automated-ipfs-version-check) + - [Version Suffix Configuration](#version-suffix-configuration) - [๐Ÿ“ Changelog](#-changelog) - [๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors](#-contributors) @@ -13,6 +15,21 @@ ### ๐Ÿ”ฆ Highlights +#### Automated `ipfs version check` + +Kubo now performs privacy-preserving version checks using the [libp2p identify protocol](https://github.com/libp2p/specs/blob/master/identify/README.md) on peers detected by the Amino DHT client. +If more than 5% of Kubo peers seen by your node are running a newer version, you will receive a log message notification. + +- For manual checks, refer to `ipfs version check --help` for details. +- To disable automated checks, set [`Version.SwarmCheckEnabled`](https://github.com/ipfs/kubo/blob/master/docs/config.md#versionswarmcheckenabled) to `false`. + +#### Version Suffix Configuration + +Defining the optional agent version suffix is now simpler. The [`Version.AgentSuffix`](https://github.com/ipfs/kubo/blob/master/docs/config.md#agentsuffix) value from the Kubo config takes precedence over any value provided via `ipfs daemon --agent-version-suffix` (which is still supported). + +> [!NOTE] +> Setting a custom version suffix helps with ecosystem analysis, such as Amino DHT reports published at https://stats.ipfs.network + ### ๐Ÿ“ Changelog ### ๐Ÿ‘จโ€๐Ÿ‘ฉโ€๐Ÿ‘งโ€๐Ÿ‘ฆ Contributors diff --git a/docs/config.md b/docs/config.md index 130f724d5d2..c7355072d4b 100644 --- a/docs/config.md +++ b/docs/config.md @@ -180,6 +180,10 @@ config file at runtime. - [`Import.UnixFSRawLeaves`](#importunixfsrawleaves) - [`Import.UnixFSChunker`](#importunixfschunker) - [`Import.HashFunction`](#importhashfunction) + - [`Version`](#version) + - [`Version.AgentSuffix`](#versionagentsuffix) + - [`Version.SwarmCheckEnabled`](#versionswarmcheckenabled) + - [`Version.SwarmCheckPercentThreshold`](#versionswarmcheckpercentthreshold) ## Profiles @@ -2435,3 +2439,39 @@ The default hash function. Commands affected: `ipfs add`, `ipfs block put`, `ipf Default: `sha2-256` Type: `optionalString` + +## `Version` + +Options to configure agent version announced to the swarm, and leveraging +other peers version for detecting when there is time to update. + +### `Version.AgentSuffix` + +Optional suffix to the AgentVersion presented by `ipfs id` and exposed via [libp2p identify protocol](https://github.com/libp2p/specs/blob/master/identify/README.md#agentversion). + +The value from config takes precedence over value passed via `ipfs daemon --agent-version-suffix`. + +> [!NOTE] +> Setting a custom version suffix helps with ecosystem analysis, such as Amino DHT reports published at https://stats.ipfs.network + +Default: `""` (no suffix, or value from `ipfs daemon --agent-version-suffix=`) + +Type: `optionalString` + +### `Version.SwarmCheckEnabled` + +Observe the AgentVersion of swarm peers and log warning when +`SwarmCheckPercentThreshold` of peers runs version higher than this node. + +Default: `true` + +Type: `flag` + +### `Version.SwarmCheckPercentThreshold` + +Control the percentage of `kubo/` peers running new version required to +trigger update warning. + +Default: `5` + +Type: `optionalInteger` (1-100) diff --git a/go.mod b/go.mod index 866a1ecc622..b466bc1f9c8 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/fsnotify/fsnotify v1.6.0 github.com/google/uuid v1.6.0 github.com/hashicorp/go-multierror v1.1.1 + github.com/hashicorp/go-version v1.6.0 github.com/ipfs-shipyard/nopfs v0.0.12 github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c github.com/ipfs/boxo v0.21.0 diff --git a/go.sum b/go.sum index 52e889275a9..a42aaa22c3d 100644 --- a/go.sum +++ b/go.sum @@ -310,6 +310,8 @@ github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= diff --git a/test/sharness/t0026-id.sh b/test/sharness/t0026-id.sh index d4248c56295..992892a39a6 100755 --- a/test/sharness/t0026-id.sh +++ b/test/sharness/t0026-id.sh @@ -65,5 +65,16 @@ iptb stop test_kill_ipfs_daemon +# Version.AgentSuffix overrides --agent-version-suffix (local, offline) +test_expect_success "setting Version.AgentSuffix in config" ' + ipfs config Version.AgentSuffix json-config-suffix +' +test_launch_ipfs_daemon --agent-version-suffix=ignored-cli-suffix +test_expect_success "checking AgentVersion with suffix set via JSON config" ' + test_id_compute_agent json-config-suffix > expected-agent-version && + ipfs id -f "\n" > actual-agent-version && + test_cmp expected-agent-version actual-agent-version +' +test_kill_ipfs_daemon test_done