From 8e67d148d04755268277c8825dfac5cc54bdc8dd Mon Sep 17 00:00:00 2001 From: Ljubisa Date: Thu, 12 Sep 2024 14:05:27 +0200 Subject: [PATCH 1/6] fix(k8s): revert back deleted function setInitContainers --- pkg/orchestration/k8s/helpers.go | 18 ++++++++++++++++++ pkg/orchestration/k8s/node_orchestrator.go | 1 + 2 files changed, 19 insertions(+) diff --git a/pkg/orchestration/k8s/helpers.go b/pkg/orchestration/k8s/helpers.go index 628d1a45..3e1939fd 100644 --- a/pkg/orchestration/k8s/helpers.go +++ b/pkg/orchestration/k8s/helpers.go @@ -56,6 +56,24 @@ withdrawal-addresses-whitelist: {{.WithdrawAddress}} ` ) +func setInitContainers() (inits containers.Containers) { + inits = append(inits, containers.Container{ + Name: "init-bee", + Image: "ethersphere/busybox:1.33", + Command: []string{"sh", "-c", `mkdir -p /home/bee/.bee/keys; +chown -R 999:999 /home/bee/.bee/keys; +echo 'bee initialization done';`}, + VolumeMounts: containers.VolumeMounts{ + { + Name: "data", + MountPath: "home/bee/.bee", + }, + }, + }) + + return +} + type setContainersOptions struct { Name string Image string diff --git a/pkg/orchestration/k8s/node_orchestrator.go b/pkg/orchestration/k8s/node_orchestrator.go index b9db858c..dd4c5cec 100644 --- a/pkg/orchestration/k8s/node_orchestrator.go +++ b/pkg/orchestration/k8s/node_orchestrator.go @@ -267,6 +267,7 @@ func (n *nodeOrchestrator) Create(ctx context.Context, o orchestration.CreateOpt Annotations: o.Annotations, Labels: o.Labels, Spec: pod.PodSpec{ + InitContainers: setInitContainers(), Containers: setContainers(setContainersOptions{ Name: sSet, Image: o.Image, From 1107eb3b6e5c7c75dd19fb40476b09dc7f69c731 Mon Sep 17 00:00:00 2001 From: Ljubisa Date: Thu, 19 Sep 2024 20:54:08 +0200 Subject: [PATCH 2/6] chore: update config to fit deployment on hetzner k8s --- cmd/beekeeper/cmd/cluster.go | 6 +- config/beekeeper-local.yaml | 2 +- config/config.yaml | 251 +++++++++++++++++---------------- pkg/check/balances/balances.go | 58 ++++---- 4 files changed, 163 insertions(+), 154 deletions(-) diff --git a/cmd/beekeeper/cmd/cluster.go b/cmd/beekeeper/cmd/cluster.go index a0664a99..04b4af61 100644 --- a/cmd/beekeeper/cmd/cluster.go +++ b/cmd/beekeeper/cmd/cluster.go @@ -144,8 +144,7 @@ func (c *command) setupCluster(ctx context.Context, clusterName string, cfg *con // fund bootnode node group if cluster is started if startCluster { - err = fund(ctx, fundAddresses, chainNodeEndpoint, walletKey, fundOpts, c.log) - if err != nil { + if err = fund(ctx, fundAddresses, chainNodeEndpoint, walletKey, fundOpts, c.log); err != nil { return nil, fmt.Errorf("funding node group bootnode: %w", err) } c.log.Infof("bootnode node group funded") @@ -159,8 +158,7 @@ func (c *command) setupCluster(ctx context.Context, clusterName string, cfg *con // fund other node groups if cluster is started if startCluster { - err = fund(ctx, fundAddresses, chainNodeEndpoint, walletKey, fundOpts, c.log) - if err != nil { + if err = fund(ctx, fundAddresses, chainNodeEndpoint, walletKey, fundOpts, c.log); err != nil { return nil, fmt.Errorf("fund other node groups: %w", err) } c.log.Infof("node groups funded") diff --git a/config/beekeeper-local.yaml b/config/beekeeper-local.yaml index adb91c32..1c32bc33 100644 --- a/config/beekeeper-local.yaml +++ b/config/beekeeper-local.yaml @@ -1,6 +1,6 @@ enable-k8s: true in-cluster: false -geth-url: http://geth-swap.localhost +geth-url: http://geth-swap.localhost #http://geth-swap.bee-playground.testnet.ethswarm.org bzz-token-address: 0x6aab14fe9cccd64a502d23842d916eb5321c26e7 eth-account: 0x62cab2b3b55f341f10348720ca18063cdb779ad5 wallet-key: 4663c222787e30c1994b59044aa5045377a6e79193a8ead88293926b535c722d diff --git a/config/config.yaml b/config/config.yaml index 943d0afe..1889b6af 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -4,12 +4,12 @@ clusters: default: _inherit: "" name: bee - namespace: beekeeper + namespace: bee-playground disable-namespace: false use-static-endpoints: false - api-domain: staging.internal + api-domain: testnet.ethswarm.org api-insecure-tls: true - api-scheme: https + api-scheme: http admin-password: test funding: eth: 0.1 @@ -19,19 +19,14 @@ clusters: mode: bootnode endpoints: - name: bootnode-0 - api-url: https://bootnode-0.beekeeper.testnet.internal - debug-api-url: https://bootnode-0-debug.beekeeper.testnet.internal + api-url: http://bootnode-0.bee-playground.testnet.ethswarm.org bee-config: bootnode config: default nodes: - name: bootnode-0 - bootnodes: /dns4/bootnode-1-headless.%s.svc.cluster.local/tcp/1634/p2p/16Uiu2HAmMw7Uj6vfraD9BYx3coDs6MK6pAmActE8fsfaZwigsaB6 - libp2p-key: '{"address":"aa6675fb77f3f84304a00d5ea09902d8a500364091a457cf21e05a41875d48f7","crypto":{"cipher":"aes-128-ctr","ciphertext":"93effebd3f015f496367e14218cb26d22de8f899e1d7b7686deb6ab43c876ea5","cipherparams":{"iv":"627434462c2f960d37338022d27fc92e"},"kdf":"scrypt","kdfparams":{"n":32768,"r":8,"p":1,"dklen":32,"salt":"a59e72e725fe3de25dd9c55aa55a93ed0e9090b408065a7204e2f505653acb70"},"mac":"dfb1e7ad93252928a7ff21ea5b65e8a4b9bda2c2e09cb6a8ac337da7a3568b8c"},"version":3}' + bootnodes: /dns4/bootnode-0-headless.%s.svc.swarm1.local/tcp/1634/p2p/QmaHzvd3iZduu275CMkMVZKwbsjXSyH3GJRj4UvFJApKcb + libp2p-key: '{"address":"28678fe31f09f722d53e77ca2395569f19959fa5","crypto":{"cipher":"aes-128-ctr","ciphertext":"0ff319684c4f8decf9c998047febe3417cfc45832b8bb62fd818183d54cf5d0183bfa021ed95addce3b33e83ce7ee73e926f00eea8241d96b349266a4d299829d3d22db0d536315b52b34db4a6778bfd3ce7631ad7256ea0bb9c50abea9de35d740b6fdc50caf929b1d19494690d9ed649105d02c14f5ec49d","cipherparams":{"iv":"4e9a50fb5852b5e61964f696be78066b"},"kdf":"scrypt","kdfparams":{"n":32768,"r":8,"p":1,"dklen":32,"salt":"4d513e81647e4150bb648ed8d2dda28d460802336bf24d620119eac66ae0c0c4"},"mac":"9ae71db96e5ddc1c214538d42082212bbbe53aeac09fcc3e3a8eff815648331e"},"version":3,"id":"ae3bc991-d89f-405a-9e6a-60e27347e22d"}' swarm-key: '{"address":"f176839c150e52fe30e5c2b5c648465c6fdfa532","crypto":{"cipher":"aes-128-ctr","ciphertext":"352af096f0fca9dfbd20a6861bde43d988efe7f179e0a9ffd812a285fdcd63b9","cipherparams":{"iv":"613003f1f1bf93430c92629da33f8828"},"kdf":"scrypt","kdfparams":{"n":32768,"r":8,"p":1,"dklen":32,"salt":"ad1d99a4c64c95c26131e079e8c8a82221d58bf66a7ceb767c33a4c376c564b8"},"mac":"cafda1bc8ca0ffc2b22eb69afd1cf5072fd09412243443be1b0c6832f57924b6"},"version":3}' - - name: bootnode-1 - bootnodes: /dns4/bootnode-0-headless.%s.svc.cluster.local/tcp/1634/p2p/16Uiu2HAm6i4dFaJt584m2jubyvnieEECgqM2YMpQ9nusXfy8XFzL - libp2p-key: '{"address":"03348ecf3adae1d05dc16e475a83c94e49e28a4d3c7db5eccbf5ca4ea7f688ddcdfe88acbebef2037c68030b1a0a367a561333e5c5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470","crypto":{"cipher":"aes-128-ctr","ciphertext":"0d0ff25e9b03292e622c5a09ec00c2acb7ff5882f02dd2f00a26ac6d3292a434","cipherparams":{"iv":"cd4082caf63320b306fe885796ba224f"},"kdf":"scrypt","kdfparams":{"n":32768,"r":8,"p":1,"dklen":32,"salt":"a4d63d56c539eb3eff2a235090127486722fa2c836cf735d50d673b730cebc3f"},"mac":"aad40da9c1e742e2b01bb8f76ba99ace97ccb0539cea40e31eb6b9bb64a3f36a"},"version":3}' - swarm-key: '{"address":"ebe269e07161c68a942a3a7fce6b4ed66867d6f0","crypto":{"cipher":"aes-128-ctr","ciphertext":"06b550c35b46099aea8f6c9f799497d34bd5ebc13af79c7cdb2a1037227544ad","cipherparams":{"iv":"fa088e69b1849e40f190a5f69f0555f8"},"kdf":"scrypt","kdfparams":{"n":32768,"r":8,"p":1,"dklen":32,"salt":"42b4f2815c0042d02eed916a7a74ecdc005f1f7eae0cfb5837c15f469df9ddba"},"mac":"23e3d0594ab94587258a33cc521edbde009b887a6f117ed7a3422d1c95123568"},"version":3}' bee: mode: node bee-config: default @@ -39,14 +34,11 @@ clusters: count: 3 endpoints: - name: bee-0 - api-url: https://bee-0.beekeeper.testnet.internal - debug-api-url: https://bee-0-debug.beekeeper.testnet.internal + api-url: http://bee-0.bee-playground.testnet.ethswarm.org - name: bee-1 - api-url: https://bee-1.beekeeper.testnet.internal - debug-api-url: https://bee-1-debug.beekeeper.testnet.internal + api-url: http://bee-1.bee-playground.testnet.ethswarm.org - name: bee-2 - api-url: https://bee-2.beekeeper.testnet.internal - debug-api-url: https://bee-2-debug.beekeeper.testnet.internal + api-url: http://bee-2.bee-playground.testnet.ethswarm.org light: mode: node bee-config: light-node @@ -54,14 +46,11 @@ clusters: count: 3 endpoints: - name: light-0 - api-url: https://light-0.beekeeper.testnet.internal - debug-api-url: https://light-0-debug.beekeeper.testnet.internal + api-url: http://light-0.bee-playground.testnet.ethswarm.org - name: light-1 - api-url: https://light-1.beekeeper.testnet.internal - debug-api-url: https://light-1-debug.beekeeper.testnet.internal + api-url: http://light-1.bee-playground.testnet.ethswarm.org - name: light-2 - api-url: https://light-2.beekeeper.testnet.internal - debug-api-url: https://light-2-debug.beekeeper.testnet.internal + api-url: http://light-2.bee-playground.testnet.ethswarm.org # node-groups defines node groups that can be registered in the cluster # node-groups may inherit it's configuration from already defined node-group and override specific fields from it @@ -70,7 +59,7 @@ node-groups: _inherit: "" image: ethersphere/bee:latest image-pull-policy: Always - image-pull-secrets: [regcred] + # image-pull-secrets: [regcred] ingress-annotations: nginx.ingress.kubernetes.io/affinity: "cookie" nginx.ingress.kubernetes.io/affinity-mode: "persistent" @@ -81,14 +70,15 @@ node-groups: nginx.ingress.kubernetes.io/session-cookie-name: "SWARMGATEWAY" nginx.ingress.kubernetes.io/session-cookie-path: "default" nginx.ingress.kubernetes.io/ssl-redirect: "true" - ingress-class: "nginx-internal" + nginx.ingress.kubernetes.io/whitelist-source-range: "109.93.45.225/32" #curl ifconfig.me/ip + ingress-class: "nginx" labels: app.kubernetes.io/component: "node" app.kubernetes.io/name: "bee" app.kubernetes.io/part-of: "bee" app.kubernetes.io/version: "latest" - node-selector: - node-group: "private" + # node-selector: + # node-group: "private" persistence-enabled: false persistence-storage-class: "local-storage" persistence-storage-request: "34Gi" @@ -119,7 +109,7 @@ bee-configs: full-node: true nat-addr: "" mainnet: false - network-id: 12345 + network-id: 0 p2p-addr: ":1634" p2p-ws-enable: false password: "beekeeper" @@ -128,23 +118,24 @@ bee-configs: payment-tolerance-percent: 25 storage-incentives-enable: true postage-stamp-start-block: 1 - postage-stamp-address: "0x538e6de1d876bbcd5667085257bc92f7c808a0f3" - price-oracle-address: "0xfc28330f1ece0ef2371b724e0d19c1ee60b728b2" - staking-address: "0xc0332d319b73cfeed89511fe1a83add141bd6f12" - redistribution-address: "0x56e442f99cafd7c890c404c1f8c16d38a569e41a" + postage-stamp-address: "0x657241f4494A2F15Ba75346E691d753A978C72Df" + price-oracle-address: "0x5aFE06fcC0855a76a15c3544b0886EDBE3294d62" + staking-address: "0xfc28330f1ecE0ef2371B724E0D19c1EE60B728b2" + redistribution-address: "0x09Ad42a7d020244920309FfA14EA376dd2D3b7d5" resolver-options: "" chequebook-enable: true swap-enable: true - swap-endpoint: "ws://geth-swap.geth-swap:8546" - swap-factory-address: "0x09ad42a7d020244920309ffa14ea376dd2d3b7d5" + swap-endpoint: "ws://geth-swap:8546" + swap-factory-address: "0xdD661f2500bA5831e3d1FEbAc379Ea1bF80773Ac" swap-initial-deposit: 500000000000000000 - tracing-enabled: true + tracing-enabled: false tracing-endpoint: "tempo-tempo-distributed-distributor.observability:6831" tracing-service-name: "bee" verbosity: 5 welcome-message: "Welcome to the Swarm, you are Bee-ing connected!" + warmup-time: 0s allow-private-cidrs: true - withdrawal-addresses-whitelist: "0xec44cb15b1b033e74d55ac5d0e24d861bde54532" + withdrawal-addresses-whitelist: 0xec44cb15b1b033e74d55ac5d0e24d861bde54532 bootnode: _inherit: "default" bootnode-mode: true @@ -152,6 +143,15 @@ bee-configs: # checks defines checks Beekeeper can execute against the cluster # type filed allows defining same check with different names and options checks: + act: + options: + file-size: 1024 + postage-depth: 20 + postage-amount: 420000000 + postage-label: act-label + seed: 0 + timeout: 5m + type: act balances: options: dry-run: false @@ -168,6 +168,12 @@ checks: node-group: bee timeout: 5m type: cashout + datadurability: + options: + ref: + concurrency: + max-attempts: + type: datadurability file-retrieval: options: file-name: file-retrieval @@ -193,21 +199,35 @@ checks: options: cache-size: 10 reserve-size: 16 - timeout: 10m + timeout: 5m type: gc kademlia: options: dynamic: false timeout: 5m type: kademlia + longavailability: + options: + refs: + retry-wait: 1m + retry-count: 3 + type: longavailability manifest: options: files-in-collection: 10 max-pathname-length: 64 postage-amount: 1000 - postage-depth: 16 + postage-depth: 17 timeout: 5m type: manifest + networkavailability: + options: + rnd-seed: + postage-amount: 50000000 + postage-depth: 24 + sleep-duration: 1h + timeout: 5m + type: networkavailability peer-count: timeout: 5m type: peer-count @@ -215,27 +235,20 @@ checks: options: timeout: 5m type: pingpong - act: - options: - file-name: act - file-size: 1024 - postage-depth: 20 - postage-amount: 420000000 - postage-label: act-label - seed: 0 + postage: + type: postage timeout: 5m - type: act - withdraw: options: - target-address: 0xec44cb15b1b033e74d55ac5d0e24d861bde54532 - timeout: 5m - type: withdraw + postage-amount: 1000 + postage-depth: 17 + postage-topup-amount: 100 + postage-new-depth: 18 pss: options: + count: 3 address-prefix: 2 - node-count: 3 postage-amount: 1000 - postage-depth: 16 + postage-depth: 17 request-timeout: 5m timeout: 5m type: pss @@ -260,36 +273,45 @@ checks: type: pushsync pushsync-chunks: options: - chunks-per-node: 1 + chunks-per-node: 3 mode: chunks postage-amount: 1000 + postage-depth: 17 + retries: 5 + retry-delay: 15s + upload-node-count: 3 exclude-node-group: - light - postage-depth: 16 - retries: 5 - retry-delay: 1s - upload-node-count: 1 + - bootnode timeout: 5m type: pushsync pushsync-light-chunks: options: - chunks-per-node: 1 + chunks-per-node: 3 mode: light-chunks - postage-amount: 1000 + postage-amount: 10000 + postage-depth: 17 + retries: 5 + retry-delay: 30s + upload-node-count: 3 exclude-node-group: - light - postage-depth: 16 - retries: 5 - retry-delay: 1s - upload-node-count: 1 + - bootnode timeout: 5m type: pushsync + redundancy: + options: + postage-amount: + postage-depth: + seed: + data-size: + type: redundancy retrieval: options: - chunks-per-node: 1 + chunks-per-node: 3 postage-amount: 1000 - postage-depth: 16 - upload-node-count: 1 + postage-depth: 17 + upload-node-count: 3 timeout: 5m type: retrieval settlements: @@ -297,44 +319,25 @@ checks: dry-run: false expect-settlements: true file-name: settlements - file-size: 1048576 # 1mb = 1*1024*1024 + file-size: 14680064 # 14mb = 14*1024*1024 postage-amount: 1000 - postage-depth: 16 - threshold: 10000000 + postage-depth: 20 + threshold: 100000000 upload-node-count: 3 - wait-before-download: 5s - timeout: 5m + wait-before-download: 15s + timeout: 15m type: settlements smoke: options: - content-size: 5000000 + content-size: 5000 postage-amount: 1000000 postage-depth: 20 nodes-sync-wait: 1m - duration: 12h - upload-timeout: 5m - download-timeout: 5m - timeout: 5m + upload-timeout: 1m + download-timeout: 1m + duration: 15m + timeout: 30m type: smoke - load: - options: - content-size: 5000000 - postage-amount: 1000000 - gas-price: "10000000000" - postage-depth: 20 - nodes-sync-wait: 1m - duration: 12h - downloader-count: 3 - uploader-count: 3 - max-storage-radius: 2 - storage-radius-check-wait: 5m - upload-groups: - - gateway - download-groups: - - bee - - light - timeout: 12h - type: load soc: options: postage-amount: 1000 @@ -342,38 +345,45 @@ checks: request-timeout: 5m timeout: 5m type: soc - postage: - type: postage - timeout: 5m - options: - postage-amount: 1000 - postage-depth: 17 - postage-topup-amount: 100 - postage-new-depth: 18 stake: type: stake timeout: 5m options: amount: 1000000000000000000 - longavailability: + insufficient-amount: 102400 + contract-addr: "0xfc28330f1ecE0ef2371B724E0D19c1EE60B728b2" + private-key: "4663c222787e30c1994b59044aa5045377a6e79193a8ead88293926b535c722d" + geth-url: "http://geth-swap.bee-playground.testnet.ethswarm.org" + geth-chain-id: 12345 + withdraw: options: - refs: - retry-wait: 1m - retry-count: 3 - type: longavailability - datadurability: + target-address: 0xec44cb15b1b033e74d55ac5d0e24d861bde54532 + timeout: 5m + type: withdraw + content-availability: + type: content-availability + timeout: 5m options: - ref: - concurrency: - max-attempts: - type: datadurability - redundancy: + content-size: 16384 + postage-amount: 1000 + postage-depth: 16 + load: options: - postage-amount: - postage-depth: - seed: - data-size: - type: redundancy + content-size: 500000 #5000000 + postage-amount: 1000000 + gas-price: 10000000000 + postage-depth: 20 + nodes-sync-wait: 2m + duration: 12h + downloader-count: 3 + uploader-count: 2 + max-storage-radius: 2 + upload-groups: + - bee + download-groups: + - light + timeout: 12h + type: load # simulations defines simulations Beekeeper can execute against the cluster # type filed allows defining same simulation with different names and options @@ -392,7 +402,6 @@ simulations: timeout: 60s upload-node-name: upload-node-percentage: 50 - sync-upload: false timeout: 5m type: upload retrieval: diff --git a/pkg/check/balances/balances.go b/pkg/check/balances/balances.go index dbed0720..25bfb214 100644 --- a/pkg/check/balances/balances.go +++ b/pkg/check/balances/balances.go @@ -46,13 +46,13 @@ var _ beekeeper.Action = (*Check)(nil) // Check instance type Check struct { - logger logging.Logger + log logging.Logger } // NewCheck returns new check -func NewCheck(logger logging.Logger) beekeeper.Action { +func NewCheck(log logging.Logger) beekeeper.Action { return &Check{ - logger: logger, + log: log, } } @@ -63,8 +63,8 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int } if o.DryRun { - c.logger.Info("running balances (dry mode)") - return dryRun(ctx, cluster, o, c.logger) + c.log.Info("running balances (dry mode)") + return dryRun(ctx, cluster, c.log) } var checkCase *test.CheckCase @@ -79,7 +79,7 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int Seed: o.Seed, } - if checkCase, err = test.NewCheckCase(ctx, cluster, caseOpts, c.logger); err != nil { + if checkCase, err = test.NewCheckCase(ctx, cluster, caseOpts, c.log); err != nil { return err } @@ -89,11 +89,11 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int } // initial validation - if err := validateBalances(balances, c.logger); err != nil { + if err := validateBalances(balances, c.log); err != nil { return fmt.Errorf("invalid initial balances: %v", err) } - c.logger.Info("Balances are valid") + c.log.Info("Balances are valid") // repeats for i := 0; i < o.UploadNodeCount; i++ { @@ -110,7 +110,7 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int return err } - if err := expectBalancesHaveChanged(ctx, balances, newBalances, c.logger); err != nil { + if err := expectBalancesHaveChanged(balances, newBalances, c.log); err != nil { return err } @@ -126,7 +126,7 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int return err } - if err := expectBalancesHaveChanged(ctx, balances, newBalances, c.logger); err != nil { + if err := expectBalancesHaveChanged(balances, newBalances, c.log); err != nil { return err } } @@ -135,35 +135,37 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int } // dryRun executes balances validation check without files uploading/downloading -func dryRun(ctx context.Context, cluster orchestration.Cluster, o Options, logger logging.Logger) (err error) { +func dryRun(ctx context.Context, cluster orchestration.Cluster, log logging.Logger) (err error) { balances, err := cluster.Balances(ctx) if err != nil { return err } flatBalances := flattenBalances(balances) - if err := validateBalances(flatBalances, logger); err != nil { + if err := validateBalances(flatBalances, log); err != nil { return fmt.Errorf("invalid balances") } - logger.Info("Balances are valid") + log.Info("Balances are valid") return } -func expectBalancesHaveChanged(ctx context.Context, balances, newBalances orchestration.NodeGroupBalances, logger logging.Logger) error { +func expectBalancesHaveChanged(balances, newBalances orchestration.NodeGroupBalances, log logging.Logger) error { for t := 0; t < 5; t++ { - time.Sleep(2 * time.Duration(t) * time.Second) + sleepTime := 2 * time.Duration(t) * time.Second + log.Infof("Waiting %s before checking balances", sleepTime) + time.Sleep(sleepTime) - balancesHaveChanged(newBalances, balances, logger) + balancesHaveChanged(newBalances, balances, log) - if err := validateBalances(newBalances, logger); err != nil { - logger.Info("Invalid balances after downloading a file:", err) - logger.Info("Retrying ...", t) + if err := validateBalances(newBalances, log); err != nil { + log.Info("Invalid balances after downloading a file:", err) + log.Info("Retrying ...", t) continue } - logger.Info("Balances are valid") + log.Info("Balances are valid") break } @@ -172,17 +174,17 @@ func expectBalancesHaveChanged(ctx context.Context, balances, newBalances orches } // validateBalances checks balances symmetry -func validateBalances(balances map[string]map[string]int64, logger logging.Logger) (err error) { +func validateBalances(balances map[string]map[string]int64, log logging.Logger) (err error) { var noSymmetry bool for node, v := range balances { for peer, balance := range v { diff := balance + balances[peer][node] if diff != 0 { - logger.Infof("Node %s has asymmetric balance with peer %s", node, peer) - logger.Infof("Node %s has balance %d with peer %s", node, balance, peer) - logger.Infof("Peer %s has balance %d with node %s", peer, balances[peer][node], node) - logger.Infof("Difference: %d", diff) + log.Infof("Node %s has asymmetric balance with peer %s", node, peer) + log.Infof("Node %s has balance %d with peer %s", node, balance, peer) + log.Infof("Peer %s has balance %d with node %s", peer, balances[peer][node], node) + log.Infof("Difference: %d", diff) noSymmetry = true } } @@ -195,16 +197,16 @@ func validateBalances(balances map[string]map[string]int64, logger logging.Logge } // balancesHaveChanged checks if balances have changed -func balancesHaveChanged(current, previous orchestration.NodeGroupBalances, logger logging.Logger) { +func balancesHaveChanged(current, previous orchestration.NodeGroupBalances, log logging.Logger) { for node, v := range current { for peer, balance := range v { if balance != previous[node][peer] { - logger.Info("Balances have changed") + log.Info("Balances have changed") return } } } - logger.Info("Balances have not changed") + log.Info("Balances have not changed") } // flattenBalances convenience function From 04b58a9915db8997ab37433eca00ada97d37b487 Mon Sep 17 00:00:00 2001 From: Ljubisa Date: Fri, 20 Sep 2024 14:14:09 +0200 Subject: [PATCH 3/6] refactor(operator): use json decode to get bee address --- pkg/k8s/pod/client.go | 15 ++++++--------- pkg/k8s/statefulset/client.go | 6 +++--- pkg/operator/operator.go | 16 +++++----------- 3 files changed, 14 insertions(+), 23 deletions(-) diff --git a/pkg/k8s/pod/client.go b/pkg/k8s/pod/client.go index 9a08e058..d7586b23 100644 --- a/pkg/k8s/pod/client.go +++ b/pkg/k8s/pod/client.go @@ -73,14 +73,13 @@ func (c *Client) Delete(ctx context.Context, name, namespace string) (err error) return } -// EventsWatch watches for events. -func (c *Client) EventsWatch(ctx context.Context, namespace string, operatorChan chan string) (err error) { +// WatchNewRunning detects new running Pods in the namespace and sends their IPs to the channel. +func (c *Client) WatchNewRunning(ctx context.Context, namespace string, newPodIps chan string) (err error) { c.log.Infof("starting events watch") defer c.log.Infof("events watch done") - defer close(operatorChan) + defer close(newPodIps) watcher, err := c.clientset.CoreV1().Pods(namespace).Watch(ctx, metav1.ListOptions{ - // TODO: add this label to beekeeper and filter on it => app.kubernetes.io/name=bee LabelSelector: "app.kubernetes.io/name=bee", }) if err != nil { @@ -88,7 +87,6 @@ func (c *Client) EventsWatch(ctx context.Context, namespace string, operatorChan } defer watcher.Stop() - // listen for either events from the watcher or a context cancellation for { select { case <-ctx.Done(): @@ -98,13 +96,12 @@ func (c *Client) EventsWatch(ctx context.Context, namespace string, operatorChan return fmt.Errorf("watch channel closed") } switch event.Type { - // case watch.Added: // if we want to do something with already running pods + // case watch.Added: // already running pods case watch.Modified: pod, ok := event.Object.(*v1.Pod) if ok { - if pod.Status.PodIP != "" && pod.ObjectMeta.DeletionTimestamp == nil { - c.log.Tracef("new pod event: {%s}, {%s}, {%s}, {%s}, {%v}", event.Type, pod.Name, pod.Status.Phase, pod.Status.PodIP, pod.ObjectMeta.DeletionTimestamp) - operatorChan <- pod.Status.PodIP + if pod.Status.PodIP != "" && pod.ObjectMeta.DeletionTimestamp == nil && pod.Status.Phase == v1.PodRunning { + newPodIps <- pod.Status.PodIP } } } diff --git a/pkg/k8s/statefulset/client.go b/pkg/k8s/statefulset/client.go index 812d5baf..d99b6c12 100644 --- a/pkg/k8s/statefulset/client.go +++ b/pkg/k8s/statefulset/client.go @@ -16,14 +16,14 @@ import ( // Client manages communication with the Kubernetes StatefulSet. type Client struct { clientset kubernetes.Interface - logger logging.Logger + log logging.Logger } // NewClient constructs a new Client. func NewClient(clientset kubernetes.Interface, l logging.Logger) *Client { return &Client{ clientset: clientset, - logger: l, + log: l, } } @@ -80,7 +80,7 @@ func (c *Client) ReadyReplicasWatch(ctx context.Context, name, namespace string) case <-ctx.Done(): return 0, ctx.Err() case <-ticker.C: - c.logger.Infof("%s is not ready yet", name) + c.log.Infof("%s is not ready yet", name) case event, ok := <-watcher.ResultChan(): if !ok { return 0, fmt.Errorf("watch channel closed") diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 50512b55..b6338247 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -88,16 +88,15 @@ func (c *Client) Run(ctx context.Context) error { }, nil, nil, funder.WithLoggerOption(c.Log)) if err != nil { c.Log.Errorf("funder: %v", err) - continue } } } }() - err := c.K8sClient.Pods.EventsWatch(ctx, c.Namespace, operatorChan) - if err != nil { + if err := c.K8sClient.Pods.WatchNewRunning(ctx, c.Namespace, operatorChan); err != nil { return fmt.Errorf("events watch: %v", err) } + return nil } @@ -121,15 +120,10 @@ func (c *Client) processPodIP(ctx context.Context, podIp string) (bee.Addresses, } defer resp.Body.Close() - body, err := io.ReadAll(resp.Body) - if err != nil { - return bee.Addresses{}, fmt.Errorf("read body: %s", err.Error()) - } - var addresses bee.Addresses - err = json.Unmarshal(body, &addresses) - if err != nil { - return bee.Addresses{}, fmt.Errorf("unmarshal body: %s", err.Error()) + + if err = json.NewDecoder(resp.Body).Decode(&addresses); err != nil { + return bee.Addresses{}, fmt.Errorf("decode body: %s", err.Error()) } return addresses, nil From 706a3fca41c9e1008355f176c33075a0966ca352 Mon Sep 17 00:00:00 2001 From: Ljubisa Date: Thu, 24 Oct 2024 15:55:35 +0200 Subject: [PATCH 4/6] chore: update config --- config/beekeeper-local.yaml | 14 +++++++++----- config/config.yaml | 33 ++++++++++++++------------------- config/light-node.yaml | 2 +- pkg/check/balances/balances.go | 18 +++++++++--------- pkg/test/case.go | 4 ++-- version.go | 2 +- 6 files changed, 36 insertions(+), 37 deletions(-) diff --git a/config/beekeeper-local.yaml b/config/beekeeper-local.yaml index 1c32bc33..3af979e8 100644 --- a/config/beekeeper-local.yaml +++ b/config/beekeeper-local.yaml @@ -1,14 +1,18 @@ +#config-dir: $HOME/.beekeeper +#config-git-repo: "" +#config-git-branch: main +#config-git-username: +#config-git-password: enable-k8s: true in-cluster: false -geth-url: http://geth-swap.localhost #http://geth-swap.bee-playground.testnet.ethswarm.org -bzz-token-address: 0x6aab14fe9cccd64a502d23842d916eb5321c26e7 +kubeconfig: "~/.kube/config" +geth-url: http://geth-swap.localhost #http://geth-swap.bee-playground.testnet.internal +bzz-token-address: 0x6aab14fe9cccd64a502d23842d916eb5321c26e7 #0x543dDb01Ba47acB11de34891cD86B675F04840db Sepolia eth-account: 0x62cab2b3b55f341f10348720ca18063cdb779ad5 wallet-key: 4663c222787e30c1994b59044aa5045377a6e79193a8ead88293926b535c722d -kubeconfig: "~/.kube/config" -# config-dir: "" log-verbosity: "info" tracing-enable: false -tracing-endpoint: tempo-tempo-distributed-distributor.observability:6831 +tracing-endpoint: 10.10.11.199:6831 # tracing-host: 127.0.0.1 # tracing-port: 6831 tracing-service-name: beekeeper diff --git a/config/config.yaml b/config/config.yaml index fdd1b28d..1b4c9991 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -7,7 +7,7 @@ clusters: namespace: bee-playground disable-namespace: false use-static-endpoints: false - api-domain: testnet.ethswarm.org + api-domain: testnet.internal # testnet.ethswarm.org api-insecure-tls: true api-scheme: http admin-password: test @@ -19,7 +19,7 @@ clusters: mode: bootnode endpoints: - name: bootnode-0 - api-url: http://bootnode-0.bee-playground.testnet.ethswarm.org + api-url: http://bootnode-0.bee-playground.testnet.internal bee-config: bootnode config: default nodes: @@ -34,32 +34,29 @@ clusters: count: 3 endpoints: - name: bee-0 - api-url: http://bee-0.bee-playground.testnet.ethswarm.org + api-url: http://bee-0.bee-playground.testnet.internal - name: bee-1 - api-url: http://bee-1.bee-playground.testnet.ethswarm.org + api-url: http://bee-1.bee-playground.testnet.internal - name: bee-2 - api-url: http://bee-2.bee-playground.testnet.ethswarm.org + api-url: http://bee-2.bee-playground.testnet.internal light: mode: node bee-config: light-node config: light-node - count: 3 + count: 2 endpoints: - name: light-0 - api-url: http://light-0.bee-playground.testnet.ethswarm.org + api-url: http://light-0.bee-playground.testnet.internal - name: light-1 - api-url: http://light-1.bee-playground.testnet.ethswarm.org - - name: light-2 - api-url: http://light-2.bee-playground.testnet.ethswarm.org + api-url: http://light-1.bee-playground.testnet.internal # node-groups defines node groups that can be registered in the cluster # node-groups may inherit it's configuration from already defined node-group and override specific fields from it node-groups: default: _inherit: "" - image: ethersphere/bee:latest + image: ethersphere/bee:2.2.0 image-pull-policy: Always - # image-pull-secrets: [regcred] ingress-annotations: nginx.ingress.kubernetes.io/affinity: "cookie" nginx.ingress.kubernetes.io/affinity-mode: "persistent" @@ -70,7 +67,7 @@ node-groups: nginx.ingress.kubernetes.io/session-cookie-name: "SWARMGATEWAY" nginx.ingress.kubernetes.io/session-cookie-path: "default" nginx.ingress.kubernetes.io/ssl-redirect: "true" - nginx.ingress.kubernetes.io/whitelist-source-range: "109.93.45.225/32" #curl ifconfig.me/ip + nginx.ingress.kubernetes.io/whitelist-source-range: "142.132.221.245/32" #VPN IP, or your IP (but ethswarm.org is needed) ingress-class: "nginx" labels: app.kubernetes.io/component: "node" @@ -78,8 +75,6 @@ node-groups: app.kubernetes.io/part-of: "bee" app.kubernetes.io/version: "latest" beekeeper.ethswarm.org/node-funder: "true" - # node-selector: - # node-group: "private" persistence-enabled: false persistence-storage-class: "local-storage" persistence-storage-request: "34Gi" @@ -129,10 +124,10 @@ bee-configs: swap-endpoint: "ws://geth-swap:8546" swap-factory-address: "0xdD661f2500bA5831e3d1FEbAc379Ea1bF80773Ac" swap-initial-deposit: 500000000000000000 - tracing-enabled: false - tracing-endpoint: "tempo-tempo-distributed-distributor.observability:6831" + tracing-enabled: true + tracing-endpoint: "10.10.11.199:6831" tracing-service-name: "bee" - verbosity: 5 + verbosity: 5 # 1=error, 2=warn, 3=info, 4=debug, 5=trace welcome-message: "Welcome to the Swarm, you are Bee-ing connected!" warmup-time: 0s allow-private-cidrs: true @@ -354,7 +349,7 @@ checks: insufficient-amount: 102400 contract-addr: "0xfc28330f1ecE0ef2371B724E0D19c1EE60B728b2" private-key: "4663c222787e30c1994b59044aa5045377a6e79193a8ead88293926b535c722d" - geth-url: "http://geth-swap.bee-playground.testnet.ethswarm.org" + geth-url: "http://geth-swap.bee-playground.testnet.internal" geth-chain-id: 12345 withdraw: options: diff --git a/config/light-node.yaml b/config/light-node.yaml index 3722e979..b4fe2532 100644 --- a/config/light-node.yaml +++ b/config/light-node.yaml @@ -8,6 +8,6 @@ bee-configs: node-groups: light-node: _inherit: default - image: ethersphere/bee:latest + image: ethersphere/bee:2.2.0 image-pull-policy: Always persistence-enabled: false diff --git a/pkg/check/balances/balances.go b/pkg/check/balances/balances.go index 25bfb214..007705be 100644 --- a/pkg/check/balances/balances.go +++ b/pkg/check/balances/balances.go @@ -46,13 +46,13 @@ var _ beekeeper.Action = (*Check)(nil) // Check instance type Check struct { - log logging.Logger + logger logging.Logger } // NewCheck returns new check func NewCheck(log logging.Logger) beekeeper.Action { return &Check{ - log: log, + logger: log, } } @@ -63,8 +63,8 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int } if o.DryRun { - c.log.Info("running balances (dry mode)") - return dryRun(ctx, cluster, c.log) + c.logger.Info("running balances (dry mode)") + return dryRun(ctx, cluster, c.logger) } var checkCase *test.CheckCase @@ -79,7 +79,7 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int Seed: o.Seed, } - if checkCase, err = test.NewCheckCase(ctx, cluster, caseOpts, c.log); err != nil { + if checkCase, err = test.NewCheckCase(ctx, cluster, caseOpts, c.logger); err != nil { return err } @@ -89,11 +89,11 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int } // initial validation - if err := validateBalances(balances, c.log); err != nil { + if err := validateBalances(balances, c.logger); err != nil { return fmt.Errorf("invalid initial balances: %v", err) } - c.log.Info("Balances are valid") + c.logger.Info("Balances are valid") // repeats for i := 0; i < o.UploadNodeCount; i++ { @@ -110,7 +110,7 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int return err } - if err := expectBalancesHaveChanged(balances, newBalances, c.log); err != nil { + if err := expectBalancesHaveChanged(balances, newBalances, c.logger); err != nil { return err } @@ -126,7 +126,7 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int return err } - if err := expectBalancesHaveChanged(balances, newBalances, c.log); err != nil { + if err := expectBalancesHaveChanged(balances, newBalances, c.logger); err != nil { return err } } diff --git a/pkg/test/case.go b/pkg/test/case.go index 860f6961..1043f343 100644 --- a/pkg/test/case.go +++ b/pkg/test/case.go @@ -45,16 +45,16 @@ func NewCheckCase(ctx context.Context, cluster orchestration.Cluster, caseOpts C return nil, err } - rnd := random.PseudoGenerator(caseOpts.Seed) logger.Infof("Seed: %d", caseOpts.Seed) + rnd := random.PseudoGenerator(caseOpts.Seed) + flatOverlays, err := cluster.FlattenOverlays(ctx) if err != nil { return nil, err } rnds := random.PseudoGenerators(caseOpts.Seed, len(flatOverlays)) - logger.Infof("Seed: %d", caseOpts.Seed) var ( nodes []BeeV2 diff --git a/version.go b/version.go index 0fc81ef3..3d8da08c 100644 --- a/version.go +++ b/version.go @@ -1,7 +1,7 @@ package beekeeper var ( - version = "0.19.1" // manually set semantic version number + version = "0.19.2" // manually set semantic version number commit string // automatically set git commit hash // Version TODO From 4f6d120f970b1bdef12a3242dcf5d2f81b217120 Mon Sep 17 00:00:00 2001 From: Ljubisa Date: Thu, 14 Nov 2024 16:13:59 +0100 Subject: [PATCH 5/6] chore: modify staging configuration --- config/staging.yaml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/config/staging.yaml b/config/staging.yaml index f0c9e18c..d829d77f 100644 --- a/config/staging.yaml +++ b/config/staging.yaml @@ -5,13 +5,13 @@ clusters: _inherit: "" name: bee namespace: beekeeper - disable-namespace: true - api-domain: staging.ethswarm.org + disable-namespace: false + api-domain: testnet.internal api-insecure-tls: true - api-scheme: https + api-scheme: http funding: - eth: 0.1 - gbzz: 2.0 + eth: 0.01 + bzz: 1.0 node-groups: bee: mode: node @@ -31,4 +31,10 @@ node-groups: bee-configs: staging: _inherit: "default" - bootnodes: "/dnsaddr/mainnet.ethswarm.org" + swap-endpoint: http://rpc-sepolia-haproxy.default.svc.swarm1.local + bootnode: /dnsaddr/testnet.ethswarm.org + network-id: 10 + swap-initial-deposit: 0 + swap-factory-address: "" + verbosity: 4 + welcome-message: Welcome to the bee staging environment created by Beekeeper! From c5ed69e72afc77b7997280e8affe4a0d95f4c2f5 Mon Sep 17 00:00:00 2001 From: Ljubisa Date: Fri, 15 Nov 2024 15:48:37 +0100 Subject: [PATCH 6/6] chore: remove whitelist anotation from ingress in config --- config/config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/config/config.yaml b/config/config.yaml index 1b4c9991..58cace0d 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -67,7 +67,6 @@ node-groups: nginx.ingress.kubernetes.io/session-cookie-name: "SWARMGATEWAY" nginx.ingress.kubernetes.io/session-cookie-path: "default" nginx.ingress.kubernetes.io/ssl-redirect: "true" - nginx.ingress.kubernetes.io/whitelist-source-range: "142.132.221.245/32" #VPN IP, or your IP (but ethswarm.org is needed) ingress-class: "nginx" labels: app.kubernetes.io/component: "node"