HA master Pipeline

banzaicloud · Apr 3, 2019 · ba37408 · ba37408
1 parent dde35e6
commit ba37408
Show file tree

Hide file tree

Showing 9 changed files with 167 additions and 41 deletions.
diff --git a/cmd/pke/app/phases/kubeadm/controlplane/controlplane.go b/cmd/pke/app/phases/kubeadm/controlplane/controlplane.go
@@ -15,6 +15,7 @@
 package controlplane
 
 import (
+	"context"
 	"crypto/rand"
 	"encoding/base64"
 	"fmt"
@@ -28,14 +29,18 @@ import (
 	"time"
 
 	"github.com/Masterminds/semver"
+	"github.com/banzaicloud/pipeline/client"
 	"github.com/banzaicloud/pke/cmd/pke/app/constants"
 	"github.com/banzaicloud/pke/cmd/pke/app/phases"
 	"github.com/banzaicloud/pke/cmd/pke/app/phases/kubeadm"
 	"github.com/banzaicloud/pke/cmd/pke/app/phases/kubeadm/node"
 	"github.com/banzaicloud/pke/cmd/pke/app/util/file"
 	"github.com/banzaicloud/pke/cmd/pke/app/util/linux"
+	"github.com/banzaicloud/pke/cmd/pke/app/util/network"
+	"github.com/banzaicloud/pke/cmd/pke/app/util/pipeline"
 	"github.com/banzaicloud/pke/cmd/pke/app/util/runner"
 	"github.com/banzaicloud/pke/cmd/pke/app/util/validator"
+	"github.com/lestrrat-go/backoff"
 	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
@@ -85,6 +90,7 @@ type ControlPlane struct {
 	imageRepository             string
 	withPluginPSP               bool
 	node                        *node.Node
+	cidr                        string
 }
 
 func NewCommand(out io.Writer) *cobra.Command {
@@ -95,6 +101,7 @@ func NewDefault(kubernetesVersion, imageRepository string) *ControlPlane {
 	return &ControlPlane{
 		kubernetesVersion: kubernetesVersion,
 		imageRepository:   imageRepository,
+		node:              &node.Node{},
 	}
 }
 
@@ -134,17 +141,20 @@ func (c *ControlPlane) RegisterFlags(flags *pflag.FlagSet) {
 	flags.String(constants.FlagImageRepository, "banzaicloud", "Prefix for image repository")
 	// PodSecurityPolicy admission plugin
 	flags.Bool(constants.FlagAdmissionPluginPodSecurityPolicy, false, "Enable PodSecurityPolicy admission plugin")
-
-	addHAControlPlaneFlags(flags)
+	// Pipeline
+	flags.StringP(constants.FlagPipelineAPIEndpoint, constants.FlagPipelineAPIEndpointShort, "", "Pipeline API server url")
+	flags.StringP(constants.FlagPipelineAPIToken, constants.FlagPipelineAPITokenShort, "", "Token for accessing Pipeline API")
+	flags.Int32(constants.FlagPipelineOrganizationID, 0, "Organization ID to use with Pipeline API")
+	flags.Int32(constants.FlagPipelineClusterID, 0, "Cluster ID to use with Pipeline API")
+	flags.String(constants.FlagInfrastructureCIDR, "192.168.64.0/20", "network CIDR for the actual machine")
+
+	c.addHAControlPlaneFlags(flags)
 }
 
-func addHAControlPlaneFlags(flags *pflag.FlagSet) {
-	var (
-		f = &pflag.FlagSet{}
-		n phases.Runnable
-	)
-	n = &node.Node{}
-	n.RegisterFlags(f)
+func (c *ControlPlane) addHAControlPlaneFlags(flags *pflag.FlagSet) {
+	var f = &pflag.FlagSet{}
+
+	c.node.RegisterFlags(f)
 
 	f.VisitAll(func(flag *pflag.Flag) {
 		if flags.Lookup(flag.Name) == nil {
@@ -188,8 +198,11 @@ func (c *ControlPlane) Validate(cmd *cobra.Command) error {
 	case "single", "default":
 		// noop
 	case "ha":
+		if err := c.pipelineJoin(cmd); err != nil {
+			return err
+		}
+
 		if c.joinControlPlane {
-			c.node = &node.Node{}
 			return c.node.Validate(cmd)
 		}
 
@@ -200,6 +213,87 @@ func (c *ControlPlane) Validate(cmd *cobra.Command) error {
 	return nil
 }
 
+func (c *ControlPlane) pipelineJoin(cmd *cobra.Command) error {
+	if pipeline.Enabled(cmd) {
+		// hostname
+		hostname, err := os.Hostname()
+		if err != nil {
+			return err
+		}
+
+		// ip
+		ips, err := network.IPv4Addresses()
+		if err != nil {
+			return err
+		}
+		ip, err := network.ContainsFirst(c.cidr, ips)
+		if err != nil {
+			return err
+		}
+
+		// Pipeline client
+		endpoint, token, orgID, clusterID, err := pipeline.CommandArgs(cmd)
+		if err != nil {
+			return err
+		}
+
+		p := pipeline.Client(os.Stdout, endpoint, token)
+
+		// elect leader
+		_, resp, err := p.ClustersApi.PostLeaderElection(context.Background(), orgID, clusterID, client.PostLeaderElectionRequest{
+			Hostname: hostname,
+			Ip:       ip.String(),
+		})
+
+		if err != nil && resp == nil {
+			return errors.Wrap(err, "failed to become leader")
+		}
+		if resp != nil && resp.StatusCode == http.StatusConflict {
+			// check if leadership is ours or not
+			var leader client.GetLeaderElectionResponse
+			leader, resp, err = p.ClustersApi.GetLeaderElection(context.Background(), orgID, clusterID)
+			if err != nil {
+				return errors.Wrap(err, "failed to get leader")
+			}
+			if resp.StatusCode == http.StatusNotFound {
+				return errors.New("unexpected condition. leader not found and cloud not acquire leadership")
+			}
+			if resp.StatusCode == http.StatusOK {
+				if hostname == leader.Hostname && ip.String() == leader.Ip {
+					// we are the leaders, proceed with master installation
+					return nil
+				}
+			}
+			// somebody already took leadership
+
+			c.joinControlPlane = true
+
+			policy := backoff.NewExponential(
+				backoff.WithInterval(time.Second),
+				backoff.WithFactor(2),
+				backoff.WithMaxElapsedTime(time.Hour),
+				backoff.WithMaxInterval(30*time.Second),
+				backoff.WithMaxRetries(0),
+			)
+			b, cancel := policy.Start(context.Background())
+			defer cancel()
+
+			for backoff.Continue(b) {
+				// Wait for master to become ready
+				var ready client.PkeClusterReadinessResponse
+				ready, resp, err = p.ClustersApi.GetReadyPKENode(context.Background(), orgID, clusterID)
+				if resp != nil && resp.StatusCode == http.StatusOK && ready.Master.Ready {
+					return nil
+				}
+			}
+			// backoff timeout
+			return errors.New("timeout exceeded. waiting for master to become ready failed")
+		}
+	}
+
+	return nil
+}
+
 func (c *ControlPlane) Run(out io.Writer) error {
 	_, _ = fmt.Fprintf(out, "[RUNNING] %s\n", c.Use())
 
@@ -305,6 +399,10 @@ func (c *ControlPlane) masterBootstrapParameters(cmd *cobra.Command) (err error)
 		return
 	}
 	c.joinControlPlane, err = cmd.Flags().GetBool(constants.FlagControlPlaneJoin)
+	if err != nil {
+		return
+	}
+	c.cidr, err = cmd.Flags().GetString(constants.FlagInfrastructureCIDR)
 
 	return
 }

diff --git a/cmd/pke/app/phases/kubeadm/node/node.go b/cmd/pke/app/phases/kubeadm/node/node.go
@@ -15,13 +15,11 @@
 package node
 
 import (
-	"context"
 	"fmt"
 	"io"
 	"os"
 	"path/filepath"
 
-	"github.com/banzaicloud/pipeline/client"
 	"github.com/banzaicloud/pke/cmd/pke/app/constants"
 	"github.com/banzaicloud/pke/cmd/pke/app/phases"
 	"github.com/banzaicloud/pke/cmd/pke/app/phases/kubeadm"
@@ -141,7 +139,7 @@ func (n *Node) workerBootstrapParameters(cmd *cobra.Command) (err error) {
 	}
 
 	if n.apiServerHostPort == "" && n.kubeadmToken == "" && n.caCertHash == "" {
-		n.apiServerHostPort, n.kubeadmToken, n.caCertHash, err = pipelineJoinArgs(cmd)
+		n.apiServerHostPort, n.kubeadmToken, n.caCertHash, err = pipeline.NodeJoinArgs(os.Stdout, cmd)
 		if err != nil {
 			return
 		}
@@ -156,29 +154,6 @@ func (n *Node) workerBootstrapParameters(cmd *cobra.Command) (err error) {
 	return
 }
 
-func pipelineJoinArgs(cmd *cobra.Command) (apiServerHostPort, kubeadmToken, caCertHash string, err error) {
-	if !pipeline.Enabled(cmd) {
-		return
-	}
-	endpoint, token, orgID, clusterID, err := pipeline.CommandArgs(cmd)
-	if err != nil {
-		return
-	}
-
-	// Pipeline client.
-	c := pipeline.Client(os.Stdout, endpoint, token)
-
-	var b client.GetClusterBootstrapResponse
-	b, _, err = c.ClustersApi.GetClusterBootstrap(context.Background(), orgID, clusterID)
-	if err != nil {
-		return
-	}
-	apiServerHostPort = b.MasterAddress
-	kubeadmToken = b.Token
-	caCertHash = b.DiscoveryTokenCaCertHash
-	return
-}
-
 func (n *Node) install(out io.Writer) error {
 	// write kubeadm config
 	if err := n.writeKubeadmConfig(out, kubeadmConfig); err != nil {

diff --git a/cmd/pke/app/util/pipeline/client.go b/cmd/pke/app/util/pipeline/client.go
@@ -15,6 +15,7 @@
 package pipeline
 
 import (
+	"context"
 	"io"
 	"time"
 
@@ -91,3 +92,26 @@ func ValidArgs(endpoint, token string, orgID, clusterID int32) error {
 		constants.FlagPipelineClusterID:      clusterID,
 	})
 }
+
+func NodeJoinArgs(out io.Writer, cmd *cobra.Command) (apiServerHostPort, kubeadmToken, caCertHash string, err error) {
+	if !Enabled(cmd) {
+		return
+	}
+	endpoint, token, orgID, clusterID, err := CommandArgs(cmd)
+	if err != nil {
+		return
+	}
+
+	// Pipeline client.
+	c := Client(out, endpoint, token)
+
+	var b client.GetClusterBootstrapResponse
+	b, _, err = c.ClustersApi.GetClusterBootstrap(context.Background(), orgID, clusterID)
+	if err != nil {
+		return
+	}
+	apiServerHostPort = b.MasterAddress
+	kubeadmToken = b.Token
+	caCertHash = b.DiscoveryTokenCaCertHash
+	return
+}
diff --git a/cmd/pke/docs/pke_install_master.md b/cmd/pke/docs/pke_install_master.md
@@ -24,7 +24,7 @@ pke install master [flags]
       --kubernetes-cluster-name string                    Kubernetes cluster name (default "pke")
       --kubernetes-controller-manager-signing-ca string   Kubernetes Controller Manager signing cert
       --kubernetes-infrastructure-cidr string             network CIDR for the actual machine (default "192.168.64.0/20")
-      --kubernetes-join-control-plane                     Join and another control plane node
+      --kubernetes-join-control-plane                     Join an another control plane node
       --kubernetes-master-mode string                     Kubernetes cluster mode (default "default")
       --kubernetes-network-provider string                Kubernetes network provider (default "weave")
       --kubernetes-node-token string                      PKE join token

diff --git a/cmd/pke/docs/pke_install_master_kubernetes-controlplane.md b/cmd/pke/docs/pke_install_master_kubernetes-controlplane.md
@@ -23,7 +23,8 @@ pke install master kubernetes-controlplane [flags]
       --kubernetes-cloud-provider string                  cloud provider. example: aws
       --kubernetes-cluster-name string                    Kubernetes cluster name (default "pke")
       --kubernetes-controller-manager-signing-ca string   Kubernetes Controller Manager signing cert
-      --kubernetes-join-control-plane                     Join and another control plane node
+      --kubernetes-infrastructure-cidr string             network CIDR for the actual machine (default "192.168.64.0/20")
+      --kubernetes-join-control-plane                     Join an another control plane node
       --kubernetes-master-mode string                     Kubernetes cluster mode (default "default")
       --kubernetes-network-provider string                Kubernetes network provider (default "weave")
       --kubernetes-node-token string                      PKE join token

diff --git a/cmd/pke/docs/pke_install_single.md b/cmd/pke/docs/pke_install_single.md
@@ -24,7 +24,7 @@ pke install single [flags]
       --kubernetes-cluster-name string                    Kubernetes cluster name (default "pke")
       --kubernetes-controller-manager-signing-ca string   Kubernetes Controller Manager signing cert
       --kubernetes-infrastructure-cidr string             network CIDR for the actual machine (default "192.168.64.0/20")
-      --kubernetes-join-control-plane                     Join and another control plane node
+      --kubernetes-join-control-plane                     Join an another control plane node
       --kubernetes-master-mode string                     Kubernetes cluster mode (default "default")
       --kubernetes-network-provider string                Kubernetes network provider (default "weave")
       --kubernetes-node-token string                      PKE join token

diff --git a/cmd/pke/docs/pke_install_single_kubernetes-controlplane.md b/cmd/pke/docs/pke_install_single_kubernetes-controlplane.md
@@ -23,7 +23,8 @@ pke install single kubernetes-controlplane [flags]
       --kubernetes-cloud-provider string                  cloud provider. example: aws
       --kubernetes-cluster-name string                    Kubernetes cluster name (default "pke")
       --kubernetes-controller-manager-signing-ca string   Kubernetes Controller Manager signing cert
-      --kubernetes-join-control-plane                     Join and another control plane node
+      --kubernetes-infrastructure-cidr string             network CIDR for the actual machine (default "192.168.64.0/20")
+      --kubernetes-join-control-plane                     Join an another control plane node
       --kubernetes-master-mode string                     Kubernetes cluster mode (default "default")
       --kubernetes-network-provider string                Kubernetes network provider (default "weave")
       --kubernetes-node-token string                      PKE join token

diff --git a/go.mod b/go.mod
@@ -5,9 +5,10 @@ go 1.12
 require (
 	github.com/Masterminds/semver v1.4.2
 	github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6
-	github.com/banzaicloud/pipeline v0.0.0-20190311122237-7db66a712b1b
+	github.com/banzaicloud/pipeline v0.0.0-20190402103326-d01205919224
 	github.com/cpuguy83/go-md2man v1.0.8 // indirect
 	github.com/ghodss/yaml v1.0.0
+	github.com/lestrrat-go/backoff v0.0.0-20190107202757-0bc2a4274cd0
 	github.com/pkg/errors v0.8.1
 	github.com/spf13/cobra v0.0.3
 	github.com/spf13/pflag v1.0.3