-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
19 changed files
with
637 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
# This section includes base Calico installation configuration. | ||
# For more information, see: https://docs.tigera.io/calico/latest/reference/installation/api#operator.tigera.io/v1.Installation | ||
apiVersion: operator.tigera.io/v1 | ||
kind: Installation | ||
metadata: | ||
name: default | ||
spec: | ||
# Configures Calico networking. | ||
calicoNetwork: | ||
# Note: The ipPools section cannot be modified post-install. | ||
ipPools: | ||
- blockSize: 26 | ||
cidr: $KOREOS_POD_SUBNET_CIDR | ||
encapsulation: VXLANCrossSubnet | ||
natOutgoing: Enabled | ||
nodeSelector: all() | ||
flexVolumePath: /var/lib/kubelet/volumeplugins | ||
--- | ||
|
||
# This section configures the Calico API server. | ||
# For more information, see: https://docs.tigera.io/calico/latest/reference/installation/api#operator.tigera.io/v1.APIServer | ||
apiVersion: operator.tigera.io/v1 | ||
kind: APIServer | ||
metadata: | ||
name: default | ||
spec: {} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,10 @@ | ||
[Service] | ||
Environment=KOREOS_CRIO_VERSION=v1.31.1 | ||
Environment=LIBEXECDIR=/usr/local # https://github.com/cri-o/packaging/issues/150 | ||
Environment=KOREOS_KUBERNETES_VERSION=v1.31.2 | ||
Environment=KOREOS_CLUSTER_NAME="your cluster name here" | ||
Environment=KOREOS_DNS_DOMAIN="your cluster dns name here" | ||
Environment=KOREOS_POD_SUBNET_CIDR=10.0.0.0/16 | ||
Environment=KOREOS_SERVICE_SUBNET_CIDR=10.96.0.0/12 | ||
Environment=KOREOS_HOSTNAME=%l | ||
Environment=KOREOS_CONTROL_PLANE_ENDPOINT="cluster FQDN and port 'eg. cluter.domain:6443" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
apiVersion: kubeadm.k8s.io/v1beta3 | ||
kind: JoinConfiguration | ||
caCertPath: /etc/kubernetes/pki/ca.crt | ||
discovery: | ||
bootstrapToken: | ||
apiServerEndpoint: $KOREOS_CONTROL_PLANE_ENDPOINT | ||
token: <you will have to update this file manualy> | ||
caCertHashes: | ||
- <you will have to update this file manualy> | ||
timeout: 5m0s | ||
nodeRegistration: | ||
criSocket: unix:///run/crio/crio.sock | ||
imagePullPolicy: IfNotPresent | ||
name: $KOREOS_HOSTNAME | ||
taints: null | ||
ignorePreflightErrors: | ||
- KubeletVersion | ||
- FileExisting-conntrack |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,40 @@ | ||
apiVersion: kubeadm.k8s.io/v1beta3 | ||
kind: ClusterConfiguration | ||
kubernetesVersion: $KOREOS_KUBERNETES_VERSION | ||
clusterName: $KOREOS_CLUSTER_NAME | ||
controlPlaneEndpoint: $KOREOS_CONTROL_PLANE_ENDPOINT | ||
certificatesDir: /etc/kubernetes/pki | ||
imageRepository: registry.k8s.io | ||
apiServer: | ||
timeoutForControlPlane: 4m0s | ||
controllerManager: | ||
extraArgs: | ||
flex-volume-plugin-dir: /var/lib/kubelet/volumeplugins | ||
dns: {} | ||
etcd: | ||
local: | ||
dataDir: /var/lib/etcd | ||
networking: | ||
dnsDomain: $KOREOS_DNS_DOMAIN | ||
serviceSubnet: $KOREOS_SERVICE_SUBNET_CIDR | ||
podSubnet: $KOREOS_POD_SUBNET_CIDR | ||
scheduler: {} | ||
--- | ||
apiVersion: kubeadm.k8s.io/v1beta3 | ||
kind: InitConfiguration | ||
nodeRegistration: | ||
criSocket: unix:///run/crio/crio.sock | ||
ignorePreflightErrors: | ||
- KubeletVersion | ||
- FileExisting-conntrack | ||
--- | ||
apiVersion: kubelet.config.k8s.io/v1beta1 | ||
kind: KubeletConfiguration | ||
failSwapOn: false | ||
featureGates: | ||
NodeSwap: true | ||
memorySwap: | ||
swapBehavior: LimitedSwap | ||
volumePluginDir: /var/lib/kubelet/volumeplugins | ||
containerRuntimeEndpoint: unix:///run/crio/crio.sock | ||
cgroupDriver: systemd |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1 +1,63 @@ | ||
# KoreOS | ||
# KoreOS | ||
|
||
The name come from ... CoreOS and the fact that every things relative to kubernetes contains a K :D so, it's not a clever name | ||
|
||
## Prerequisites | ||
|
||
You should know FedoraCoreOS quite well and how to write `*.bu` files and convert them to `*.ign` files : see <https://docs.fedoraproject.org/en-US/fedora-coreos/producing-ign/> | ||
|
||
I'm used to rely on a PXE boot server and a Http server for provisioning and everything here is more or less based on this in mind. | ||
|
||
You should have some k8s/kubeadm basics in your bag ! | ||
|
||
## Feel free to contribute | ||
|
||
As the first readme file everything needs to be done | ||
|
||
This is a real WIP, I did promise it on <https://discussion.fedoraproject.org/t/the-right-way-to-deploy-kubernetes-on-coreos-virtual-machines/117779/1> so I deliver my work as is | ||
|
||
This work has been possible thanks to GOAT <https://github.com/poseidon/typhoon> and his great idea <https://quay.io/repository/poseidon/kubelet?tab=tags&tag=latest> | ||
|
||
## How to use it ? | ||
|
||
### ./KoreOS/environment.conf | ||
|
||
Contains some variables used for provisionning, change it with you favorite values, typîcally : | ||
|
||
```ini | ||
[Service] | ||
Environment=KOREOS_CRIO_VERSION=v1.31.1 | ||
# https://github.com/cri-o/packaging/issues/150 | ||
Environment=LIBEXECDIR=/usr/local | ||
Environment=KOREOS_KUBERNETES_VERSION=v1.31.2 | ||
Environment=KOREOS_CLUSTER_NAME=MyCLuster | ||
Environment=KOREOS_DNS_DOMAIN=MyCluster.MyDomain | ||
Environment=KOREOS_POD_SUBNET_CIDR=10.0.0.0/16 | ||
Environment=KOREOS_SERVICE_SUBNET_CIDR=10.96.0.0/12 | ||
Environment=KOREOS_HOSTNAME=%l | ||
Environment=KOREOS_CONTROL_PLANE_ENDPOINT=mycluster.mydomain:6443 | ||
``` | ||
|
||
`KOREOS_POD_SUBNET_CIDR` and `KOREOS_SERVICE_SUBNET_CIDR` are default values from Kubernetes documentation. | ||
|
||
### ./merge/KoreOS-dependencies-xxx.bu | ||
|
||
These 2 files contain some other dependencies not linked to `environment.conf` but you should set same version numbers. | ||
|
||
### controller.bu and worker.bu | ||
|
||
These files contain some node specific informations like `storage` or `hostname file` and the links to you http(s) server where ign files are stored/published | ||
|
||
I don't know how to manage multiple master nodes ... feel free to contribute | ||
You will have to duplicate worker.bu for each worker node | ||
|
||
## updateignitions.sh | ||
|
||
A small script that will convert `*.bu` files in `*.ign` files and if you want can publish them via ssh on your web server. | ||
|
||
## The right way ... | ||
|
||
You should start your cluster with the master/controller node and wait for `koreos-installer.service` (from KoreOS-installer.ign) to finish. | ||
Then you can check the logs with `journalctl -eu koreos-installer.service` and get the token like `kzh5ow.k7658m9zlmec5cni` and the caCertHashes: like `sha256:e46b74d087a9b14ebb03f7f65d75944a8dead52d0dddc654aa9878985ff9b0be` | ||
|
||
You can boot your worker nodes and then connect them via ssh and `cd /opt/koreos` and update the `join.config.yaml` with token and caCertHashes taken from master node and then run `sudo kubeadm join --config join.config.yaml` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,36 @@ | ||
variant: fcos | ||
version: 1.5.0 | ||
|
||
ignition: | ||
security: | ||
tls: | ||
certificate_authorities: | ||
- source: http://YourHttpServerName:port/path/YourCertificate.crt | ||
verification: | ||
hash: sha256-xxxxx | ||
|
||
config: | ||
merge: | ||
- source: https://YourHttpServerName:port/ign/KoreOS-ssh-key.ign | ||
- source: https://YourHttpServerName:port/ign/KoreOS-cri-o.ign | ||
- source: https://YourHttpServerName:port/ign/KoreOS-installer.ign | ||
- source: https://YourHttpServerName:port/ign/KoreOS-etc-mount.ign | ||
- source: https://YourHttpServerName:port/ign/KoreOS-services-conf.ign | ||
- source: https://YourHttpServerName:port/ign/KoreOS-kubelet-controller.ign | ||
- source: https://YourHttpServerName:port/ign/KoreOS-dependencies-controller.ign | ||
- source: https://YourHttpServerName:port/ign/KoreOS-template-controller.ign | ||
|
||
storage: | ||
filesystems: | ||
- path: /var | ||
device: /dev/sda | ||
format: xfs | ||
label: Var | ||
wipe_filesystem: true | ||
with_mount_unit: true | ||
|
||
files: | ||
- path: /etc/hostname | ||
mode: 0644 | ||
contents: | ||
inline: NodeName |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
variant: fcos | ||
version: 1.5.0 | ||
|
||
systemd: | ||
units: | ||
- name: crio-installer.service | ||
enabled: true | ||
dropins: | ||
- name: env.conf | ||
contents_local: KoreOS/environment.conf | ||
contents: | | ||
[Unit] | ||
Description=Install crio | ||
Before=kubelet.service | ||
After=network-online.target | ||
|
||
[Service] | ||
TimeoutSec=300 | ||
Type=oneshot | ||
RemainAfterExit=true | ||
ExecStartPre=/usr/bin/bash -c 'curl -vL https://storage.googleapis.com/cri-o/artifacts/cri-o.amd64.${KOREOS_CRIO_VERSION}.tar.gz | tar -xz --directory /opt' | ||
ExecStart=/usr/bin/bash -c 'cd /opt/cri-o && ./install' | ||
ExecStart=/usr/bin/systemctl enable --now crio.service | ||
|
||
[Install] | ||
WantedBy=multi-user.target |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,64 @@ | ||
variant: fcos | ||
version: 1.5.0 | ||
|
||
storage: | ||
files: | ||
- path: /opt/koreos/tigera-operator.yaml | ||
mode: 0751 | ||
overwrite: true | ||
contents: | ||
source: https://raw.githubusercontent.com/projectcalico/calico/v3.28.2/manifests/tigera-operator.yaml | ||
|
||
- path: /usr/local/bin/calicoctl | ||
mode: 0751 | ||
overwrite: true | ||
contents: | ||
source: https://github.com/projectcalico/calico/releases/download/v3.28.2/calicoctl-linux-amd64 | ||
|
||
- path: /usr/local/bin/kubeadm | ||
mode: 0751 | ||
overwrite: true | ||
contents: | ||
source: https://dl.k8s.io/release/v1.31.2/bin/linux/amd64/kubeadm | ||
|
||
- path: /usr/local/bin/kubectl | ||
mode: 0751 | ||
overwrite: true | ||
contents: | ||
source: https://dl.k8s.io/release/v1.31.2/bin/linux/amd64/kubectl | ||
|
||
- path: /var/lib/calico/nodename | ||
mode: 0644 | ||
overwrite: true | ||
contents: | ||
inline: $HOSTNAME | ||
|
||
- path: /var/lib/calico/mtu | ||
mode: 0644 | ||
overwrite: true | ||
contents: | ||
inline: "1450" | ||
|
||
- path: /opt/koreos/templates/calico.config.template.yaml | ||
mode: 0644 | ||
overwrite: true | ||
contents: | ||
local: KoreOS/calico.config.template.yaml | ||
|
||
- path: /opt/koreos/templates/kubeadm.config.template.yaml | ||
mode: 0644 | ||
overwrite: true | ||
contents: | ||
local: KoreOS/kubeadm.config.template.yaml | ||
|
||
- path: /etc/profile.d/kubeCLI_completions.sh | ||
contents: | ||
inline: | | ||
source <(kubectl completion bash) | ||
source <(kubeadm completion bash) | ||
|
||
directories: | ||
- path: /opt/cni/bin | ||
- path: /etc/cni/net.d | ||
- path: /var/lib/kubelet/volumeplugins | ||
- path: /var/lib/kubernetes/manifests |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,34 @@ | ||
variant: fcos | ||
version: 1.5.0 | ||
|
||
storage: | ||
files: | ||
- path: /usr/local/bin/kubeadm | ||
mode: 0751 | ||
overwrite: true | ||
contents: | ||
source: https://dl.k8s.io/release/v1.31.2/bin/linux/amd64/kubeadm | ||
|
||
- path: /usr/local/bin/kubectl | ||
mode: 0751 | ||
overwrite: true | ||
contents: | ||
source: https://dl.k8s.io/release/v1.31.2/bin/linux/amd64/kubectl | ||
|
||
- path: /opt/koreos/templates/join.config.template.yaml | ||
mode: 0644 | ||
overwrite: true | ||
contents: | ||
local: KoreOS/join.config.template.yaml | ||
|
||
- path: /etc/profile.d/kubeCLI_completions.sh | ||
contents: | ||
inline: | | ||
source <(kubectl completion bash) | ||
source <(kubeadm completion bash) | ||
|
||
directories: | ||
- path: /opt/cni/bin | ||
- path: /etc/cni/net.d | ||
- path: /var/lib/kubelet/volumeplugins | ||
- path: /var/lib/kubernetes/manifests |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
variant: fcos | ||
version: 1.5.0 | ||
|
||
systemd: | ||
units: | ||
- name: etc-kubernetes.mount | ||
enabled: true | ||
contents: | | ||
[Unit] | ||
Description=Bind mount for etc/kubernetes | ||
|
||
[Mount] | ||
What=/var/lib/kubernetes | ||
Where=/etc/kubernetes | ||
Options=bind | ||
|
||
[Install] | ||
WantedBy=multi-user.target |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,34 @@ | ||
variant: fcos | ||
version: 1.5.0 | ||
|
||
systemd: | ||
units: | ||
- name: koreos-installer.service | ||
enabled: true | ||
contents: | | ||
[Unit] | ||
Description=Install KoreOS | ||
RequiresMountsFor=/etc/kubernetes | ||
After=network-online.target crio-installer.service crio.service koreos-template.service | ||
Wants=crio.service koreos-template.service | ||
|
||
[Service] | ||
TimeoutSec=300 | ||
Type=oneshot | ||
RemainAfterExit=true | ||
ExecStartPre=/usr/local/bin/kubeadm config images pull --config /opt/koreos/kubeadm.config.yaml | ||
ExecStart=/usr/local/bin/kubeadm init --v=5 --config /opt/koreos/kubeadm.config.yaml | ||
ExecStartPost=/usr/bin/install -D -o root -g root /etc/kubernetes/super-admin.conf /root/.kube/config | ||
ExecStartPost=/usr/bin/install -d -o core -g core /home/core/.kube | ||
ExecStartPost=/usr/bin/install -o core -g core /etc/kubernetes/admin.conf /home/core/.kube/config | ||
ExecStartPost=/usr/local/bin/kubectl --kubeconfig /root/.kube/config cluster-info | ||
ExecStartPost=/usr/local/bin/kubectl wait --kubeconfig /root/.kube/config nodes --all --for condition=Ready | ||
ExecStartPost=/usr/local/bin/kubectl wait --kubeconfig /root/.kube/config --all-namespaces deployments --all --for condition=Available --timeout=60s | ||
ExecStartPost=/usr/local/bin/kubectl wait --kubeconfig /root/.kube/config --all-namespaces pods --all --for condition=Ready | ||
ExecStartPost=/usr/local/bin/kubectl create --kubeconfig /root/.kube/config -f /opt/koreos/tigera-operator.yaml | ||
ExecStartPost=/usr/local/bin/kubectl wait --kubeconfig /root/.kube/config --namespace tigera-operator deployments --all --for condition=Available --timeout=60s | ||
ExecStartPost=/usr/local/bin/kubectl create --kubeconfig /root/.kube/config -f /opt/koreos/calico.config.yaml | ||
|
||
[Install] | ||
WantedBy=multi-user.target | ||
|
Oops, something went wrong.