-
Notifications
You must be signed in to change notification settings - Fork 0
249 lines (223 loc) · 10.3 KB
/
run-test-cases.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
name: Test K3s, Kubernetes, and MicroK8s
on:
pull_request:
branches: [main]
paths:
- e2e/**
- .github/workflows/run-test-cases.yml
- build/Dockerfile.rust
- version.txt
- Makefile
push:
branches: [main]
paths:
- e2e/**
- .github/workflows/run-test-cases.yml
- build/Dockerfile.rust
- version.txt
- Makefile
release:
types:
- published
jobs:
build-containers:
runs-on: ubuntu-latest
timeout-minutes: 60
steps:
- name: Checkout the head commit of the branch
uses: actions/checkout@v3
with:
persist-credentials: false
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Build local containers for PR tests
if: startsWith(github.event_name, 'pull_request')
env:
PREFIX: ghcr.io/project-akri/akri
LABEL_PREFIX: pr
run: |
make LOAD=1
docker save ${PREFIX}/udev-discovery:${LABEL_PREFIX} > udev-discovery.tar
- name: Upload UDEV discovery container as artifact
if: startsWith(github.event_name, 'pull_request')
uses: actions/upload-artifact@v3
with:
name: udev-discovery.tar
path: udev-discovery.tar
test-cases:
needs: build-containers
runs-on: ubuntu-latest
timeout-minutes: 95
strategy:
fail-fast: false
matrix:
kube:
- runtime: microk8s
version: 1.25/stable
- runtime: microk8s
version: 1.26/stable
- runtime: microk8s
version: 1.27/stable
- runtime: microk8s
version: 1.28/stable
- runtime: k3s
version: v1.25.13+k3s1
- runtime: k3s
version: v1.26.8+k3s1
- runtime: k3s
version: v1.27.5+k3s1
- runtime: k3s
version: v1.28.1+k3s1
- runtime: k8s
version: 1.25.13
- runtime: k8s
version: 1.26.8
- runtime: k8s
version: 1.27.5
- runtime: k8s
version: 1.28.1
steps:
- name: Checkout the head commit of the branch
uses: actions/checkout@v3
with:
persist-credentials: false
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: "3.10"
- name: Install Poetry and dependencies
working-directory: ./e2e
run: |
POETRY_HOME=/opt/poetry
python3 -m venv $POETRY_HOME
$POETRY_HOME/bin/pip install poetry==1.5.1
$POETRY_HOME/bin/poetry --version
$POETRY_HOME/bin/poetry install --no-root
- name: Download container artifacts
if: startsWith(github.event_name, 'pull_request')
uses: actions/download-artifact@v3
with:
path: /tmp/images
- if: startsWith(matrix.kube.runtime, 'k3s')
name: Install K3s
env:
INSTALL_K3S_VERSION: ${{ matrix.kube.version }}
run: |
sudo curl -sfL https://get.k3s.io -o install.sh
sudo chmod +x install.sh
./install.sh server --kubelet-arg=eviction-hard="imagefs.available<1%,nodefs.available<1%" --kubelet-arg=eviction-minimum-reclaim="imagefs.available=1%,nodefs.available=1%"
mkdir -p $HOME/.kube
sudo cp -i /etc/rancher/k3s/k3s.yaml $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
until kubectl get node ${HOSTNAME,,} -o jsonpath='{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status}' | grep 'Ready=True'; do echo "waiting for k3s to become ready"; sleep 10; done
- if: (startsWith(github.event_name, 'pull_request')) && (startsWith(matrix.kube.runtime, 'K3s'))
name: Import local agent and controller to K3s
working-directory: /tmp/images
run: |
sudo find -name "*.tar" -type f -exec k3s ctr image import {} \;
- if: startsWith(matrix.kube.runtime, 'k8s')
name: Install Kubernetes
run: |
SHORTVERSION=$(cut -d '.' -f 1,2 <<< "${{ matrix.kube.version }}")
sudo apt-get update -y
sudo apt-get install -y apt-transport-https ca-certificates curl
curl -fsSL https://pkgs.k8s.io/core:/stable:/v${SHORTVERSION}/deb/Release.key | sudo gpg --yes --batch --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v${SHORTVERSION}/deb/ /" | sudo tee /etc/apt/sources.list.d/kubernetes.list
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --yes --batch --dearmor -o /usr/share/keyrings/docker.gpg
echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list
sudo apt-get update
sudo apt-get install -o Dpkg::Options::="--force-overwrite" -y --allow-downgrades kubelet=${{ matrix.kube.version }}-* kubeadm=${{ matrix.kube.version }}-* kubectl=${{ matrix.kube.version }}-* containerd.io
kubectl version && echo "kubectl return code: $?" || echo "kubectl return code: $?"
kubeadm version && echo "kubeadm return code: $?" || echo "kubeadm return code: $?"
kubelet --version && echo "kubelet return code: $?" || echo "kubelet return code: $?"
cat <<EOF | sudo tee /etc/modules-load.d/containerd.conf
overlay
br_netfilter
EOF
sudo modprobe overlay
sudo modprobe br_netfilter
cat <<EOF | sudo tee /etc/sysctl.d/99-kubernetes-cri.conf
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
EOF
sudo sysctl --system
sudo mkdir -p /etc/containerd
containerd config default | sudo tee /etc/containerd/config.toml
sudo sed -i "s/SystemdCgroup = false/SystemdCgroup = true/g" /etc/containerd/config.toml
sudo systemctl restart containerd
sudo swapoff -a
sudo kubeadm init --pod-network-cidr=10.244.0.0/16 --cri-socket=unix:///run/containerd/containerd.sock
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
verlte() { [ "$1" = "`echo -e "$1\n$2" | sort -V | head -n1`" ] ; }
verlt() { [ "$1" = "$2" ] && return 1 || verlte $1 $2 ; }
# Remove control plane label
kubectl taint nodes --all node-role.kubernetes.io/control-plane-
until kubectl get node ${HOSTNAME,,} -o jsonpath='{@.metadata.name}:{range @.status.conditions[*]}{@.type}={@.status}' | grep 'Ready=True'; do echo "waiting for kubernetes to become ready"; sleep 10; done
- if: startsWith(matrix.kube.runtime, 'k8s')
name: Output Kubelet, Containerd, Docker Logs
run: |
echo "Kubelet Logs:"
sudo journalctl -xeu kubelet --no-pager
echo "\nContainerd Logs"
sudo journalctl -xeu containerd --no-pager
echo "\nDocker Logs"
sudo journalctl -xeu docker --no-pager
- if: (startsWith(github.event_name, 'pull_request')) && (startsWith(matrix.kube.runtime, 'k8s'))
name: Import local agent and controller to Kubernetes
working-directory: /tmp/images
run: |
# Need to load to containerd with ctr
# -n allows kubernetes to see the image
sudo find -name "*.tar" -type f -exec ctr -n=k8s.io image import {} \;
sudo crictl --runtime-endpoint unix:///var/run/containerd/containerd.sock images
- if: startsWith(matrix.kube.runtime, 'microk8s')
name: Install MicroK8s
run: |
set -x
sudo snap install microk8s --classic --channel=${{ matrix.kube.version }}
sudo microk8s status --wait-ready
sudo usermod -a -G microk8s $USER
mkdir -p $HOME/.kube
sudo microk8s.kubectl config view --raw > $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
sudo microk8s.enable rbac dns
sudo sed -i 's/memory.available<100Mi,nodefs.available<1Gi,imagefs.available<1Gi/memory.available<25Mi,nodefs.available<50Mi,imagefs.available<50Mi/' /var/snap/microk8s/current/args/kubelet
sudo systemctl restart snap.microk8s.daemon-kubelite
until sudo microk8s.status --wait-ready; do sleep 5s; echo "Try again"; done
- if: (startsWith(github.event_name, 'pull_request')) && (startsWith(matrix.kube.runtime, 'microk8s'))
name: Import local agent and controller to MicroK8s
working-directory: /tmp/images
run: |
sudo microk8s.status --wait-ready
until sudo microk8s ctr images ls; do sleep 5s; echo "Try again"; done
sudo microk8s ctr images ls
sudo find -name "*.tar" -type f -exec microk8s ctr --debug --timeout 60s images import {} \;
sudo microk8s ctr images ls
- name: Add Akri Helm Chart
run: helm repo add akri-helm-charts https://project-akri.github.io/akri/
# For push and release, we need to wait for the Helm chart and
# associated containers to build.
- if: github.event_name == 'push' || github.event_name == 'release'
name: sleep before running script for 80 minutes
run: sleep 4800
- name: Execute test script
working-directory: ./e2e
run: |
case "${{ github.event_name }}" in "push");; "release") extra_param="--release";; *) extra_param="--use-local";; esac
/opt/poetry/bin/poetry run pytest -v --distribution ${{ matrix.kube.runtime}} --test-version $(cat ../../version.txt) $extra_param
- name: Sanitize logs artifact name
id: sanitize-artifact-name
if: always()
run: echo "name=${{ matrix.kube.runtime }}-${{ matrix.kube.version }}-logs" | tr "/:" "_" >> $GITHUB_OUTPUT
- name: Upload logs as artifact
if: always()
uses: actions/upload-artifact@v3
with:
name: ${{ steps.sanitize-artifact-name.outputs.name }}
path: /tmp/logs/