diff --git a/.github/workflows/periodic.yaml b/.github/workflows/periodic.yaml new file mode 100644 index 0000000..8800832 --- /dev/null +++ b/.github/workflows/periodic.yaml @@ -0,0 +1,55 @@ +name: Patu Periodic Cloud Performance and Scale Testing + +on: + schedule: + - cron: '0 12 * * *' + +jobs: + deploy-perf-scale: + name: deploy-perf-scale + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + cni: ["patu", "flannel"] + kube-distribution: ["kubeadm"] + env: + JOB_NAME: "patu-periodic-perfscale-${{ matrix.k8s-distro }}-${{ matrix.cni }}" + MATRIX_CNI: ${{ matrix.cni }} + KUBE_DIST: ${{ matrix.k8s-distro }} + AWS_REGION: "us-east-1" + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + steps: + - name: checkout + uses: actions/checkout@v2 + + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install Ansible and dependencies + run: pip3.10 install boto boto3 ansible-vault ansible-core==2.13.3 + + - name: Install amazon.aws Ansible library + run: ansible-galaxy collection install amazon.aws + + - name: Create ansible ssh key + run: | + echo "${{ secrets.ANSIBLE_SSH_KEY }}" > ./test/ansible/periodic/patu-ci.pem + chmod 0400 ./test/ansible/periodic/patu-ci.pem + + - name: Create vault password file + run: | + echo "${{ secrets.ANSIBLE_VAULT_PASSWORD }}" > /home/runner/work/patu/patu/vault-secret.txt + chmod 0400 vault-secret.txt + + - name: Deploy EC2 Playbooks + run: | + ansible-playbook -vv ./test/ansible/periodic/deploy.yml --extra-vars "MATRIX_CNI=${{ matrix.cni }}" --vault-password-file /home/runner/work/patu/patu/vault-secret.txt + rm vault-secret.txt + rm patu-ci.pem + + - name: Display Iperf3 Results for ${{ matrix.cni }} + run: cat ./test/ansible/periodic/iperf-results-${{ matrix.cni }}.txt diff --git a/.gitignore b/.gitignore index 008c8bc..7c0ddd7 100644 --- a/.gitignore +++ b/.gitignore @@ -58,3 +58,6 @@ dkms.conf # jetbrains config files .idea/ + +# miscellaneous +*.pem diff --git a/.licenserc.yaml b/.licenserc.yaml index ca49d2d..684054d 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -34,3 +34,4 @@ header: - '**/*.yaml' - '**/*.yml' - '.clang-format' + - 'test/ansible/' diff --git a/test/ansible/periodic/ansible.cfg b/test/ansible/periodic/ansible.cfg new file mode 100644 index 0000000..353c06a --- /dev/null +++ b/test/ansible/periodic/ansible.cfg @@ -0,0 +1,10 @@ +[defaults] +host_key_checking = false +deprecation_warnings = false +ask_pass = false +stdout_callback = yaml +remote_user = ubuntu +# defaults to the base directory in the project +inventory = inventory.txt +# create .pem private_key_file and provide location +private_key_file = patu-ci.pem diff --git a/test/ansible/periodic/deploy.yml b/test/ansible/periodic/deploy.yml new file mode 100644 index 0000000..d8423ec --- /dev/null +++ b/test/ansible/periodic/deploy.yml @@ -0,0 +1,64 @@ +# roles get branched from here +- hosts: localhost + vars_files: + - vars.yml + roles: + - role: setup-ec2 + +- hosts: singleNodeCluster + roles: + - role: install-kubeadm + environment: + KUBECONFIG: /home/{{ ansible_user }}/.kube/config + when: MATRIX_CNI == "patu" + +- hosts: singleNodeCluster + roles: + - role: install-kubeadm + environment: + KUBECONFIG: /home/{{ ansible_user }}/.kube/config + when: MATRIX_CNI == "flannel" + +- hosts: singleNodeCluster + roles: + - role: install-cni + environment: + KUBECONFIG: /home/{{ ansible_user }}/.kube/config + when: MATRIX_CNI == "patu" + +- hosts: singleNodeCluster + roles: + - role: install-cni + environment: + KUBECONFIG: /home/{{ ansible_user }}/.kube/config + when: MATRIX_CNI == "flannel" + +- hosts: singleNodeCluster + roles: + - role: run-iperf + environment: + KUBECONFIG: /home/{{ ansible_user }}/.kube/config + vars: + MATRIX: patu-kpng-kubeadm + when: MATRIX_CNI == "patu" + +- hosts: singleNodeCluster + roles: + - role: run-iperf + environment: + KUBECONFIG: /home/{{ ansible_user }}/.kube/config + vars: + MATRIX: flannel-kubeproxy-kubeadm + when: MATRIX_CNI == "flannel" + +- hosts: singleNodeCluster + roles: + - role: reset-kubeadm + +# TODO: cleanup using explicit node names from inventory instead of NodeTag +# TODO: but what about a scenario where the runners are spun up but a step fails? +#- hosts: localhost +# vars_files: +# - vars.yml +# roles: +# - role: terminate-ec2 diff --git a/test/ansible/periodic/install-cni/tasks/main.yml b/test/ansible/periodic/install-cni/tasks/main.yml new file mode 100644 index 0000000..209a1c2 --- /dev/null +++ b/test/ansible/periodic/install-cni/tasks/main.yml @@ -0,0 +1,47 @@ +--- +# tasks file for install-cni +- name: Verify kubectl + command: kubectl get pods --all-namespaces + +### Patu Installer Section ### +- name: Copy the Patu repo to the remote host + copy: + src: ../../../../patu/ + dest: /home/{{ ansible_user }}/patu/ + when: MATRIX_CNI == "patu" + +- name: Change file ownership, group and permissions + ansible.builtin.file: + path: "/home/{{ ansible_user }}/patu/deploy/kubernetes/patu-installer" + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + mode: "0755" + when: MATRIX_CNI == "patu" + +- name: Install KPNG and Patu + shell: | + PATU_CONFIG=/home/{{ ansible_user }}/patu/deploy/patu.yaml \ + KPNG_CONFIG=/home/{{ ansible_user }}/patu/deploy/kpngebpf.yaml \ + /home/{{ ansible_user }}/patu/deploy/kubernetes/patu-installer apply all + when: MATRIX_CNI == "patu" + +- name: Wait for CoreDNS pods to become ready + shell: kubectl wait --for=condition=ready pods -l k8s-app=kube-dns -n kube-system --timeout=30s + when: MATRIX_CNI == "patu" + +### Flannel Installer Section ### +- name: Deploy kubeadm for the Flannel CNI for the Flannel matrix + shell: kubectl apply -f https://github.com/coreos/flannel/raw/master/Documentation/kube-flannel.yml + when: MATRIX_CNI == "flannel" + +- name: Remove kubeadm taints + shell: kubectl taint nodes --all node-role.kubernetes.io/control-plane- node-role.kubernetes.io/master- + when: MATRIX_CNI == "flannel" + +- name: Pause for flannel convergence + pause: + seconds: 10 + when: MATRIX_CNI == "flannel" + +- name: Display kube pods + command: kubectl get pods --all-namespaces diff --git a/test/ansible/periodic/install-cni/vars/main.yml b/test/ansible/periodic/install-cni/vars/main.yml new file mode 100644 index 0000000..a9a9c94 --- /dev/null +++ b/test/ansible/periodic/install-cni/vars/main.yml @@ -0,0 +1,2 @@ +--- +# vars file for install-cni diff --git a/test/ansible/periodic/install-kubeadm/tasks/main.yml b/test/ansible/periodic/install-kubeadm/tasks/main.yml new file mode 100644 index 0000000..2d5ffc5 --- /dev/null +++ b/test/ansible/periodic/install-kubeadm/tasks/main.yml @@ -0,0 +1,108 @@ +--- +# tasks file for install-kubeadm + +- name: Update repo cache + become: yes + apt: + update_cache: yes + +- name: Install dependencies + become: yes + apt: + name: + - apt-transport-https + - ca-certificates + - curl + - gnupg2 + - software-properties-common + state: latest + +- name: Host configurations + shell: | + sudo sysctl -w net.ipv4.ip_forward=1 + sudo modprobe br_netfilter + +- name: Host configurations + shell: | + sudo sysctl -w net.ipv4.ip_forward=1 + sudo modprobe br_netfilter + +- name: Configure cri-o repos + vars: + OS: "xUbuntu_20.04" + CRIO_VERSION: "1.23" + shell: | + echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/{{ OS }}/ /"|sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list + echo "deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/{{ CRIO_VERSION }}/{{ OS }}/ /"|sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable:cri-o:{{ CRIO_VERSION }}.list + curl -L https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable:cri-o:{{ CRIO_VERSION }}/{{ OS }}/Release.key | sudo apt-key add - + curl -L https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/{{ OS }}/Release.key | sudo apt-key add - + ignore_errors: true + +- name: Configure kube repos + shell: | + sudo curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packages.cloud.google.com/apt/doc/apt-key.gpg + echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list + +- name: Update repo cache + become: yes + apt: + update_cache: yes + +- name: Install cri-o + become: yes + apt: + name: + - cri-o + - cri-o-runc + state: latest + +- name: Enable cri-o systemd + shell: | + sudo systemctl enable crio.service + sudo systemctl start crio.service + +- name: Install kube binaries + vars: + K8S_VERSION: "1.24.4-00" + shell: sudo apt install -y kubeadm={{ K8S_VERSION }} kubelet={{ K8S_VERSION }} kubectl={{ K8S_VERSION }} + +- name: Deploy kubeadm for the Patu matrix with kubeproxy disabled + shell: sudo kubeadm init --upload-certs --pod-network-cidr=10.200.0.0/16 --v=6 --skip-phases=addon/kube-proxy + when: MATRIX_CNI == "patu" + +- name: Deploy kubeadm for the Flannel CNI matrix with kubeproxy enabled + shell: sudo kubeadm init --pod-network-cidr=10.244.0.0/16 + when: MATRIX_CNI == "flannel" + +- name: Wait for kubeconfig to be created + become: yes + wait_for: + path: /etc/kubernetes/admin.conf + state: present + timeout: 30 + ignore_errors: True + +- name: Creating the .kube directory + file: + path: /home/{{ ansible_user }}/.kube/ + state: directory + +- name: Copying kubeconfig to .kube directory + become: yes + copy: + remote_src: yes + src: /etc/kubernetes/admin.conf + dest: /home/{{ ansible_user }}/.kube/config + +- name: Change the owner of .kube/config + shell: "sudo chown $(id -u {{ ansible_user }}):$(id -g {{ ansible_user }}) /home/{{ ansible_user }}/.kube/config" + +- name: export KUBECONFIG + shell: export KUBECONFIG=/home/{{ ansible_user }}/.kube/config + +- name: Pause for convergence + pause: + seconds: 15 + +- name: Verify kubectl + command: kubectl get pods --all-namespaces diff --git a/test/ansible/periodic/install-kubeadm/vars/main.yml b/test/ansible/periodic/install-kubeadm/vars/main.yml new file mode 100644 index 0000000..b986df2 --- /dev/null +++ b/test/ansible/periodic/install-kubeadm/vars/main.yml @@ -0,0 +1,2 @@ +--- +# vars file for install-kubeadm diff --git a/test/ansible/periodic/inventory.txt b/test/ansible/periodic/inventory.txt new file mode 100644 index 0000000..a1df41a --- /dev/null +++ b/test/ansible/periodic/inventory.txt @@ -0,0 +1 @@ +[singleNodeCluster] diff --git a/test/ansible/periodic/reset-kubeadm/tasks/main.yml b/test/ansible/periodic/reset-kubeadm/tasks/main.yml new file mode 100644 index 0000000..66ba1e8 --- /dev/null +++ b/test/ansible/periodic/reset-kubeadm/tasks/main.yml @@ -0,0 +1,18 @@ +--- +- name: Verify kubectl + command: kubectl get pods --all-namespaces + +- name: Reset kubeadm + shell: | + sudo kubeadm -f reset + sudo crictl rm -f `crictl ps -a | grep "k8s_" | awk '{print $1}'` + # Remove all the patu images. + sudo apt purge kubectl kubeadm kubelet kubernetes-cni -y --allow-change-held-packages && apt autoremove -y + sudo rm -fr /etc/kubernetes/; sudo rm -fr ~/.kube/; sudo rm -fr /var/lib/etcd; sudo rm -rf /var/lib/cni/ + sudo systemctl restart crio.service + sudo systemctl daemon-reload + sudo iptables -F + sudo iptables -t nat -F + sudo iptables -t mangle -F + sudo iptables -X + sudo iptables -L diff --git a/test/ansible/periodic/reset-kubeadm/vars/main.yml b/test/ansible/periodic/reset-kubeadm/vars/main.yml new file mode 100644 index 0000000..f25b9a7 --- /dev/null +++ b/test/ansible/periodic/reset-kubeadm/vars/main.yml @@ -0,0 +1,2 @@ +--- +# vars file for reset-kubeadm diff --git a/test/ansible/periodic/run-iperf/tasks/main.yml b/test/ansible/periodic/run-iperf/tasks/main.yml new file mode 100644 index 0000000..98fdc9c --- /dev/null +++ b/test/ansible/periodic/run-iperf/tasks/main.yml @@ -0,0 +1,71 @@ +--- +- name: Verify kubectl + command: kubectl get pods --all-namespaces + +- name: Deploy the iperf3 server + shell: | + cat << EOF | kubectl apply -f - + apiVersion: v1 + kind: Pod + metadata: + name: iperf3-svr + labels: + app: iperf3-svr + spec: + containers: + - name: iperf3-svr + image: networkstatic/iperf3 + ports: + - containerPort: 5201 + args: ["-s"] + EOF + +- name: Pause for the iperf3 server to initialize + pause: + seconds: 10 + +- name: Register the IP address of the iperf-svr pod + shell: | + kubectl get pods -l app=iperf3-svr -o custom-columns=IP:status.podIP --no-headers + register: iperf_svr_ip + +- name: Display the IP address of the iperf-svr pod + debug: + msg: iperf-svr pod address "{{ iperf_svr_ip.stdout }}" + +- name: Print the iperf3 client yaml to file + shell: | + cat < iperf-client.yaml + apiVersion: v1 + kind: Pod + metadata: + name: iperf3-client + labels: + app: iperf3-client + spec: + containers: + - name: iperf3-client + image: networkstatic/iperf3 + args: ["-c", "{{ iperf_svr_ip.stdout }}"] + restartPolicy: Never + EOF + +- name: Run the iperf3 client + shell: kubectl apply -f iperf-client.yaml + +- name: Pause for the iperf3-client to initialize and run performance test + pause: + seconds: 25 + +- name: Create a results file + shell: | + printf "====== Performance Matrix: {{ MATRIX }} ======\n" > iperf-results-{{ MATRIX_CNI }}.txt + +- name: Log test results to a file + shell: kubectl logs iperf3-svr >> iperf-results-{{ MATRIX_CNI }}.txt + +- name: Copy iperf results back to runner + ansible.builtin.fetch: + src: /home/{{ ansible_user }}/iperf-results-{{ MATRIX_CNI }}.txt + dest: ./ + flat: true \ No newline at end of file diff --git a/test/ansible/periodic/run-iperf/vars/main.yml b/test/ansible/periodic/run-iperf/vars/main.yml new file mode 100644 index 0000000..3de7b3d --- /dev/null +++ b/test/ansible/periodic/run-iperf/vars/main.yml @@ -0,0 +1,2 @@ +--- +# vars file for run-iperf diff --git a/test/ansible/periodic/setup-ec2/tasks/main.yml b/test/ansible/periodic/setup-ec2/tasks/main.yml new file mode 100644 index 0000000..2d0b164 --- /dev/null +++ b/test/ansible/periodic/setup-ec2/tasks/main.yml @@ -0,0 +1,52 @@ +--- +# tasks file for setup-ec2 +- name: Installing boto library + pip: + name: boto + state: present + +- name: Generate a UUID for the ec2 host + shell: uuidgen | head -c 6 + register: uuid + +- name: Creating Security Group for Patu CI + amazon.aws.ec2_group: + name: "{{ secgroup_name }}" + aws_region: "{{ aws_region }}" + description: "{{ security_group_description }}" + vpc_id: "{{ vpc_id }}" + rules: + - proto: all + cidr_ip: "0.0.0.0/0" + +- name: Launching Single Node Cluster Machines + amazon.aws.ec2_instance: + name: "single-node-cluster-{{ item+1 }}-{{ uuid.stdout }}" + aws_region: "{{ aws_region }}" + key_name: "{{ aws_key_name }}" + instance_type: "{{ aws_instance_type }}" + image_id: "{{ aws_image_id }}" + security_group: "{{ secgroup_name }}" + network: + assign_public_ip: true + subnet_id: "{{ aws_subnet }}" + tags: + NodeType: "patu-ci-single-node-cluster" + state: running + wait: true + register: nodeIP + loop: "{{ range(0, node_count | int) }}" + +- name: Updating the node's public ip in inventory + lineinfile: + path: "{{ inventory_location }}" + regexp: "singleNodeCluster" + line: "[singleNodeCluster]\n{{ nodeIP['results'][item]['instances'][0]['public_ip_address']}} ansible_user={{ ansible_user }} ansible_connection=ssh node-name=single-node-cluster-{{ item+1 }}-{{ uuid.stdout }}" + loop: "{{ range(0, node_count | int) }}" + +- name: Refresh inventory to ensure new instaces exist in inventory + meta: refresh_inventory + +- name: Pause for a few seconds to allow the instances to finish booting + pause: + seconds: 20 \ No newline at end of file diff --git a/test/ansible/periodic/setup-ec2/vars/main.yml b/test/ansible/periodic/setup-ec2/vars/main.yml new file mode 100644 index 0000000..2452dd9 --- /dev/null +++ b/test/ansible/periodic/setup-ec2/vars/main.yml @@ -0,0 +1,2 @@ +--- +# vars file for setup-ec2 diff --git a/test/ansible/periodic/terminate-ec2/tasks/main.yml b/test/ansible/periodic/terminate-ec2/tasks/main.yml new file mode 100644 index 0000000..d5f863d --- /dev/null +++ b/test/ansible/periodic/terminate-ec2/tasks/main.yml @@ -0,0 +1,9 @@ +--- +# tasks file for terminate-ec2 +- name: Terminate all ec2 instances in a region with the tag patu-ci-single-node-cluster + become: false + ec2_instance: + state: absent + filters: + "tag:NodeType": "{{ aws_nodetype_tag }}" + instance-state-name: running \ No newline at end of file diff --git a/test/ansible/periodic/terminate-ec2/vars/main.yml b/test/ansible/periodic/terminate-ec2/vars/main.yml new file mode 100644 index 0000000..6ca22e4 --- /dev/null +++ b/test/ansible/periodic/terminate-ec2/vars/main.yml @@ -0,0 +1,2 @@ +--- +# vars file for terminate-ec2 diff --git a/test/ansible/periodic/vars.yml b/test/ansible/periodic/vars.yml new file mode 100644 index 0000000..3aa99c4 --- /dev/null +++ b/test/ansible/periodic/vars.yml @@ -0,0 +1,64 @@ +$ANSIBLE_VAULT;1.1;AES256 +63613164666163393863346664313261306139613737633936653961363932613361363735396135 +3038656533383437346537353634363935656237353130330a343563373433666130646636366662 +35613063623535353432623030316335616661373039643065316436333566653738376565306665 +3462643432363462380a313731396637613134353265363561663966636236313561633235316636 +38343962653031386362333163366361316333313461313361623739613539316236633934653432 +65313433656335663961633233303633343163333962633435326633323934313366623231303731 +31336630353064363565326139336633653638343735613961363339343531333561376136356230 +30666466356663346338616161343036306564656166616333383431356661636138336161343562 +36373731633630316365356266313139666164303331656633346633313761643338323637383637 +38323739373764643731326261363336656532376436393535646339393835313735373835333963 +64363137303739653537336561633231373438303934313765333830643037356463646633383561 +39373966363430336264363939656636333037383864356138393136656235343463356432333734 +39643134643339646635616432303763666136643666643261663335346366373763656638386437 +38383333306339393864646430653230663938656566623765333063366639623562326135663636 +35383239353534653432646435353832323830363036373537646338626362353964303064343933 +31613866343662353630623339656130346333376365333331313739343633383366333264383262 +30616636393463653232393534323439336263663663366439643364653831663563633365626166 +64386532623732626266653934313161313237623064313737663933656364633465396165616165 +38303834303430333231626238653130646365343230383761656638383433333137386332386563 +33356531346563376532346333363234396438303037666134643261396262396462393630633330 +61343839313231636632633537396465613237376335393838343362643935343862313663353563 +38373736613934303665643061663438663536346332323235393035326631356436313431373433 +61363234396262316664336132386533303835393238386463636539613263343362616131336339 +31643735396538613734653233323134323661323863353561396562323030623339323736356165 +34643266326233393632346430326465363738656133356463373863633030336633356534393736 +34303964393039346337336134383930653831313062623632346136663865663339326238313530 +34623364363433656465373863626166326135393935333636653063346236616430393238346330 +30373261346436623265383438666632373164663835346339306536316239356436333237653936 +66363161636333383239653637326133353830356538646237303032656465336432336137636262 +35313932306432613963303735613438363739663666393231646137653339373664636236373066 +64393336323135616461396533666231643130373166376138616666343234653632626639656163 +30363364663836623434396466613964316539343363336235613466333664663839666433383134 +63353662373830343732383162636662666562333039356235643333326561366438373461396636 +31393133363765653432336666333762663765623234646531386562346234616639386130356537 +34613638343164666563393735613532656134353063613937383732343061613331396363633333 +30636439373839383766663361386464646633383336323531643264313633323731666561623638 +31643831336239663231343365386539666333626362633963653638366230653334336264373830 +63306336666431326431363363656337376562333031386335396663326537313534393936373736 +63316563353230666634333464623734353863613337666432343330353138323631613430363965 +62613837383062663439333665666237313563396265666636626435663664303664613463663334 +36643864623865353662613461366339376338386433653637373739323230376134373735646130 +38366633353933613037663630346335306134336438396664316330383535363330336264336535 +64663035333461653935303764346630303532623561373830383533363539343832313938393234 +64303933353734343261626130353337633533313961623132663031313965323366306130623162 +64333333623432636338653932653439663434333837363833356232363062333338616162626663 +31633661353662336361306433396663626435623130346365353833633031326237666534353332 +36356465373061313166383263333531363630356261306664666638623466303630663465643932 +61393337623532363334363564653432616238323262363763663231336263633463613262623934 +63303961616235383663303939353633613833313664353831653332623036366434396333326436 +32343831636263396537663831336637353261313832393630643337373331623663396338613463 +32393239313833346563653664656338353833613661356431353262383438333138663637333839 +36333939656665656339356462653330643030656161323839336534646466306135633839386664 +62663336316438316339653861653036353165646135383765313661323132626530393431363763 +39633763346430343032336165366437623033656137383438653538356239393330636436663364 +65633638383934623635656231386531653439663064666436346361613230303861313866316338 +34356434303666326462653463316664366336623337653737666435313030643966353466653339 +63326230303531326132336264636530656530323731663662336535386461333931333164333934 +34383930613230356166613861363766306162333636316535303861366437326536636138663435 +65616664313266356334656664303065663763626632323238643664643261363230383637663232 +33626635383061333734643731363735656138316237643137306333383766356337363465666638 +63626533373739343539653632653232343033646336343161303233666362363763653238373232 +63633137346164366333393635613432346635636534336432656532396166386232386335316433 +663632636436386639313438666632353633