Initial commit

.clang-format

@@ -0,0 +1,16 @@
+---
+Language: Cpp
+BasedOnStyle: LLVM
+IndentWidth: 4
+ColumnLimit: 100
+AllowShortBlocksOnASingleLine: Empty
+# AllowShortFunctionsOnASingleLine: Inline
+AllowShortLoopsOnASingleLine: true
+AllowShortIfStatementsOnASingleLine: Never
+AlignConsecutiveMacros: true
+AlignEscapedNewlines: Left
+BreakBeforeBraces: Custom
+BraceWrapping:
+    AfterFunction: true
+PointerAlignment: Right
+---

.github/workflows/build.yml

@@ -0,0 +1,30 @@
+name: Build CI
+
+on: [push, pull_request]
+
+jobs:
+  build-libos:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+    steps:
+    - uses: actions/checkout@v3
+    - name: Install dependencies
+      run: sudo apt-get install -y cmake libnuma-dev libgflags-dev zlib1g-dev libzstd-dev
+    - name: Cache build
+      uses: actions/cache@v3
+      with:
+        path: ${{github.workspace}}/build
+        key: ${{ runner.os }}-build-${{ env.cache-name }}
+    - name: Build libos and apps
+      run: CMAKE_ARGS="-DROCKSDB_JOBS=1" SCHED=fifo make DPDK=0
+
+
+  build-kmod:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+    steps:
+    - uses: actions/checkout@v3
+    - name: Build kernel module
+      run: make kmod

.gitignore

@@ -0,0 +1,17 @@
+.vscode
+*.o
+*.ko
+*.cmd
+*.mod
+*.mod.c
+*.symvers
+*.order
+.DS_Store
+build/
+main
+.cache/
+__pycache__/
+compile_commands.json
+!extra.symvers
+include/skyloft/uapi/params.h
+include/skyloft/params.h

.gitmodules

@@ -0,0 +1,8 @@
+[submodule "apps/schbench"]
+	path = apps/schbench
+	url = [email protected]:yhtzd/schbench.git
+	branch = skyloft
+[submodule "apps/memcached"]
+	path = apps/memcached
+	url = [email protected]:yhtzd/memcached.git
+	branch = skyloft

CMakeLists.txt

@@ -0,0 +1,93 @@
+cmake_minimum_required(VERSION 3.5)
+project(skyloft LANGUAGES C CXX ASM)
+
+if(DEBUG)
+    set(CMAKE_BUILD_TYPE "Debug")
+    add_definitions(-DDEBUG)
+endif()
+
+set(CMAKE_C_STANDARD 23)
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O3 -Wall -Wextra -Wno-unused-parameter")
+if (DPDK)
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
+elseif (SCHED_POLICY MATCHES "^(fifo|rr|cfs)$")
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mno-sse")
+endif()
+
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wextra -Wall")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wsign-compare -Wshadow -Wno-unused-parameter -Wno-unused-variable -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers -Wno-strict-aliasing")
+
+set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -T ${CMAKE_CURRENT_SOURCE_DIR}/libos/libos.ld")
+
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin)
+
+if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer")
+
+    include(CheckCXXCompilerFlag)
+    CHECK_CXX_COMPILER_FLAG("-momit-leaf-frame-pointer" HAVE_OMIT_LEAF_FRAME_POINTER)
+
+    if(HAVE_OMIT_LEAF_FRAME_POINTER)
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -momit-leaf-frame-pointer")
+    endif()
+endif()
+
+if(NOT USE_RTTI)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-rtti")
+endif()
+
+# Hook exit in all targets
+add_compile_options(-Wl,--wrap=exit)
+add_link_options(-Wl,--wrap=exit)
+
+set(CMAKE_SCRIPTS ${CMAKE_CURRENT_SOURCE_DIR}/scripts/cmake/)
+
+if(STAT)
+    add_definitions(-DSKYLOFT_STAT)
+endif()
+
+if(UINTR)
+    add_definitions(-DSKYLOFT_UINTR)
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -muintr")
+endif()
+
+if(LOG_LEVEL)
+    if(NOT LOG_LEVEL MATCHES "^(debug|info|notice|warn|err|crit)$")
+        message(FATAL_ERROR "Invalid log level: ${LOG_LEVEL}")
+    endif()
+
+    string(TOUPPER ${LOG_LEVEL} LOG_LEVEL_UPPER)
+    add_definitions(-DLOG_LEVEL_${LOG_LEVEL_UPPER})
+endif()
+
+message(STATUS "Log level: ${LOG_LEVEL}")
+
+if(NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "")
+    set(CMAKE_BUILD_TYPE "Release" CACHE STRING "" FORCE)
+endif()
+
+message(STATUS "Build type: ${CMAKE_BUILD_TYPE}")
+message(STATUS "Schedule policy: ${SCHED_POLICY}")
+
+if(SCHED_POLICY STREQUAL "fifo")
+    add_definitions(-DSKYLOFT_SCHED_FIFO)
+elseif(SCHED_POLICY STREQUAL "rr")
+    add_definitions(-DSKYLOFT_SCHED_FIFO2)
+elseif(SCHED_POLICY STREQUAL "cfs")
+    add_definitions(-DSKYLOFT_SCHED_CFS)
+elseif(SCHED_POLICY STREQUAL "sq")
+    add_definitions(-DSKYLOFT_SCHED_SQ)
+elseif(SCHED_POLICY STREQUAL "sq_lcbe")
+    add_definitions(-DSKYLOFT_SCHED_SQ_LCBE)
+endif()
+
+add_subdirectory(utils)
+add_subdirectory(libos)
+
+if (SCHED_POLICY MATCHES "^(sq|sq_lcbe)$")
+    add_subdirectory(synthetic)
+else()
+    add_subdirectory(apps)
+endif()
+

Makefile

@@ -0,0 +1,60 @@
+SIGNAL ?=
+DPDK ?= 1
+TIMER ?= 1
+UINTR ?= 0
+SCHED ?= fifo
+DAEMON ?=
+DEBUG ?=
+STAT ?=
+LOG ?= info
+
+CC ?= gcc
+CFLAGS := -Wall -O2 -D_GNU_SOURCE
+CMAKE_ARGS ?=
+
+CMAKE_ARGS += -DSCHED_POLICY=$(SCHED)
+CMAKE_ARGS += -DSIGNAL=$(SIGNAL) -DDPDK=$(DPDK) -DTIMER=$(TIMER) -DUINTR=$(UINTR) -DDAEMON=$(DAEMON) -DDEBUG=$(DEBUG) -DSTAT=$(STAT) -DLOG_LEVEL=$(LOG)
+CMAKE_ARGS += -DCMAKE_INSTALL_PREFIX=install
+
+all: build
+
+build:
+	mkdir -p build
+	cd build && cmake .. $(CMAKE_ARGS) && make VERBOSE=1 -j4
+
+install: build
+	cd build && make install
+
+kmod:
+	cd kmod && make
+
+insmod:
+	cd kmod && make insmod
+
+tests:
+	cd tests && make
+
+memcached: install
+	cd build && make memcached VERBOSE=1
+
+zstd: install
+	cd build && make zstd VERBOSE=1
+
+schbench: install
+	cd build && make schbench VERBOSE=1
+
+rocksdb: install
+	cd build && make rocksdb_server VERBOSE=1
+
+fmt:
+	@clang-format --style=file -i $(shell find utils/ libos/ apps/ tests/ experiments/ -iname '*.c' -o -iname '*.cc' -o -iname '*.h')
+
+clean:
+	make -C build clean
+
+distclean: clean
+	rm -rf build
+	make -C tests clean
+	make -C kmod clean
+
+.PHONY: all clean fmt kmod build tests install

README.md

@@ -0,0 +1,156 @@
+# Skyloft
+
+## Contents
+
+- [Overview](#overview)
+- [Setup](#setup)
+- [Related Work](#related-work)
+
+## Overview
+
+<div align="left">
+<img src=docs/imgs/overview.jpg width=60% />
+</div>
+
+### Why User Interrupt?
+
+<div align="left">
+<img src=docs/imgs/uintr.svg width=40% />
+</div>
+
+### Layout
+
+- `apps/`: Benchmark real-world applications
+- `docs/`: Documents and images
+- `synthetic/`: Benchmark c-FCFS and PS scheduling policies
+    - `rocksdb/`: Latency-critical application
+    - `antagonist/`: Batch application
+- `kmod/`: Skyloft kernel module
+- `libos/`: Skyloft main code
+    - `io/`: IO thread
+    - `net/`: Network stack
+    - `shim/`: Shim layer for POSIX APIs
+    - `sync/`: Synchronization primitives
+    - `mm/`: Memory management
+    - `sched/`: Schedulers
+- `utils/`: Useful tools
+- `scripts/`: Setup machine; run experiments
+- `tests/`: Microbenchmarks and prototypes
+
+## Setup
+
+### Testbed
+
+* Intel(R) Xeon(R) Gold 5418Y @ 2.0GHZ
+* 2 sockets, 24 cores per socket
+* 128 GB memory
+* hyperthreading disabled, dynamic frequency scaling disabled
+
+Build kmod:
+
+```sh
+make
+CPU_LIST=<cpu list> make insmod
+```
+
+### Schbench
+
+24 cores
+
+```sh
+make schbench DPDK=0 UINTR=1 SCHED=<rr|cfs>
+./scripts/run.sh schbench -n10 -F128 -m1 -r10 -i5 -t<24|48|96>
+```
+
+### Synthetic
+
+20 workers, 1 dispatcher, 99.5% 4us, 5% 10000us
+
+1. LC
+
+```sh
+make build SCHED=sq UINTR=1 DPDK=0
+./scripts/run.sh shinjuku \
+    --get_service_time=4000 \
+    --range_query_service_time=10000000 \
+    --fake_work \
+    --load=0.4 \ # change the target throughput
+    --range_query_ratio=0.005 \
+    --preemption_quantum=30 \ # change the preemption quantum
+    --output_path=./data \
+    --run_time=5 \
+    --num_workers=20
+```
+
+2. LC + BE
+
+```sh
+make build SCHED=sq_lcbe UINTR=1 DPDK=0
+# 1
+./scripts/run.sh shinjuku \
+    --get_service_time=4000 \
+    --range_query_service_time=10000000 \
+    --fake_work \
+    --load=0.4 \
+    --range_query_ratio=0.005 \
+    --preemption_quantum=30 \
+    --output_path=./data \
+    --run_time=5 \
+    --guaranteed_cpus=4 \ # change the cores LC app gets at first
+    --num_workers=20
+# 2
+sudo ./build/bin/antagonist --num_workers=20 --run_time=5
+```
+
+
+### Mircobenchmarks
+
+#### Operation latency
+
+| Operations                  | Latency (cycles) | Latency (ns) |
+| --------------------------- | ---------------- | ------------ |
+| `rdtsc`                     | 34               | 17           |
+| `clock_gettime`             | 54               | 27           |
+| Null syscall (ioctl)        | 317              | 159          |
+| `senduipi` (not suppressed) | 354              | 177          |
+| `senduipi` (suppressed)     | 123              | 62           |
+| Timer handled by user       | 628              | 314          |
+| Send APIC IPI (ioctl)       | 633              | 317          |
+| Send-receive (senduipi)     | 1211             | 606          |
+| Send-receive (APIC IPI)     | 1431             | 715          |
+
+## Related Work
+
+| Related Works             | Cross-App Scheduling | Preemption Support and Overhead | Generality | Kernel Modification | App Modification |
+| ------------------------- | :------------------: | :-----------------------------: | :--------: | :-----------------: | :--------------: |
+| User-level threads        |          ✗           |                -                |    Poor    |         No          |        No        |
+| IX                        |          ✗           |                -                |    Poor    |        Major        |      Minor       |
+| ZygOS                     |          ✗           |                -                |    Poor    |        Major        |      Minor       |
+|                           |                      |                                 |            |                     |                  |
+| Arachne                   |          ✓           |                -                |    Poor    |         No          |      Minor       |
+| Shenango                  |          ✓           |                -                |    Poor    |         No          |      Minor       |
+| Caladan                   |          ✓           |                -                |    Poor    |        Minor        |      Minor       |
+|                           |                      |                                 |            |                     |                  |
+| Shinjuku                  |          ✗           |               Low               |    Poor    |        Major        |      Major       |
+| Concord                   |          ✗           |            Very low             |    Poor    |         No          |      Major       |
+| LibPreemptible            |          ✗           |            Very low             |    Poor    |         No          |      Major       |
+|                           |                      |                                 |            |                     |                  |
+| Linux scheduler framework |          ✓           |              High               |    Good    |         No          |        No        |
+| ghOSt                     |          ✓           |              High               |    Good    |        Major        |        No        |
+| Our work                  |          ✓           |            Very low             |    Good    |        Minor        |        No        |
+
+* Multi-App Scheduling: Whether multiple applications are supported. For example, running LC + BE applications at the same time can benefit CPU utilization.
+
+* Preemption Support and Overhead: Whether preemption is supported. This significantly affects the tail delay in long and short request scenarios.
+
+* Generality: Whether it is generic and can be adapted to various scheduling strategies. For example, Arachne/Shenango/Caladan requires a global dispatcher and does not support per-CPU scheduling or deadline-based scheduling.
+
+* Kernel Modification:
+  * **No**: there is no change
+  * **Minor**: Only one kernel module is required
+  * **Major**: The kernel source code needs to be modified.
+
+* App Modification:
+  * **No**: there is no change.
+  * **Minor**: Simple API replacement.
+  * **Major**: Provides an API different from the POSIX interface and requires a lot of modifications.