Update

yhtzd · Aug 29, 2024 · ef40bd8 · ef40bd8
1 parent 42290e6
commit ef40bd8
Show file tree

Hide file tree

Showing 107 changed files with 2,664 additions and 1,058 deletions.
diff --git a/.clang-format b/.clang-format
@@ -4,7 +4,7 @@ BasedOnStyle: LLVM
 IndentWidth: 4
 ColumnLimit: 100
 AllowShortBlocksOnASingleLine: Empty
-# AllowShortFunctionsOnASingleLine: Inline
+AllowShortFunctionsOnASingleLine: Inline
 AllowShortLoopsOnASingleLine: true
 AllowShortIfStatementsOnASingleLine: Never
 AlignConsecutiveMacros: true

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -8,11 +8,6 @@ endif()
 
 set(CMAKE_C_STANDARD 23)
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O3 -Wall -Wextra -Wno-unused-parameter")
-if (DPDK)
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
-elseif (SCHED_POLICY MATCHES "^(fifo|rr|cfs)$")
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mno-sse")
-endif()
 
 set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wextra -Wall")
@@ -52,6 +47,10 @@ if(UINTR)
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -muintr")
 endif()
 
+if(FXSAVE)
+    add_definitions(-DSKYLOFT_FXSAVE)
+endif()
+
 if(LOG_LEVEL)
     if(NOT LOG_LEVEL MATCHES "^(debug|info|notice|warn|err|crit)$")
         message(FATAL_ERROR "Invalid log level: ${LOG_LEVEL}")
@@ -60,7 +59,6 @@ if(LOG_LEVEL)
     string(TOUPPER ${LOG_LEVEL} LOG_LEVEL_UPPER)
     add_definitions(-DLOG_LEVEL_${LOG_LEVEL_UPPER})
 endif()
-
 message(STATUS "Log level: ${LOG_LEVEL}")
 
 if(NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "")
@@ -91,3 +89,9 @@ else()
     add_subdirectory(apps)
 endif()
 
+add_custom_target(
+    microbench
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/microbench
+    COMMAND make clean && make SKYLOFT_DIR=${CMAKE_CURRENT_BINARY_DIR}/install
+    COMMAND ln -sf ${CMAKE_CURRENT_SOURCE_DIR}/microbench/thread ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/thread
+)
diff --git a/Makefile b/Makefile
@@ -5,15 +5,25 @@ UINTR ?= 0
 SCHED ?= fifo
 DAEMON ?=
 DEBUG ?=
-STAT ?=
+STAT ?= 0
 LOG ?= info
+FXSAVE ?= 0
 
 CC ?= gcc
 CFLAGS := -Wall -O2 -D_GNU_SOURCE
 CMAKE_ARGS ?=
 
 CMAKE_ARGS += -DSCHED_POLICY=$(SCHED)
-CMAKE_ARGS += -DSIGNAL=$(SIGNAL) -DDPDK=$(DPDK) -DTIMER=$(TIMER) -DUINTR=$(UINTR) -DDAEMON=$(DAEMON) -DDEBUG=$(DEBUG) -DSTAT=$(STAT) -DLOG_LEVEL=$(LOG)
+CMAKE_ARGS += \
+	-DSIGNAL=$(SIGNAL) \
+	-DDPDK=$(DPDK) \
+	-DTIMER=$(TIMER) \
+	-DUINTR=$(UINTR) \
+	-DDAEMON=$(DAEMON) \
+	-DDEBUG=$(DEBUG) \
+	-DSTAT=$(STAT) \
+	-DLOG_LEVEL=$(LOG) \
+	-DFXSAVE=$(FXSAVE)
 CMAKE_ARGS += -DCMAKE_INSTALL_PREFIX=install
 
 all: build
@@ -46,8 +56,11 @@ schbench: install
 rocksdb: install
 	cd build && make rocksdb_server VERBOSE=1
 
+microbench: install
+	cd build && make microbench VERBOSE=1
+
 fmt:
-	@clang-format --style=file -i $(shell find utils/ libos/ apps/ tests/ experiments/ -iname '*.c' -o -iname '*.cc' -o -iname '*.h')
+	@clang-format --style=file -i $(shell find utils/ libos/ apps/ synthetic/ -iname '*.c' -o -iname '*.cc' -o -iname '*.h')
 
 clean:
 	make -C build clean

diff --git a/README.md b/README.md
@@ -1,13 +1,23 @@
 # Skyloft
 
-Skyloft: A General High-Efficient Scheduling Framework in User Space
+## Contents
+
+- [Overview](#overview)
+- [Setup](#setup)
+- [Related Work](#related-work)
 
 ## Overview
 
 <div align="left">
 <img src=docs/imgs/overview.jpg width=60% />
 </div>
 
+### Why User Interrupt?
+
+<div align="left">
+<img src=docs/imgs/uintr.svg width=40% />
+</div>
+
 ### Layout
 
 - `apps/`: Benchmark real-world applications
@@ -25,5 +35,122 @@ Skyloft: A General High-Efficient Scheduling Framework in User Space
     - `sched/`: Schedulers
 - `utils/`: Useful tools
 - `scripts/`: Setup machine; run experiments
-- `microbench/`: Microbenchmarks and prototypes
-- `paper_results/`: Experimental results in the paper
+- `tests/`: Microbenchmarks and prototypes
+
+## Setup
+
+### Testbed
+
+* Intel(R) Xeon(R) Gold 5418Y @ 2.0GHZ
+* 2 sockets, 24 cores per socket
+* 128 GB memory
+* hyperthreading disabled, dynamic frequency scaling disabled
+
+Build kmod:
+
+```sh
+make
+CPU_LIST=<cpu list> make insmod
+```
+
+### Schbench
+
+24 cores
+
+```sh
+make schbench DPDK=0 UINTR=1 SCHED=<rr|cfs>
+./scripts/run.sh schbench -n10 -F128 -m1 -r10 -i5 -t<24|48|96>
+```
+
+### Synthetic
+
+20 workers, 1 dispatcher, 99.5% 4us, 5% 10000us
+
+1. LC
+
+```sh
+make build SCHED=sq UINTR=1 DPDK=0
+./scripts/run.sh shinjuku \
+    --get_service_time=4000 \
+    --range_query_service_time=10000000 \
+    --fake_work \
+    --load=0.4 \ # change the target throughput
+    --range_query_ratio=0.005 \
+    --preemption_quantum=30 \ # change the preemption quantum
+    --output_path=./data \
+    --run_time=5 \
+    --num_workers=20
+```
+
+2. LC + BE
+
+```sh
+make build SCHED=sq_lcbe UINTR=1 DPDK=0
+# 1
+./scripts/run.sh shinjuku \
+    --get_service_time=4000 \
+    --range_query_service_time=10000000 \
+    --fake_work \
+    --load=0.4 \
+    --range_query_ratio=0.005 \
+    --preemption_quantum=30 \
+    --output_path=./data \
+    --run_time=5 \
+    --guaranteed_cpus=4 \ # change the cores LC app gets at first
+    --num_workers=20
+# 2
+sudo ./build/bin/antagonist --num_workers=20 --run_time=5
+```
+
+
+### Mircobenchmarks
+
+#### Operation latency
+
+| Operations                  | Latency (cycles) | Latency (ns) |
+| --------------------------- | ---------------- | ------------ |
+| `rdtsc`                     | 34               | 17           |
+| `clock_gettime`             | 54               | 27           |
+| Null syscall (ioctl)        | 317              | 159          |
+| `senduipi` (not suppressed) | 354              | 177          |
+| `senduipi` (suppressed)     | 123              | 62           |
+| Timer handled by user       | 628              | 314          |
+| Send APIC IPI (ioctl)       | 633              | 317          |
+| Send-receive (senduipi)     | 1211             | 606          |
+| Send-receive (APIC IPI)     | 1431             | 715          |
+
+## Related Work
+
+| Related Works             | Cross-App Scheduling | Preemption Support and Overhead | Generality | Kernel Modification | App Modification |
+| ------------------------- | :------------------: | :-----------------------------: | :--------: | :-----------------: | :--------------: |
+| User-level threads        |          ✗           |                -                |    Poor    |         No          |        No        |
+| IX                        |          ✗           |                -                |    Poor    |        Major        |      Minor       |
+| ZygOS                     |          ✗           |                -                |    Poor    |        Major        |      Minor       |
+|                           |                      |                                 |            |                     |                  |
+| Arachne                   |          ✓           |                -                |    Poor    |         No          |      Minor       |
+| Shenango                  |          ✓           |                -                |    Poor    |         No          |      Minor       |
+| Caladan                   |          ✓           |                -                |    Poor    |        Minor        |      Minor       |
+|                           |                      |                                 |            |                     |                  |
+| Shinjuku                  |          ✗           |               Low               |    Poor    |        Major        |      Major       |
+| Concord                   |          ✗           |            Very low             |    Poor    |         No          |      Major       |
+| LibPreemptible            |          ✗           |            Very low             |    Poor    |         No          |      Major       |
+|                           |                      |                                 |            |                     |                  |
+| Linux scheduler framework |          ✓           |              High               |    Good    |         No          |        No        |
+| ghOSt                     |          ✓           |              High               |    Good    |        Major        |        No        |
+| Our work                  |          ✓           |            Very low             |    Good    |        Minor        |        No        |
+
+* Multi-App Scheduling: Whether multiple applications are supported. For example, running LC + BE applications at the same time can benefit CPU utilization.
+
+* Preemption Support and Overhead: Whether preemption is supported. This significantly affects the tail delay in long and short request scenarios.
+
+* Generality: Whether it is generic and can be adapted to various scheduling strategies. For example, Arachne/Shenango/Caladan requires a global dispatcher and does not support per-CPU scheduling or deadline-based scheduling.
+
+* Kernel Modification:
+  * **No**: there is no change
+  * **Minor**: Only one kernel module is required
+  * **Major**: The kernel source code needs to be modified.
+
+* App Modification:
+  * **No**: there is no change.
+  * **Minor**: Simple API replacement.
+  * **Major**: Provides an API different from the POSIX interface and requires a lot of modifications.
diff --git a/apps/CMakeLists.txt b/apps/CMakeLists.txt
@@ -21,19 +21,14 @@ target_link_libraries(test_timer skyloft utils)
 add_executable(test_rcu test_rcu.c)
 target_link_libraries(test_rcu skyloft utils)
 
-add_executable(test_net test_net.c)
-target_link_libraries(test_net skyloft utils)
-
 if(DPDK)
-    add_executable(fakework fakework.c)
-    target_link_libraries(fakework skyloft utils)
-
     include(${CMAKE_SCRIPTS}/rocksdb.mk)
     add_custom_target(
         rocksdb_server
         WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/rocksdb_server
         COMMAND make clean && make SKYLOFT_DIR=${CMAKE_CURRENT_BINARY_DIR}/../install ROCKSDB_SRC=${rocksdb_SOURCE_DIR}/
         COMMAND ln -sf ${CMAKE_CURRENT_SOURCE_DIR}/rocksdb_server/rocksdb_server ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/rocksdb_server
+        COMMAND ln -sf ${CMAKE_CURRENT_SOURCE_DIR}/rocksdb_server/create_db ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/create_db
     )
     add_dependencies(rocksdb_server librocksdb)
 endif()

diff --git a/apps/fakework.c b/apps/fakework.c
diff --git a/apps/rocksdb_server/.gitignore b/apps/rocksdb_server/.gitignore
@@ -0,0 +1,2 @@
+rocksdb_server
+create_db