Update eevdf

CMakeLists.txt

@@ -10,7 +10,7 @@ set(CMAKE_C_STANDARD 23)
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -O3 -Wall -Wextra -Wno-unused-parameter")
 if (DPDK)
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
-elseif (SCHED_POLICY MATCHES "^(fifo|rr|cfs)$")
+elseif (SCHED_POLICY MATCHES "^(fifo|rr|cfs|eevdf)$")
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mno-sse")
 endif()
 
@@ -79,6 +79,8 @@ elseif(SCHED_POLICY STREQUAL "rr")
     add_definitions(-DSKYLOFT_SCHED_FIFO2)
 elseif(SCHED_POLICY STREQUAL "cfs")
     add_definitions(-DSKYLOFT_SCHED_CFS)
+elseif(SCHED_POLICY STREQUAL "eevdf")
+    add_definitions(-DSKYLOFT_SCHED_EEVDF)
 elseif(SCHED_POLICY STREQUAL "sq")
     add_definitions(-DSKYLOFT_SCHED_SQ)
 elseif(SCHED_POLICY STREQUAL "sq_lcbe")

include/skyloft/sched/ops.h

@@ -16,6 +16,10 @@
 #include "policy/cfs.h"
 #define SCHED_NAME        "cfs"
 #define SCHED_OP(op_name) cfs_##op_name
+#elif defined(SKYLOFT_SCHED_EEVDF)
+#include "policy/eevdf.h"
+#define SCHED_NAME        "eevdf"
+#define SCHED_OP(op_name) eevdf_##op_name
 #elif defined(SKYLOFT_SCHED_SQ)
 #include "policy/sq.h"
 #define SCHED_NAME        "sq"

include/skyloft/sched/policy/cfs.h

@@ -19,6 +19,7 @@ struct load_weight {
 /* CFS percpu state */
 struct cfs_rq {
     spinlock_t lock;
+    uint8_t pad0[60];
     /* sum of load of all tasks */
     struct load_weight load;
     uint32_t nr_running;
@@ -82,6 +83,7 @@ struct task *cfs_sched_pick_next();
 int cfs_sched_spawn(struct task *, int);
 void cfs_sched_yield();
 void cfs_sched_wakeup(struct task *);
+void cfs_sched_block();
 bool cfs_sched_preempt();
 int cfs_sched_init_task(struct task *);
 void cfs_sched_finish_task(struct task *);

include/skyloft/sched/policy/eevdf.h

@@ -0,0 +1,108 @@
+#pragma once
+
+#include <skyloft/percpu.h>
+#include <skyloft/task.h>
+
+#include <utils/defs.h>
+#include <utils/rbtree.h>
+#include <utils/spinlock.h>
+#include <utils/time.h>
+
+#include "dummy.h"
+#include "utils/log.h"
+
+/* EEVDF load weight: weight * inv_weight = 2^32 */
+struct load_weight {
+    uint64_t weight;
+    uint32_t inv_weight;
+};
+
+/* EEVDF percpu state */
+struct eevdf_rq {
+    spinlock_t lock;
+    uint8_t pad0[60];
+    /* sum of load of all tasks */
+    struct load_weight load;
+    /* rbtree root with leftmost node cached */
+    struct rb_root_cached tasks_timeline;
+    uint32_t nr_running;
+    int64_t avg_vruntime;
+    uint64_t avg_load;
+    uint64_t min_vruntime;
+    struct eevdf_task *curr;
+} __aligned_cacheline;
+
+struct eevdf_task {
+    int last_run;
+    struct load_weight load;
+    /* rbtree link */
+    struct rb_node run_node;
+    uint64_t deadline;
+    uint64_t min_vruntime;
+    /* task states */
+    bool on_rq;
+    /* task scheduled time */
+    __nsec exec_start;
+    __nsec sum_exec_runtime;
+    __nsec prev_sum_exec_runtime;
+    uint64_t vruntime;
+    int64_t vlag;
+    uint64_t slice;
+} __aligned_cacheline;
+
+#define WEIGHT_IDLEPRIO 3
+#define WMULT_IDLEPRIO  1431655765
+#define WMULT_CONST     (~0U)
+#define WMULT_SHIFT     32
+#define NICE_0_SHIFT    10
+#define NICE_0_LOAD     (1L << NICE_0_SHIFT)
+
+#define SCHED_FIXEDPOINT_SHIFT 10
+#define NICE_0_LOAD_SHIFT      (SCHED_FIXEDPOINT_SHIFT + SCHED_FIXEDPOINT_SHIFT)
+
+DECLARE_PERCPU(struct eevdf_rq *, percpu_rqs);
+
+#define this_rq()           percpu_get(percpu_rqs)
+#define cpu_rq(cpu)         percpu_get_remote(percpu_rqs, cpu)
+#define eevdf_task_of(task) ((struct eevdf_task *)task->policy_task_data)
+#define task_of(task)       (container_of((void *)task, struct task, policy_task_data))
+
+#define eevdf_sched_init       dummy_sched_init
+#define eevdf_sched_set_params dummy_sched_set_params
+#define eevdf_sched_poll       dummy_sched_poll
+#define eevdf_sched_dump_tasks dummy_sched_dump_tasks
+#define eevdf_sched_balance    dummy_sched_balance
+
+static inline void eevdf_sched_percpu_lock(int cpu)
+{
+    // log_debug("%s: %d", __func__, cpu);
+    spin_lock(&cpu_rq(cpu)->lock);
+}
+
+static inline void eevdf_sched_percpu_unlock(int cpu)
+{
+    spin_unlock(&cpu_rq(cpu)->lock);
+    // log_debug("%s: %d", __func__, cpu);
+}
+
+static inline int eevdf_sched_init_percpu(void *percpu_data)
+{
+    struct eevdf_rq *eevdf_rq = percpu_data;
+    percpu_get(percpu_rqs) = eevdf_rq;
+    spin_lock_init(&eevdf_rq->lock);
+    eevdf_rq->curr = NULL;
+    eevdf_rq->tasks_timeline = RB_ROOT_CACHED;
+    eevdf_rq->min_vruntime = (uint64_t)(-(1LL << 20));
+    eevdf_rq->nr_running = 0;
+    eevdf_rq->load.weight = eevdf_rq->load.inv_weight = 0;
+    return 0;
+}
+
+struct task *eevdf_sched_pick_next();
+int eevdf_sched_spawn(struct task *, int);
+void eevdf_sched_yield();
+void eevdf_sched_wakeup(struct task *);
+void eevdf_sched_block();
+bool eevdf_sched_preempt();
+int eevdf_sched_init_task(struct task *);
+void eevdf_sched_finish_task(struct task *);

include/skyloft/stat.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include <skyloft/sched.h>
+#include <utils/assert.h>
 
 /*
  * These are per-kthread stat counters. It's recommended that most counters be
@@ -17,8 +17,6 @@ enum {
     STAT_TASKS_STOLEN,
     STAT_IDLE,
     STAT_IDLE_CYCLES,
-    // STAT_WAKEUP,
-    // STAT_WAKEUP_CYCLES,
     STAT_SOFTIRQS_LOCAL,
     STAT_SOFTIRQ_CYCLES,
     STAT_ALLOC,
@@ -27,30 +25,24 @@ enum {
     STAT_TX,
 #ifdef SKYLOFT_UINTR
     STAT_UINTR,
-    // STAT_UINTR_CYCLES,
+#ifdef UTIMER
+    STAT_UTIMER_SENDS,
+    STAT_UTIMER_CYCLES,
+#endif
 #endif
 
     /* total number of counters */
     STAT_NR,
 };
 
 static const char *STAT_STR[] = {
-    "local_spawns",
-    "switch_to",
-    "tasks_stolen",
-    "idle",
-    "idle_cycles",
-    // "wakeup",
-    // "wakeup_cycles",
-    "softirqs_local",
-    "softirq_cycles",
-    "alloc",
-    "alloc_cycles",
-    "rx",
-    "tx",
+    "local_spawns",   "switch_to",     "tasks_stolen", "idle", "idle_cycles", "softirqs_local",
+    "softirq_cycles", "alloc",         "alloc_cycles", "rx",   "tx",
 #ifdef SKYLOFT_UINTR
     "uintr",
-    // "uintr_cycles",
+#ifdef UTIMER
+    "utimer_sends",   "utimer_cycles",
+#endif
 #endif
 };
 
@@ -70,13 +62,9 @@ static inline const char *stat_str(int idx)
  */
 #define ADD_STAT_FORCE(counter, val) (thisk()->stats[STAT_##counter] += val)
 #ifdef SKYLOFT_STAT
-#define ADD_STAT(counter, val) ADD_STAT_FORCE(counter, val)
-#define STAT_CYCLES_BEGIN(timer) ({ \
-    timer = now_tsc(); \
-})
-#define ADD_STAT_CYCLES(counter, timer) ({ \
-    ADD_STAT_FORCE(counter, now_tsc() - timer); \
-})
+#define ADD_STAT(counter, val)          ADD_STAT_FORCE(counter, val)
+#define STAT_CYCLES_BEGIN(timer)        ({ timer = now_tsc(); })
+#define ADD_STAT_CYCLES(counter, timer) ({ ADD_STAT_FORCE(counter, now_tsc() - timer); })
 #else
 #define ADD_STAT(counter, val)
 #define STAT_CYCLES_BEGIN(timer) ((void)timer)