New px-fuse forked from v2.6.0 with limited set of changes from v2.7.0

Makefile.in

@@ -1,4 +1,4 @@
-px-objs = pxd.o dev.o iov_iter.o px_version.o
+px-objs = pxd.o dev.o iov_iter.o px_version.o pxd_fastpath_stub.o
 obj-m = px.o
 
 KBUILD_CPPFLAGS := -D__KERNEL__
@@ -81,26 +81,26 @@ ifeq ($(shell echo "$(CHK_KVER)" | grep -Eq '.*\.el8.*\.x86_64'; echo $$?),0)
 PXDEFINES += -D__PX_BLKMQ__ -D__EL8__
 endif
 
-## fastpath specific checks
-ifeq ($(call verlater,${kmajor},${fp_major}),0)
-PXDEFINES += -D__PX_FASTPATH__
-px-objs += pxd_fastpath.o
-$(info Kernel fast path enabled, current kernel version $(CHK_KVER) need minimum $(FPATH_MINKVER))
-else
-ifeq ($(call versame,${kmajor},${fp_major}),0)
-ifeq ($(call versameorlater,${kminor},${fp_minor}),0)
-PXDEFINES += -D__PX_FASTPATH__
-px-objs += pxd_fastpath.o
-$(info Kernel fast path enabled, current kernel version $(CHK_KVER) need minimum $(FPATH_MINKVER))
-else
-px-objs += pxd_fastpath_stub.o
-$(info Kernel fast path disabled, current kernel version $(CHK_KVER) need minimum $(FPATH_MINKVER))
-endif
-else
-px-objs += pxd_fastpath_stub.o
-$(info Kernel fast path disabled, current kernel version $(CHK_KVER) need minimum $(FPATH_MINKVER))
-endif
-endif
+## fastpath specific checks -- disabled
+# ifeq ($(call verlater,${kmajor},${fp_major}),0)
+# PXDEFINES += -D__PX_FASTPATH__
+# px-objs += pxd_fastpath.o
+# $(info Kernel fast path enabled, current kernel version $(CHK_KVER) need minimum $(FPATH_MINKVER))
+# else
+# ifeq ($(call versame,${kmajor},${fp_major}),0)
+# ifeq ($(call versameorlater,${kminor},${fp_minor}),0)
+# PXDEFINES += -D__PX_FASTPATH__
+# px-objs += pxd_fastpath.o
+# $(info Kernel fast path enabled, current kernel version $(CHK_KVER) need minimum $(FPATH_MINKVER))
+# else
+# px-objs += pxd_fastpath_stub.o
+# $(warning Kernel fast path disabled, current kernel version $(CHK_KVER) need minimum $(FPATH_MINKVER))
+# endif
+# else
+# px-objs += pxd_fastpath_stub.o
+# $(warning Kernel fast path disabled, current kernel version $(CHK_KVER) need minimum $(FPATH_MINKVER))
+# endif
+# endif
 
 
 ifdef KERNELOTHER

dev.c

@@ -88,7 +88,8 @@ static struct fuse_req *__fuse_get_req(struct fuse_conn *fc)
 	struct fuse_req *req;
 	int err;
 
-	if (!fc->connected && !fc->allow_disconnected) {
+	// 'allow_disconnected' check subsumes 'connected' as well
+	if (!READ_ONCE(fc->allow_disconnected)) {
 		 err = -ENOTCONN;
 		goto out;
 	}
@@ -166,7 +167,13 @@ static void fuse_put_unique(struct fuse_conn *fc, u64 uid)
 {
 	struct fuse_per_cpu_ids *my_ids;
 	int num_free;
-	int cpu = get_cpu();
+	int cpu;
+
+	if (uid == 0) {
+		return;
+	}
+
+	cpu = get_cpu();
 
 	my_ids = per_cpu_ptr(fc->per_cpu_ids, cpu);
 
@@ -192,6 +199,12 @@ static void fuse_put_unique(struct fuse_conn *fc, u64 uid)
 
 static void queue_request(struct fuse_conn *fc, struct fuse_req *req)
 {
+	req->in.h.len = sizeof(struct fuse_in_header) +
+		len_args(req->in.numargs, (struct fuse_arg *)req->in.args);
+
+	req->in.h.unique = fuse_get_unique(fc);
+	fc->request_map[req->in.h.unique & (FUSE_MAX_REQUEST_IDS - 1)] = req;
+
 	list_add_tail(&req->list, &fc->pending);
 }
 
@@ -239,23 +252,15 @@ static void fuse_request_send_nowait_locked(struct fuse_conn *fc,
 
 void fuse_request_send_nowait(struct fuse_conn *fc, struct fuse_req *req)
 {
-	req->in.h.len = sizeof(struct fuse_in_header) +
-		len_args(req->in.numargs, (struct fuse_arg *)req->in.args);
-
-	req->in.h.unique = fuse_get_unique(fc);
-	fc->request_map[req->in.h.unique & (FUSE_MAX_REQUEST_IDS - 1)] = req;
-
 	/*
 	 * Ensures checking the value of allow_disconnected and adding request to
 	 * queue is done atomically.
 	 */
 	rcu_read_lock();
 
-	if (fc->connected || fc->allow_disconnected) {
-		spin_lock(&fc->lock);
+	// 'allow_disconnected' check subsumes 'connected' as well
+	if (READ_ONCE(fc->allow_disconnected)) {
 		fuse_request_send_nowait_locked(fc, req);
-		spin_unlock(&fc->lock);
-
 		rcu_read_unlock();
 
 		fuse_conn_wakeup(fc);
@@ -280,7 +285,7 @@ __acquires(fc->lock)
 	DECLARE_WAITQUEUE(wait, current);
 
 	add_wait_queue_exclusive(&fc->waitq, &wait);
-	while (fc->connected && !request_pending(fc)) {
+	while (READ_ONCE(fc->connected) && !request_pending(fc)) {
 		set_current_state(TASK_INTERRUPTIBLE);
 		if (signal_pending(current))
 			break;
@@ -421,11 +426,11 @@ static ssize_t fuse_dev_do_read(struct fuse_conn *fc, struct file *file,
 	spin_lock(&fc->lock);
 	if (!request_pending(fc)) {
 		err = -EAGAIN;
-		if ((file->f_flags & O_NONBLOCK) && fc->connected)
+		if ((file->f_flags & O_NONBLOCK) && READ_ONCE(fc->connected))
 			goto err_unlock;
 		request_wait(fc);
 		err = -ENODEV;
-		if (!fc->connected)
+		if (!READ_ONCE(fc->connected))
 			goto err_unlock;
 		err = -ERESTARTSYS;
 		if (!request_pending(fc))
@@ -992,7 +997,7 @@ static ssize_t fuse_dev_do_write(struct fuse_conn *fc, struct iov_iter *iter)
 	}
 
 	spin_lock(&fc->lock);
-	if (!fc->connected) {
+	if (!READ_ONCE(fc->connected)) {
 		spin_unlock(&fc->lock);
 		return err;
 	}
@@ -1053,7 +1058,7 @@ static unsigned fuse_dev_poll(struct file *file, poll_table *wait)
 	poll_wait(file, &fc->waitq, wait);
 
 	spin_lock(&fc->lock);
-	if (!fc->connected)
+	if (!READ_ONCE(fc->connected))
 		mask = POLLERR;
 	else if (request_pending(fc))
 		mask |= POLLIN | POLLRDNORM;
@@ -1185,8 +1190,8 @@ struct fuse_conn *fuse_conn_get(struct fuse_conn *fc)
 void fuse_abort_conn(struct fuse_conn *fc)
 {
 	spin_lock(&fc->lock);
-	if (fc->connected) {
-		fc->connected = 0;
+	if (READ_ONCE(fc->connected)) {
+		WRITE_ONCE(fc->connected, 0);
 		fuse_end_queued_requests(fc);
 		wake_up_all(&fc->waitq);
 		kill_fasync(&fc->fasync, SIGIO, POLL_IN);
@@ -1199,7 +1204,7 @@ int fuse_dev_release(struct inode *inode, struct file *file)
 	struct fuse_conn *fc = fuse_get_conn(file);
 	if (fc) {
 		spin_lock(&fc->lock);
-		fc->connected = 0;
+		WRITE_ONCE(fc->connected, 0);
 		fuse_end_queued_requests(fc);
 		spin_unlock(&fc->lock);
 		fuse_conn_put(fc);

fuse.h

@@ -670,6 +670,17 @@ struct fuse_in_header {
 	uint32_t	padding;
 };
 
+struct fuse_in_header_v1 {
+	uint32_t	len;
+	uint32_t	opcode;
+	uint64_t	unique;
+	uint64_t	nodeid;
+	uint32_t	uid;
+	uint32_t	gid;
+	uint32_t	pid;
+	uint32_t	padding;
+};
+
 struct fuse_out_header {
 	uint32_t	len;
 	int32_t		error;

fuse_i.h

@@ -9,6 +9,7 @@
 #ifndef _FS_FUSE_I_H
 #define _FS_FUSE_I_H
 
+#ifdef __KERNEL__
 #include <linux/fs.h>
 #include <linux/mount.h>
 #include <linux/wait.h>
@@ -112,7 +113,98 @@ struct ____cacheline_aligned fuse_per_cpu_ids {
 	/** followed by list of free ids */
 	u64 free_ids[FUSE_MAX_PER_CPU_IDS];
 };
+#endif
+
+#ifndef __KERNEL__
+#define ____cacheline_aligned alignas(64)
+#endif
+
+/** Maximum number of outstanding background requests */
+#define FUSE_DEFAULT_MAX_BACKGROUND (PXD_MAX_QDEPTH * PXD_MAX_DEVICES)
+
+/** size of request ring buffer */
+#define FUSE_REQUEST_QUEUE_SIZE (2 * FUSE_DEFAULT_MAX_BACKGROUND)
+
+#ifdef __KERNEL__
+/** writer control block */
+struct ____cacheline_aligned fuse_queue_writer {
+	uint32_t write;         /** cached write index */
+	uint32_t read;		/** cached read index */
+	spinlock_t lock;	/** writer lock */
+	uint32_t pad_0;
+	uint64_t sequence;        /** next request sequence number */
+	uint64_t pad[5];
+};
+
+/** reader control block */
+struct ____cacheline_aligned fuse_queue_reader {
+	uint32_t read;          /** read index updated by reader */
+	uint32_t write;		/** write index updated by writer */
+	uint32_t need_wake_up;	/** if true reader needs wake up call */
+	uint32_t pad;
+	uint64_t pad_2[6];
+};
+
+#else
+
+#include <pthread.h>
+#include <atomic>
+#include "spin_lock.h"
+
+/** writer control block */
+struct alignas(64) fuse_queue_writer {
+	uint32_t write;         	/** cached write index */
+	uint32_t read;			/** cached read index */
+	pthread_spinlock_t lock;	/** writer lock */
+	bool in_runq;			/** a thread is processing the queue */
+	char pad_1[3];
+	uint64_t sequence;        	/** next request sequence number */
+	uint64_t pad[5];
+};
+
+/** reader control block */
+struct alignas(64) fuse_queue_reader {
+	std::atomic<uint32_t> read;	/** read index updated by reader */
+	std::atomic<uint32_t> write;	/** write index updated by writer */
+	px::spinlock lock;
+	uint64_t pad_2[6];
+};
 
+#endif
+
+/** opcodes for fuse_user_request */
+#define FUSE_USER_OP_NOP 0		/** nop */
+#define FUSE_USER_OP_REQ_DONE 1		/** request completion */
+
+/** request from user space to kernel */
+struct fuse_user_request {
+	uint8_t opcode;		/** operation code */
+	uint16_t len;		/** number of entries in iovec array */
+	uint8_t pad;		/** padding */
+	int32_t res;		/** result code */
+	uint64_t unique;	/** unique id of request */
+	uint64_t user_data;	/** user data returned in response */
+	uint64_t iov_addr;	/** address of iovec array */
+};
+
+/** queue control block */
+struct fuse_queue_cb {
+	struct fuse_queue_writer w;
+	struct fuse_queue_reader r;
+};
+
+/** fuse connection queues */
+struct ____cacheline_aligned fuse_conn_queues {
+	/** requests from kernel to user space */
+	struct fuse_queue_cb requests_cb;
+	struct rdwr_in requests[FUSE_REQUEST_QUEUE_SIZE];
+
+	/** requests from user space to kernel */
+	struct fuse_queue_cb user_requests_cb;
+	struct fuse_user_request user_requests[FUSE_REQUEST_QUEUE_SIZE];
+};
+
+#ifdef __KERNEL__
 /**
  * A Fuse connection.
  *
@@ -240,4 +332,5 @@ int pxd_set_fastpath(struct fuse_conn *fc, struct pxd_fastpath_out*);
 
 void fuse_request_init(struct fuse_req *req);
 void fuse_req_init_context(struct fuse_req *req);
+#endif
 #endif /* _FS_FUSE_I_H */