extmod/vfs_reader: Add file ioctl to set read buffer size.

Can be used to speed up importing a file from a vfs based filesystem.

Signed-off-by: Andrew Leech <[email protected]>

extmod/vfs_reader.c

@@ -26,6 +26,7 @@
 
 #include <stdio.h>
 #include <string.h>
+#include <stdlib.h>
 
 #include "py/runtime.h"
 #include "py/stream.h"
@@ -34,33 +35,39 @@
 
 #if MICROPY_READER_VFS
 
+#ifndef MICROPY_READER_VFS_DEFAULT_BUFFER_SIZE
+#define MICROPY_READER_VFS_DEFAULT_BUFFER_SIZE (2 * MICROPY_BYTES_PER_GC_BLOCK - offsetof(mp_reader_vfs_t, buf))
+#endif
+#define MICROPY_READER_VFS_MIN_BUFFER_SIZE (MICROPY_BYTES_PER_GC_BLOCK - offsetof(mp_reader_vfs_t, buf))
+#define MICROPY_READER_VFS_MAX_BUFFER_SIZE (255)
+
 typedef struct _mp_reader_vfs_t {
     mp_obj_t file;
-    uint16_t len;
-    uint16_t pos;
-    byte buf[24];
+    uint8_t bufpos;
+    uint8_t buflen;
+    uint8_t bufsize;
+    byte buf[];
 } mp_reader_vfs_t;
 
 STATIC mp_uint_t mp_reader_vfs_readbyte(void *data) {
     mp_reader_vfs_t *reader = (mp_reader_vfs_t *)data;
-    if (reader->pos >= reader->len) {
-        if (reader->len < sizeof(reader->buf)) {
+    if (reader->bufpos >= reader->buflen) {
+        if (reader->buflen < reader->bufsize) {
             return MP_READER_EOF;
         } else {
             int errcode;
-            reader->len = mp_stream_rw(reader->file, reader->buf, sizeof(reader->buf),
-                &errcode, MP_STREAM_RW_READ | MP_STREAM_RW_ONCE);
+            reader->buflen = mp_stream_rw(reader->file, reader->buf, reader->bufsize, &errcode, MP_STREAM_RW_READ | MP_STREAM_RW_ONCE);
             if (errcode != 0) {
                 // TODO handle errors properly
                 return MP_READER_EOF;
             }
-            if (reader->len == 0) {
+            if (reader->buflen == 0) {
                 return MP_READER_EOF;
             }
-            reader->pos = 0;
+            reader->bufpos = 0;
         }
     }
-    return reader->buf[reader->pos++];
+    return reader->buf[reader->bufpos++];
 }
 
 STATIC void mp_reader_vfs_close(void *data) {
@@ -70,18 +77,31 @@ STATIC void mp_reader_vfs_close(void *data) {
 }
 
 void mp_reader_new_file(mp_reader_t *reader, qstr filename) {
-    mp_reader_vfs_t *rf = m_new_obj(mp_reader_vfs_t);
     mp_obj_t args[2] = {
         MP_OBJ_NEW_QSTR(filename),
         MP_OBJ_NEW_QSTR(MP_QSTR_rb),
     };
-    rf->file = mp_vfs_open(MP_ARRAY_SIZE(args), &args[0], (mp_map_t *)&mp_const_empty_map);
-    int errcode;
-    rf->len = mp_stream_rw(rf->file, rf->buf, sizeof(rf->buf), &errcode, MP_STREAM_RW_READ | MP_STREAM_RW_ONCE);
+    mp_obj_t file = mp_vfs_open(MP_ARRAY_SIZE(args), &args[0], (mp_map_t *)&mp_const_empty_map);
+
+    const mp_stream_p_t *stream_p = mp_get_stream(file);
+    int errcode = 0;
+    mp_uint_t bufsize = stream_p->ioctl(file, MP_STREAM_GET_BUFFER_SIZE, 0, &errcode);
+    if (bufsize == MP_STREAM_ERROR || bufsize == 0) {
+        // bufsize == 0 is included here to support mpremote v1.21 and older where mount file ioctl
+        // returned 0 by default.
+        bufsize = MICROPY_READER_VFS_DEFAULT_BUFFER_SIZE;
+    } else {
+        bufsize = MIN(MICROPY_READER_VFS_MAX_BUFFER_SIZE, MAX(MICROPY_READER_VFS_MIN_BUFFER_SIZE, bufsize));
+    }
+
+    mp_reader_vfs_t *rf = m_new_obj_var(mp_reader_vfs_t, buf, byte, bufsize);
+    rf->file = file;
+    rf->bufsize = bufsize;
+    rf->buflen = mp_stream_rw(rf->file, rf->buf, rf->bufsize, &errcode, MP_STREAM_RW_READ | MP_STREAM_RW_ONCE);
     if (errcode != 0) {
         mp_raise_OSError(errcode);
     }
-    rf->pos = 0;
+    rf->bufpos = 0;
     reader->data = rf;
     reader->readbyte = mp_reader_vfs_readbyte;
     reader->close = mp_reader_vfs_close;

py/stream.h

@@ -43,6 +43,7 @@
 #define MP_STREAM_GET_DATA_OPTS (8)  // Get data/message options
 #define MP_STREAM_SET_DATA_OPTS (9)  // Set data/message options
 #define MP_STREAM_GET_FILENO    (10) // Get fileno of underlying file
+#define MP_STREAM_GET_BUFFER_SIZE (11) // Get preferred buffer size for file
 
 // These poll ioctl values are compatible with Linux
 #define MP_STREAM_POLL_RD       (0x0001)

tests/extmod/vfs_userfs.py

@@ -16,6 +16,8 @@
 
 
 class UserFile(io.IOBase):
+    buffer_size = 16
+
     def __init__(self, mode, data):
         assert isinstance(data, bytes)
         self.is_text = mode.find("b") == -1
@@ -39,7 +41,11 @@ def readinto(self, buf):
 
     def ioctl(self, req, arg):
         print("ioctl", req, arg)
-        return 0
+        if req == 4:  # MP_STREAM_CLOSE
+            return 0
+        if req == 11:  # MP_STREAM_GET_BUFFER_SIZE
+            return UserFile.buffer_size
+        return -1
 
 
 class UserFS:
@@ -70,6 +76,8 @@ def open(self, path, mode):
     "/usermod2.py": b"print('in usermod2')",
     "/usermod3.py": b"syntax error",
     "/usermod4.mpy": b"syntax error",
+    "/usermod5.py": b"print('in usermod5')",
+    "/usermod6.py": b"print('in usermod6')",
 }
 os.mount(UserFS(user_files), "/userfs")
 
@@ -93,6 +101,14 @@ def open(self, path, mode):
 except ValueError:
     print("ValueError in usermod4")
 
+# Test an import with largest buffer size
+UserFile.buffer_size = 255
+import usermod5
+
+# Test an import with over-size buffer size (should be safely limited internally)
+UserFile.buffer_size = 1024
+import usermod6
+
 # unmount and undo path addition
 os.umount("/userfs")
 sys.path.pop()

tests/extmod/vfs_userfs.py.exp

@@ -3,21 +3,37 @@ some data in a text file
 stat /usermod1
 stat /usermod1.py
 open /usermod1.py rb
+ioctl 11 0
 ioctl 4 0
 in usermod1
 stat /usermod2
 stat /usermod2.py
 open /usermod2.py rb
+ioctl 11 0
 ioctl 4 0
 in usermod2
 stat /usermod3
 stat /usermod3.py
 open /usermod3.py rb
+ioctl 11 0
 ioctl 4 0
 SyntaxError in usermod3
 stat /usermod4
 stat /usermod4.py
 stat /usermod4.mpy
 open /usermod4.mpy rb
+ioctl 11 0
 ioctl 4 0
 ValueError in usermod4
+stat /usermod5
+stat /usermod5.py
+open /usermod5.py rb
+ioctl 11 0
+ioctl 4 0
+in usermod5
+stat /usermod6
+stat /usermod6.py
+open /usermod6.py rb
+ioctl 11 0
+ioctl 4 0
+in usermod6