extmod/vfs_reader: Add file ioctl to set read buffer size.

Can be used to speed up importing a file from a vfs based filesystem. Signed-off-by: Andrew Leech <[email protected]>
andrewleech · Nov 1, 2023 · eef2ace · eef2ace
1 parent a614c1d
commit eef2ace
Show file tree

Hide file tree

Showing 3 changed files with 40 additions and 7 deletions.
diff --git a/extmod/vfs_reader.c b/extmod/vfs_reader.c
@@ -34,21 +34,39 @@
 
 #if MICROPY_READER_VFS
 
+
 typedef struct _mp_reader_vfs_t {
     mp_obj_t file;
     uint16_t len;
     uint16_t pos;
-    byte buf[24];
+    uint8_t bufsize;
+    byte buf[];
 } mp_reader_vfs_t;
 
 STATIC mp_uint_t mp_reader_vfs_readbyte(void *data) {
     mp_reader_vfs_t *reader = (mp_reader_vfs_t *)data;
+    if (!reader->bufsize) {
+        // If buffering is disabled read byte from stream directly
+        byte c;
+        int errcode = 0;
+        mp_uint_t len = mp_stream_rw(reader->file, &c, 1,
+            &errcode, MP_STREAM_RW_READ | MP_STREAM_RW_ONCE);
+        if (errcode != 0) {
+            // TODO handle errors properly
+            return MP_READER_EOF;
+        }
+        if (len == 0) {
+            return MP_READER_EOF;
+        }
+        return c;
+    }
+
     if (reader->pos >= reader->len) {
-        if (reader->len < sizeof(reader->buf)) {
+        if (reader->len < reader->bufsize) {
             return MP_READER_EOF;
         } else {
             int errcode;
-            reader->len = mp_stream_rw(reader->file, reader->buf, sizeof(reader->buf),
+            reader->len = mp_stream_rw(reader->file, reader->buf, reader->bufsize,
                 &errcode, MP_STREAM_RW_READ | MP_STREAM_RW_ONCE);
             if (errcode != 0) {
                 // TODO handle errors properly
@@ -70,14 +88,24 @@ STATIC void mp_reader_vfs_close(void *data) {
 }
 
 void mp_reader_new_file(mp_reader_t *reader, qstr filename) {
-    mp_reader_vfs_t *rf = m_new_obj(mp_reader_vfs_t);
     mp_obj_t args[2] = {
         MP_OBJ_NEW_QSTR(filename),
         MP_OBJ_NEW_QSTR(MP_QSTR_rb),
     };
-    rf->file = mp_vfs_open(MP_ARRAY_SIZE(args), &args[0], (mp_map_t *)&mp_const_empty_map);
-    int errcode;
-    rf->len = mp_stream_rw(rf->file, rf->buf, sizeof(rf->buf), &errcode, MP_STREAM_RW_READ | MP_STREAM_RW_ONCE);
+    mp_obj_t file = mp_vfs_open(MP_ARRAY_SIZE(args), &args[0], (mp_map_t *)&mp_const_empty_map);
+
+    const mp_stream_p_t *stream_p = mp_get_stream(file);
+    int errcode = 0;
+    mp_uint_t bufsize = stream_p->ioctl(file, MP_STREAM_GET_BUFSIZE, 0, &errcode);
+    if (errcode || bufsize > 255) {
+        errcode = 0;
+        bufsize = MICROPY_READER_VFS_DEFAULT_BUF_SIZE;
+    }
+
+    mp_reader_vfs_t *rf = m_new_obj_var(mp_reader_vfs_t, uint8_t, bufsize);
+    rf->file = file;
+    rf->bufsize = bufsize;
+    rf->len = mp_stream_rw(rf->file, rf->buf, rf->bufsize, &errcode, MP_STREAM_RW_READ | MP_STREAM_RW_ONCE);
     if (errcode != 0) {
         mp_raise_OSError(errcode);
     }

diff --git a/py/mpconfig.h b/py/mpconfig.h
@@ -597,6 +597,10 @@
 #define MICROPY_READER_VFS (0)
 #endif
 
+#ifndef MICROPY_READER_VFS_DEFAULT_BUF_SIZE
+#define MICROPY_READER_VFS_DEFAULT_BUF_SIZE (48)
+#endif
+
 // Whether any readers have been defined
 #ifndef MICROPY_HAS_FILE_READER
 #define MICROPY_HAS_FILE_READER (MICROPY_READER_POSIX || MICROPY_READER_VFS)

diff --git a/py/stream.h b/py/stream.h
@@ -43,6 +43,7 @@
 #define MP_STREAM_GET_DATA_OPTS (8)  // Get data/message options
 #define MP_STREAM_SET_DATA_OPTS (9)  // Set data/message options
 #define MP_STREAM_GET_FILENO    (10) // Get fileno of underlying file
+#define MP_STREAM_GET_BUFSIZE   (11) // Get preferred buffer size for file
 
 // These poll ioctl values are compatible with Linux
 #define MP_STREAM_POLL_RD       (0x0001)