Merge tag 'v1.9.1'

Fixes for stmhal USB mass storage, lwIP bindings and VFS regressions

This release provides an important fix for the USB mass storage device in
the stmhal port by implementing the SCSI SYNCHRONIZE_CACHE command, which
is now require by some Operating Systems.  There are also fixes for the
lwIP bindings to improve non-blocking sockets and error codes.  The VFS has
some regressions fixed including the ability to statvfs the root.

All changes are listed below.

py core:
- modbuiltins: add core-provided version of input() function
- objstr: catch case of negative "maxsplit" arg to str.rsplit()
- persistentcode: allow to compile with complex numbers disabled
- objstr: allow to compile with obj-repr D, and unicode disabled
- modsys: allow to compile with obj-repr D and PY_ATTRTUPLE disabled
- provide mp_decode_uint_skip() to help reduce stack usage
- makeqstrdefs.py: make script run correctly with Python 2.6
- objstringio: if created from immutable object, follow copy on write policy

extmod:
- modlwip: connect: for non-blocking mode, return EINPROGRESS
- modlwip: fix error codes for duplicate calls to connect()
- modlwip: accept: fix error code for non-blocking mode
- vfs: allow to statvfs the root directory
- vfs: allow "buffering" and "encoding" args to VFS's open()
- modframebuf: fix signed/unsigned comparison pendantic warning

lib:
- libm: use isfinite instead of finitef, for C99 compatibility
- utils/interrupt_char: remove support for KBD_EXCEPTION disabled

tests:
- basics/string_rsplit: add tests for negative "maxsplit" argument
- float: convert "sys.exit()" to "raise SystemExit"
- float/builtin_float_minmax: PEP8 fixes
- basics: convert "sys.exit()" to "raise SystemExit"
- convert remaining "sys.exit()" to "raise SystemExit"

unix port:
- convert to use core-provided version of built-in import()
- Makefile: replace references to make with $(MAKE)

windows port:
- convert to use core-provided version of built-in import()

qemu-arm port:
- Makefile: adjust object-file lists to get correct dependencies
- enable micropython.mem_*() functions to allow more tests

stmhal port:
- boards: enable DAC for NUCLEO_F767ZI board
- add support for NUCLEO_F446RE board
- pass USB handler as parameter to allow more than one USB handler
- usb: use local USB handler variable in Start-of-Frame handler
- usb: make state for USB device private to top-level USB driver
- usbdev: for MSC implement SCSI SYNCHRONIZE_CACHE command
- convert from using stmhal's input() to core provided version

cc3200 port:
- convert from using stmhal's input() to core provided version

teensy port:
- convert from using stmhal's input() to core provided version

esp8266 port:
- Makefile: replace references to make with $(MAKE)
- Makefile: add clean-modules target
- convert from using stmhal's input() to core provided version

zephyr port:
- modusocket: getaddrinfo: Fix mp_obj_len() usage
- define MICROPY_PY_SYS_PLATFORM (to "zephyr")
- machine_pin: use native Zephyr types for Zephyr API calls

docs:
- machine.Pin: remove out_value() method
- machine.Pin: add on() and off() methods
- esp8266: consistently replace Pin.high/low methods with .on/off
- esp8266/quickref: polish Pin.on()/off() examples
- network: move confusingly-named cc3200 Server class to its reference
- uos: deconditionalize, remove minor port-specific details
- uos: move cc3200 port legacy VFS mounting functions to its ref doc
- machine: sort machine classes in logical order, not alphabetically
- network: first step to describe standard network class interface

examples:
- embedding: use core-provided KeyboardInterrupt object

.gitattributes

@@ -10,11 +10,11 @@
 *.png binary
 *.jpg binary
 *.dxf binary
+*.mpy binary
 
 # These should also not be modified by git.
 tests/basics/string_cr_conversion.py -text
 tests/basics/string_crlf_conversion.py -text
-stmhal/startup_stm32f40xx.s -text
 stmhal/pybcdc.inf_template -text
 stmhal/usbd_* -text
 stmhal/boards/*/stm32f4xx_hal_conf.h -text
@@ -28,4 +28,3 @@ cc3200/hal/des.c -text
 cc3200/hal/i2s.c -text
 cc3200/hal/i2s.h -text
 cc3200/version.h -text
-lib/fatfs/** -text

.gitignore

@@ -10,7 +10,7 @@
 *.dis
 *.exe
 
-# Packages 
+# Packages
 ############
 
 # Logs and Databases
@@ -44,3 +44,7 @@ user.props
 # Sphinx output
 ###############
 _build
+
+# Generated rst files
+######################
+genrst/

.travis.yml

@@ -60,7 +60,7 @@ script:
   - (cd tests && MICROPY_CPYTHON3=python3.4 MICROPY_MICROPYTHON=../unix/micropython_coverage ./run-tests --emit native)
   - (cd tests && MICROPY_CPYTHON3=python3.4 MICROPY_MICROPYTHON=../unix/micropython_coverage ./run-tests --via-mpy -d basics float)
 
-after_success:
+  # run coveralls coverage analysis (try to, even if some builds/tests failed)
   - (cd unix && coveralls --root .. --build-root . --gcov $(which gcov) --gcov-options '\-o build-coverage/' --include py --include extmod)
 
 after_failure:

atmel-samd/Makefile

@@ -125,7 +125,7 @@ CFLAGS_CORTEX_M0 = \
 	-DCIRCUITPY_CANARY_WORD=0xADAF00 \
 	-DCIRCUITPY_SAFE_RESTART_WORD=0xDEADBEEF \
 	--param max-inline-insns-single=500
-CFLAGS = $(INC) -Wall -Werror -std=gnu11 -nostdlib $(CFLAGS_CORTEX_M0) $(COPT)
+CFLAGS = $(INC) -Wall -Werror -std=gnu11 -nostdlib $(CFLAGS_CORTEX_M0) $(CFLAGS_MOD) $(COPT)
 
 #Debugging/Optimization
 # TODO(tannewt): Figure out what NDEBUG does. Adding it to the debug build
@@ -184,7 +184,6 @@ SRC_C = \
 	access_vfs.c \
 	autoreload.c \
 	background.c \
-	builtin_open.c \
 	fatfs_port.c \
 	flash_api.c \
 	main.c \
@@ -213,17 +212,15 @@ SRC_C = \
 	boards/$(BOARD)/board.c \
 	boards/$(BOARD)/pins.c \
 	freetouch/adafruit_ptc.c \
-	lib/fatfs/ff.c \
-	lib/fatfs/option/ccsbcs.c \
+	lib/oofatfs/ff.c \
+	lib/oofatfs/option/ccsbcs.c \
 	lib/timeutils/timeutils.c \
 	lib/utils/buffer_helper.c \
 	lib/utils/context_manager_helpers.c \
 	lib/utils/interrupt_char.c \
 	lib/utils/pyexec.c \
-	lib/utils/pyhelp.c \
 	lib/utils/stdout_helpers.c \
 	lib/libc/string0.c \
-	lib/mp-readline/builtin_input.c \
 	lib/mp-readline/readline.c
 
 STM_SRC_C = $(addprefix stmhal/,\
@@ -261,14 +258,14 @@ SRC_COMMON_HAL = \
 SRC_BINDINGS_ENUMS = \
 	digitalio/Direction.c \
 	digitalio/DriveMode.c \
-	digitalio/Pull.c
+	digitalio/Pull.c \
+	help.c
 
 SRC_COMMON_HAL_EXPANDED = $(addprefix shared-bindings/, $(SRC_COMMON_HAL)) \
                           $(addprefix shared-bindings/, $(SRC_BINDINGS_ENUMS)) \
                           $(addprefix common-hal/, $(SRC_COMMON_HAL))
 
 SRC_SHARED_MODULE = \
-	help.c \
 	bitbangio/__init__.c \
 	bitbangio/I2C.c \
 	bitbangio/OneWire.c \

atmel-samd/access_vfs.c

@@ -30,8 +30,11 @@
 #include "autoreload.h"
 
 #include "asf/common/services/usb/class/msc/device/udi_msc.h"
-#include "extmod/fsusermount.h"
-#include "lib/fatfs/diskio.h"
+#include "extmod/vfs.h"
+#include "extmod/vfs_fat.h"
+#include "lib/oofatfs/ff.h"
+#include "lib/oofatfs/diskio.h"
+#include "lib/oofatfs/ffconf.h"
 #include "py/mpconfig.h"
 #include "py/mphal.h"
 #include "py/mpstate.h"
@@ -47,15 +50,14 @@
 //!   An error occurred          ->    CTRL_FAIL
 Ctrl_status vfs_test_unit_ready(void)
 {
-    if (VFS_INDEX >= MP_ARRAY_SIZE(MP_STATE_PORT(fs_user_mount))) {
-        return CTRL_FAIL;
-    }
-    DSTATUS status = disk_status(VFS_INDEX);
-    if (status == STA_NOINIT) {
-        return CTRL_NO_PRESENT;
+    mp_vfs_mount_t* current_mount = MP_STATE_VM(vfs_mount_table);
+    for (uint8_t i = 0; current_mount != NULL; i++) {
+        if (i == VFS_INDEX) {
+            return CTRL_GOOD;
+        }
+        current_mount = current_mount->next;
     }
-
-    return CTRL_GOOD;
+    return CTRL_NO_PRESENT;
 }
 
 //! This function returns the address of the last valid sector
@@ -81,13 +83,17 @@ Ctrl_status vfs_read_capacity(uint32_t *last_valid_sector)
 //!
 bool vfs_wr_protect(void)
 {
-    if (VFS_INDEX >= MP_ARRAY_SIZE(MP_STATE_PORT(fs_user_mount))) {
-        return true;
+    mp_vfs_mount_t* current_mount = MP_STATE_VM(vfs_mount_table);
+    for (uint8_t i = 0; current_mount != NULL; i++) {
+        if (i == VFS_INDEX) {
+            break;
+        }
+        current_mount = current_mount->next;
     }
-    fs_user_mount_t *vfs = MP_STATE_PORT(fs_user_mount)[VFS_INDEX];
-    if (vfs == NULL) {
+    if (current_mount == NULL) {
         return true;
     }
+    fs_user_mount_t *vfs = (fs_user_mount_t *) current_mount->obj;
 
     // This is used to determine the writeability of the disk from USB.
     if (vfs->writeblocks[0] == MP_OBJ_NULL ||
@@ -170,9 +176,18 @@ Ctrl_status vfs_usb_write_10(uint32_t addr, volatile uint16_t nb_sector)
         }
         // Since by getting here we assume the mount is read-only to MicroPython
         // lets update the cached FatFs sector if its the one we just wrote.
-        fs_user_mount_t *vfs =  MP_STATE_PORT(fs_user_mount)[VFS_INDEX];
-        volatile uint16_t x = addr;
-        (void) x;
+        mp_vfs_mount_t* current_mount = MP_STATE_VM(vfs_mount_table);
+        for (uint8_t i = 0; current_mount != NULL; i++) {
+            if (i == VFS_INDEX) {
+                break;
+            }
+            current_mount = current_mount->next;
+        }
+        if (current_mount == NULL) {
+            return CTRL_NO_PRESENT;
+        }
+        fs_user_mount_t *vfs = (fs_user_mount_t *) current_mount->obj;
+
         #if _MAX_SS != _MIN_SS
         if (vfs->ssize == FILESYSTEM_BLOCK_SIZE) {
         #else

atmel-samd/common-hal/audioio/AudioOut.c

@@ -27,6 +27,7 @@
 #include <stdint.h>
 #include <string.h>
 
+#include "extmod/vfs_fat_file.h"
 #include "py/gc.h"
 #include "py/mperrno.h"
 #include "py/runtime.h"

atmel-samd/fatfs_port.c

@@ -26,19 +26,8 @@
 
 #include "py/mphal.h"
 #include "py/runtime.h"
-#include "lib/fatfs/ff.h"        /* FatFs lower layer API */
-#include "lib/fatfs/diskio.h"    /* FatFs lower layer API */
-
-const PARTITION VolToPart[MICROPY_FATFS_VOLUMES] = {
-    {0, 1},     // Logical drive 0 ==> Physical drive 0, 1st partition
-    {1, 0},     // Logical drive 1 ==> Physical drive 1 (auto detection)
-    {2, 0},     // Logical drive 2 ==> Physical drive 2 (auto detection)
-    {3, 0},     // Logical drive 3 ==> Physical drive 3 (auto detection)
-    /*
-    {0, 2},     // Logical drive 2 ==> Physical drive 0, 2nd partition
-    {0, 3},     // Logical drive 3 ==> Physical drive 0, 3rd partition
-    */
-};
+#include "lib/oofatfs/ff.h"        /* FatFs lower layer API */
+#include "lib/oofatfs/diskio.h"    /* FatFs lower layer API */
 
 DWORD get_fattime(void) {
     // TODO(tannewt): Support the RTC.

atmel-samd/flash_api.c

@@ -31,13 +31,17 @@
 #define VFS_INDEX 0
 
 void flash_set_usb_writeable(bool usb_writeable) {
-    if (VFS_INDEX >= MP_ARRAY_SIZE(MP_STATE_PORT(fs_user_mount))) {
-        return;
+    mp_vfs_mount_t* current_mount = MP_STATE_VM(vfs_mount_table);
+    for (uint8_t i = 0; current_mount != NULL; i++) {
+        if (i == VFS_INDEX) {
+            break;
+        }
+        current_mount = current_mount->next;
     }
-    fs_user_mount_t *vfs = MP_STATE_PORT(fs_user_mount)[VFS_INDEX];
-    if (vfs == NULL) {
+    if (current_mount == NULL) {
         return;
     }
+    fs_user_mount_t *vfs = (fs_user_mount_t *) current_mount->obj;
 
     if (usb_writeable) {
         vfs->flags |= FSUSER_USB_WRITEABLE;

atmel-samd/flash_api.h

@@ -26,7 +26,7 @@
 #ifndef __MICROPY_INCLUDED_ATMEL_SAMD_FLASH_API_H__
 #define __MICROPY_INCLUDED_ATMEL_SAMD_FLASH_API_H__
 
-#include "extmod/fsusermount.h"
+#include "extmod/vfs_fat.h"
 
 extern void flash_init_vfs(fs_user_mount_t *vfs);
 extern void flash_flush(void);

atmel-samd/internal_flash.c

@@ -28,11 +28,12 @@
 #include <stdint.h>
 #include <string.h>
 
+#include "extmod/vfs.h"
+#include "extmod/vfs_fat.h"
 #include "py/mphal.h"
 #include "py/obj.h"
 #include "py/runtime.h"
-#include "lib/fatfs/ff.h"
-#include "extmod/fsusermount.h"
+#include "lib/oofatfs/ff.h"
 
 #include "asf/sam0/drivers/nvm/nvm.h"
 #include "asf/sam0/drivers/port/port.h"

atmel-samd/main.c

@@ -9,11 +9,11 @@
 #include "py/gc.h"
 #include "py/stackctrl.h"
 
-#include "lib/fatfs/ff.h"
-#include "lib/fatfs/diskio.h"
+#include "extmod/vfs_fat.h"
+#include "lib/oofatfs/ff.h"
+#include "lib/oofatfs/diskio.h"
 #include "lib/mp-readline/readline.h"
 #include "lib/utils/pyexec.h"
-#include "extmod/fsusermount.h"
 
 #include "asf/common/services/sleepmgr/sleepmgr.h"
 #include "asf/common/services/usb/udc/udc.h"
@@ -48,6 +48,7 @@
 #include "tick.h"
 
 fs_user_mount_t fs_user_mount_flash;
+mp_vfs_mount_t mp_vfs_mount_flash;
 
 typedef enum {
     NO_SAFE_MODE = 0,
@@ -80,39 +81,42 @@ void do_str(const char *src, mp_parse_input_kind_t input_kind) {
 // want it to be executed without using stack within main() function
 void init_flash_fs(void) {
     // init the vfs object
-    fs_user_mount_t *vfs = &fs_user_mount_flash;
-    vfs->str = "/flash";
-    vfs->len = 6;
-    vfs->flags = 0;
-    flash_init_vfs(vfs);
-
-    // put the flash device in slot 0 (it will be unused at this point)
-    MP_STATE_PORT(fs_user_mount)[0] = vfs;
+    fs_user_mount_t *vfs_fat = &fs_user_mount_flash;
+    vfs_fat->flags = 0;
+    flash_init_vfs(vfs_fat);
 
     // try to mount the flash
-    FRESULT res = f_mount(&vfs->fatfs, vfs->str, 1);
+    FRESULT res = f_mount(&vfs_fat->fatfs);
 
     if (res == FR_NO_FILESYSTEM) {
         // no filesystem so create a fresh one
 
-        res = f_mkfs("/flash", 0, 0);
+        uint8_t working_buf[_MAX_SS];
+        res = f_mkfs(&vfs_fat->fatfs, FM_FAT, 0, working_buf, sizeof(working_buf));
         // Flush the new file system to make sure its repaired immediately.
         flash_flush();
         if (res != FR_OK) {
-            MP_STATE_PORT(fs_user_mount)[0] = NULL;
             return;
         }
 
         // set label
-        f_setlabel("CIRCUITPY");
+        f_setlabel(&vfs_fat->fatfs, "CIRCUITPY");
     } else if (res != FR_OK) {
-        MP_STATE_PORT(fs_user_mount)[0] = NULL;
         return;
     }
+    mp_vfs_mount_t *vfs = m_new_obj_maybe(mp_vfs_mount_t);
+    if (vfs == NULL) {
+        return;
+    }
+    vfs->str = "/";
+    vfs->len = 1;
+    vfs->obj = MP_OBJ_FROM_PTR(vfs_fat);
+    vfs->next = NULL;
+    MP_STATE_VM(vfs_mount_table) = vfs;
 
     // The current directory is used as the boot up directory.
     // It is set to the internal flash filesystem by default.
-    f_chdrive("/flash");
+    MP_STATE_PORT(vfs_cur) = vfs;
 }
 
 static char heap[16384];
@@ -124,9 +128,7 @@ void reset_mp(void) {
 
     // Sync the file systems in case any used RAM from the GC to cache. As soon
     // as we re-init the GC all bets are off on the cache.
-    disk_ioctl(0, CTRL_SYNC, NULL);
-    disk_ioctl(1, CTRL_SYNC, NULL);
-    disk_ioctl(2, CTRL_SYNC, NULL);
+    flash_flush();
 
     // Clear the readline history. It references the heap we're about to destroy.
     readline_init0();
@@ -230,13 +232,8 @@ void reset_samd21(void) {
 }
 
 bool maybe_run(const char* filename, pyexec_result_t* exec_result) {
-    FILINFO fno;
-#if _USE_LFN
-    fno.lfname = NULL;
-    fno.lfsize = 0;
-#endif
-    FRESULT res = f_stat(filename, &fno);
-    if (res != FR_OK || fno.fattrib & AM_DIR) {
+    mp_import_stat_t stat = mp_import_stat(filename);
+    if (stat != MP_IMPORT_STAT_FILE) {
         return false;
     }
     mp_hal_stdout_tx_str(filename);
@@ -607,7 +604,8 @@ int main(void) {
         #ifdef CIRCUITPY_BOOT_OUTPUT_FILE
         FIL file_pointer;
         boot_output_file = &file_pointer;
-        f_open(boot_output_file, CIRCUITPY_BOOT_OUTPUT_FILE, FA_WRITE | FA_CREATE_ALWAYS);
+        f_open(&((fs_user_mount_t *) MP_STATE_VM(vfs_mount_table)->obj)->fatfs,
+            boot_output_file, CIRCUITPY_BOOT_OUTPUT_FILE, FA_WRITE | FA_CREATE_ALWAYS);
         #endif
 
         // TODO(tannewt): Re-add support for flashing boot error output.
@@ -688,17 +686,8 @@ void gc_collect(void) {
     gc_collect_end();
 }
 
-mp_import_stat_t fat_vfs_import_stat(const char *path);
-mp_import_stat_t mp_import_stat(const char *path) {
-    #if MICROPY_VFS_FAT
-    return fat_vfs_import_stat(path);
-    #else
-    (void)path;
-    return MP_IMPORT_STAT_NO_EXIST;
-    #endif
-}
-
-void nlr_jump_fail(void *val) {
+void NORETURN nlr_jump_fail(void *val) {
+    while (1);
 }
 
 void NORETURN __fatal_error(const char *msg) {