8000 extmod/vfs_reader: Add file ioctl to set read buffer size. · micropython/micropython@4cf7410 · GitHub
[go: up one dir, main page]
Skip to content

Commit 4cf7410

Browse files
pi-anldpgeorge
pi-anl
authored and
dpgeorge
committed
extmod/vfs_reader: Add file ioctl to set read buffer size.
Can be used to speed up importing a file from a vfs based filesystem. Signed-off-by: Andrew Leech <andrew.leech@planetinnovation.com.au>
1 parent dff2938 commit 4cf7410

File tree

4 files changed

+69
-16
lines changed

4 files changed

+69
-16
lines changed

‎extmod/vfs_reader.c‎

Lines changed: 35 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626

2727
#include <stdio.h>
2828
#include <string.h>
29+
#include <stdlib.h>
2930

3031
#include "py/runtime.h"
3132
#include "py/stream.h"
@@ -34,33 +35,39 @@
3435

3536
#if MICROPY_READER_VFS
3637

38+
#ifndef MICROPY_READER_VFS_DEFAULT_BUFFER_SIZE
39+
#define MICROPY_READER_VFS_DEFAULT_BUFFER_SIZE (2 * MICROPY_BYTES_PER_GC_BLOCK - offsetof(mp_reader_vfs_t, buf))
40+
#endif
41+
#define MICROPY_READER_VFS_MIN_BUFFER_SIZE (MICROPY_BYTES_PER_GC_BLOCK - offsetof(mp_reader_vfs_t, buf))
42+
#define MICROPY_READER_VFS_MAX_BUFFER_SIZE (255)
43+
3744
typedef struct _mp_reader_vfs_t {
3845
mp_obj_t file;
39-
uint16_t len;
40-
uint16_t pos;
41-
byte buf[24];
46+
uint8_t bufpos;
47+
uint8_t buflen;
48+
uint8_t bufsize;
49+
byte buf[];
4250
} mp_reader_vfs_t;
4351

4452
STATIC mp_uint_t mp_reader_vfs_readbyte(void *data) {
4553
mp_reader_vfs_t *reader = (mp_reader_vfs_t *)data;
46-
if (reader->pos >= reader->len) {
47-
if (reader->len < sizeof(reader->buf)) {
54+
if (reader->bufpos >= reader->buflen) {
55+
if (reader->buflen < reader->bufsize) {
4856
return MP_READER_EOF;
4957
} else {
5058
int errcode;
51-
reader->len = mp_stream_rw(reader->file, reader->buf, sizeof(reader->buf),
52-
&errcode, MP_STREAM_RW_READ | MP_STREAM_RW_ONCE);
59+
reader->buflen = mp_stream_rw(reader->file, reader->buf, reader->bufsize, &errcode, MP_STREAM_RW_READ | MP_STREAM_RW_ONCE);
5360
if (errcode != 0) {
5461
// TODO handle errors properly
5562
return MP_READER_EOF;
5663
}
57-
if (reader->len == 0) {
64+
if (reader->buflen == 0) {
5865
return MP_READER_EOF;
5966
}
60-
reader->pos = 0;
67+
reader->bufpos = 0;
6168
}
6269
}
63-
return reader->buf[reader->pos++];
70+
return reader->buf[reader->bufpos++];
6471
}
6572

6673
STATIC void mp_reader_vfs_close(void *data) {
@@ -70,18 +77,31 @@ STATIC void mp_reader_vfs_close(void *data) {
7077
}
7178

7279
void mp_reader_new_file(mp_reader_t *reader, qstr filename) {
73-
mp_reader_vfs_t *rf = m_new_obj(mp_reader_vfs_t);
7480
mp_obj_t args[2] = {
7581
MP_OBJ_NEW_QSTR(filename),
7682
MP_OBJ_NEW_QSTR(MP_QSTR_rb),
7783
};
78-
rf->file = mp_vfs_open(MP_ARRAY_SIZE(args), &args[0], (mp_map_t *)&mp_const_empty_map);
79-
int errcode;
80-
rf->len = mp_stream_rw(rf->file, rf->buf, sizeof(rf->buf), &errcode, MP_STREAM_RW_READ | MP_STREAM_RW_ONCE);
84+
mp_obj_t file = mp_vfs_open(MP_ARRAY_SIZE(args), &args[0], (mp_map_t *)&mp_const_empty_map);
85+
86+
const mp_stream_p_t *stream_p = mp_get_stream(file);
87+
int errcode = 0;
88+
mp_uint_t bufsize = stream_p->ioctl(file, MP_STREAM_GET_BUFFER_SIZE, 0, &errcode);
89+
if (bufsize == MP_STREAM_ERROR || bufsize == 0) {
90+
// bufsize == 0 is included here to support mpremote v1.21 and older where mount file ioctl
91+
// returned 0 by default.
92+
bufsize = MICROPY_READER_VFS_DEFAULT_BUFFER_SIZE;
93+
} else {
94+
bufsize = MIN(MICROPY_READER_VFS_MAX_BUFFER_SIZE, MAX(MICROPY_READER_VFS_MIN_BUFFER_SIZE, bufsize));
95+
}
96+
97+
mp_reader_vfs_t *rf = m_new_obj_var(mp_reader_vfs_t, buf, byte, bufsize);
98+
rf->file = file;
99+
rf->bufsize = bufsize;
100+
rf->buflen = mp_stream_rw(rf->file, rf->buf, rf->bufsize, &errcode, MP_STREAM_RW_READ | MP_STREAM_RW_ONCE);
81101
if (errcode != 0) {
82102
mp_raise_OSError(errcode);
83103
}
84-
rf->pos = 0;
104+
rf->bufpos = 0;
85105
reader->data = rf;
86106
reader->readbyte = mp_reader_vfs_readbyte;
87107
reader->close = mp_reader_vfs_close;

‎py/stream.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
#define MP_STREAM_GET_DATA_OPTS (8) // Get data/message options
4444
#define MP_STREAM_SET_DATA_OPTS (9) // Set data/message options
4545
#define MP_STREAM_GET_FILENO (10) // Get fileno of underlying file
46+
#define MP_STREAM_GET_BUFFER_SIZE (11) // Get preferred buffer size for file
4647

4748
// These poll ioctl values are compatible with Linux
4849
#define MP_STREAM_POLL_RD (0x0001)

‎tests/extmod/vfs_userfs.py‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616

1717

1818
class UserFile(io.IOBase):
19+
buffer_size = 16
20+
1921
def __init__(self, mode, data):
2022
assert isinstance(data, bytes)
2123
self.is_text = mode.find("b") == -1
@@ -39,7 +41,11 @@ def readinto(self, buf):
3941

4042
def ioctl(self, req, arg):
4143
print("ioctl", req, arg)
42-
return 0
44+
if req == 4: # MP_STREAM_CLOSE
45+
return 0
46+
if req == 11: # MP_STREAM_GET_BUFFER_SIZE
47+
return UserFile.buffer_size
48+
return -1
4349

4450

4551
class UserFS:
@@ -70,6 +76,8 @@ def open(self, path, mode):
7076
"/usermod2.py": b"print('in usermod2')",
7177
"/usermod3.py": b"syntax error",
7278
"/usermod4.mpy": b"syntax error",
79+
"/usermod5.py": b"print('in usermod5')",
80+
"/usermod6.py": b"print('in usermod6')",
7381
}
7482
os.mount(UserFS(user_files), "/userfs")
7583

@@ -93,6 +101,14 @@ def open(self, path, mode):
93101
except ValueError:
94102
print("ValueError in usermod4")
95103

104+
# Test an import with largest buffer size
105+
UserFile.buffer_size = 255
106+
import usermod5
107+
108+
# Test an import with over-size buffer size (should be safely limited internally)
109+
UserFile.buffer_size = 1024
110+
import usermod6
111+
96112
# unmount and undo path addition
97113
os.umount("/userfs")
98114
sys.path.pop()

‎tests/extmod/vfs_userfs.py.exp‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,21 +3,37 @@ some data in a text file
33
stat /usermod1
44
stat /usermod1.py
55
open /usermod1.py rb
6+
ioctl 11 0
67
ioctl 4 0
78
in usermod1
89
stat /usermod2
910
stat /usermod2.py
1011
open /usermod2.py rb
12+
ioctl 11 0
1113
ioctl 4 0
1214
in usermod2
1315
stat /usermod3
1416
stat /usermod3.py
1517
open /usermod3.py rb
18+
ioctl 11 0
1619
ioctl 4 0
1720
SyntaxError in usermod3
1821
stat /usermod4
1922
stat /usermod4.py
2023
stat /usermod4.mpy
2124
open /usermod4.mpy rb
25+
ioctl 11 0
2226
ioctl 4 0
2327
ValueError in usermod4
28+
stat /usermod5
29+
stat /usermod5.py
30+
open /usermod5.py rb
31+
ioctl 11 0
32+
ioctl 4 0
33+
in usermod5
34+
stat /usermod6
35+
stat /usermod6.py
36+
open /usermod6.py rb
37+
ioctl 11 0
38+
ioctl 4 0
39+
in usermod6

0 commit comments

Comments
 (0)
0