10000 bpo-24905: Support BLOB incremental I/O in sqlite module by palaviv · Pull Request #271 · python/cpython · GitHub
[go: up one dir, main page]
Skip to content

bpo-24905: Support BLOB incremental I/O in sqlite module #271

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 18 commits into from
Closed

bpo-24905: Support BLOB incremental I/O in sqlite module #271

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
4fa0e91
support BLOB incremental I/O in sqlite module
palaviv Feb 24, 2017
3fe9108
Note that blob size cannot be changed using the blob object
palaviv Feb 25, 2017
865c1c8
Use assertRaises in tests
palaviv Feb 25, 2017
788fd54
Fix doc error
palaviv Feb 25, 2017
a1361e5
blob support sequence protocol
palaviv Mar 4, 2017
5aedfba
Calculate blob length once at creation
palaviv Mar 4, 2017
db6ef32
Add initial Doc for sequence protocol
palaviv Mar 4, 2017
219f4cb
Don't support blob operation
palaviv Apr 18, 2018
6dafe0e
move news entry to blurb
palaviv Apr 18, 2018
ffac901
Add blob to PCBuild
palaviv Apr 18, 2018
3475fa1
Update version
palaviv May 8, 2019
f6015ff
Fix memory leak
palaviv May 8, 2019
01e526c
Update version
palaviv Jul 29, 2020
354bebf
Fix CR comments in documentation and testing
palaviv Aug 3, 2020
9709456
Fix CR comments in code
palaviv Aug 3, 2020
e6e5099
Make readonly and dbname keyword arguements only
palaviv Aug 3, 2020
e9a8080
Fix more CR comments
palaviv Aug 3, 2020
d4fb1b5
Add more indicative error on write bigger then blob length
palaviv Aug 3, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Calculate blob length once at creation
  • Loading branch information
@palaviv
palaviv committed Jul 29, 2020
commit 5aedfba9a1541473e72faaaaabf3d63e0b972ad6
73 changes: 20 additions & 53 deletions Modules/_sqlite/blob.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,10 @@ int pysqlite_blob_init(pysqlite_Blob *self, pysqlite_Connection* connection,
self->blob = blob;
self->in_weakreflist = NULL;

Py_BEGIN_ALLOW_THREADS
self->length = sqlite3_blob_bytes(self->blob);
Py_END_ALLOW_THREADS

if (!pysqlite_check_thread(self->connection)) {
return -1;
}
Expand Down Expand Up @@ -94,15 +98,11 @@ PyObject* pysqlite_blob_close(pysqlite_Blob *self)

static Py_ssize_t pysqlite_blob_length(pysqlite_Blob *self)
{
int blob_length;
if (!pysqlite_check_blob(self)) {
return -1;
}
Py_BEGIN_ALLOW_THREADS
blob_length = sqlite3_blob_bytes(self->blob);
Py_END_ALLOW_THREADS

return blob_length;
return self->length;
};

static PyObject* inner_read(pysqlite_Blob *self, int read_length, int offset)
Expand Down Expand Up @@ -140,7 +140,6 @@ static PyObject* inner_read(pysqlite_Blob *self, int read_length, int offset)
PyObject* pysqlite_blob_read(pysqlite_Blob *self, PyObject *args)
{
int read_length = -1;
int blob_length = 0;
PyObject *buffer;

if (!PyArg_ParseTuple(args, "|i", &read_length)) {
Expand All @@ -151,20 +150,14 @@ PyObject* pysqlite_blob_read(pysqlite_Blob *self, PyObject *args)
return NULL;
}


/* TODO: make this multithreaded and safe! */
Py_BEGIN_ALLOW_THREADS
blob_length = sqlite3_blob_bytes(self->blob);
Py_END_ALLOW_THREADS

if (read_length < 0) {
/* same as file read. */
read_length = blob_length;
read_length = self->length;
}

/* making sure we don't read more then blob size */
if (read_length > blob_length - self->offset) {
read_length = blob_length - self->offset;
if (read_length > self->length - self->offset) {
read_length = self->length - self->offset;
}

buffer = inner_read(self, read_length, self->offset);
Expand Down Expand Up @@ -236,7 +229,7 @@ PyObject* pysqlite_blob_write(pysqlite_Blob *self, PyObject *data)

PyObject* pysqlite_blob_seek(pysqlite_Blob *self, PyObject *args)
{
int blob_length, offset, from_what = 0;
int offset, from_what = 0;

if (!PyArg_ParseTuple(args, "i|i", &offset, &from_what)) {
return NULL;
Expand All @@ -247,10 +240,6 @@ PyObject* pysqlite_blob_seek(pysqlite_Blob *self, PyObject *args)
return NULL;
}

Py_BEGIN_ALLOW_THREADS
blob_length = sqlite3_blob_bytes(self->blob);
Py_END_ALLOW_THREADS

switch (from_what) {
case 0: // relative to blob begin
break;
Expand All @@ -261,17 +250,17 @@ PyObject* pysqlite_blob_seek(pysqlite_Blob *self, PyObject *args)
offset = self->offset + offset;
break;
case 2: // relative to blob end
if (offset > INT_MAX - blob_length) {
if (offset > INT_MAX - self->length) {
goto overflow;
}
offset = blob_length + offset;
offset = self->length + offset;
break;
default:
return PyErr_Format(PyExc_ValueError,
"from_what should be 0, 1 or 2");
}

if (offset < 0 || offset > blob_length) {
if (offset < 0 || offset > self->length) {
return PyErr_Format(PyExc_ValueError, "offset out of blob range");
}

Expand Down Expand Up @@ -340,17 +329,11 @@ static PyObject* pysqlite_blob_repeat(pysqlite_Blob *self, PyObject *args)

static PyObject* pysqlite_blob_item(pysqlite_Blob *self, Py_ssize_t i)
{
int blob_length = 0;

if (!pysqlite_check_blob(self)) {
return NULL;
}

Py_BEGIN_ALLOW_THREADS
blob_length = sqlite3_blob_bytes(self->blob);
Py_END_ALLOW_THREADS

if (i < 0 || i >= blob_length) {
if (i < 0 || i >= self->length) {
PyErr_SetString(PyExc_IndexError, "Blob index out of range");
return NULL;
}
Expand All @@ -360,18 +343,13 @@ static PyObject* pysqlite_blob_item(pysqlite_Blob *self, Py_ssize_t i)

static int pysqlite_blob_ass_item(pysqlite_Blob *self, Py_ssize_t i, PyObject *v)
{
int blob_length = 0;
const char *buf;

if (!pysqlite_check_blob(self)) {
return -1;
}

Py_BEGIN_ALLOW_THREADS
blob_length = sqlite3_blob_bytes(self->blob);
Py_END_ALLOW_THREADS

if (i < 0 || i >= blob_length) {
if (i < 0 || i >= self->length) {
PyErr_SetString(PyExc_IndexError, "Blob index out of range");
return -1;
}
Expand All @@ -393,23 +371,17 @@ static int pysqlite_blob_ass_item(pysqlite_Blob *self, Py_ssize_t i, PyObject *v

static PyObject * pysqlite_blob_subscript(pysqlite_Blob *self, PyObject *item)
{
int blob_length = 0;

if (!pysqlite_check_blob(self)) {
return NULL;
}

Py_BEGIN_ALLOW_THREADS
blob_length = sqlite3_blob_bytes(self->blob);
Py_END_ALLOW_THREADS

if (PyIndex_Check(item)) {
Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError);
if (i == -1 && PyErr_Occurred())
return NULL;
if (i < 0)
i += blob_length;
if (i < 0 || i >= blob_length) {
i += self->length;
if (i < 0 || i >= self->length) {
PyErr_SetString(PyExc_IndexError,
"Blob index out of range");
return NULL;
Expand All @@ -420,7 +392,7 @@ static PyObject * pysqlite_blob_subscript(pysqlite_Blob *self, PyObject *item)
else if (PySlice_Check(item)) {
Py_ssize_t start, stop, step, slicelen;

if (PySlice_GetIndicesEx(item, blob_length,
if (PySlice_GetIndicesEx(item, self->length,
&start, &stop, &step, &slicelen) < 0) {
return NULL;
}
Expand Down Expand Up @@ -484,26 +456,21 @@ static PyObject * pysqlite_blob_subscript(pysqlite_Blob *self, PyObject *item)

static int pysqlite_blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyObject *value)
{
int blob_length = 0;
int rc;

if (!pysqlite_check_blob(self)) {
return -1;
}

Py_BEGIN_ALLOW_THREADS
blob_length = sqlite3_blob_bytes(self->blob);
Py_END_ALLOW_THREADS

if (PyIndex_Check(item)) {
Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError);
const char *buf;

if (i == -1 && PyErr_Occurred())
return -1;
if (i < 0)
i += blob_length;
if (i < 0 || i >= blob_length) {
i += self->length;
if (i < 0 || i >= self->length) {
PyErr_SetString(PyExc_IndexError,
"Blob index out of range");
return -1;
Expand All @@ -527,7 +494,7 @@ static int pysqlite_blob_ass_subscript(pysqlite_Blob *self, PyObject *item, PyOb
Py_buffer vbuf;

if (PySlice_GetIndicesEx(item,
blob_length, &start, &stop,
self->length, &start, &stop,
&step, &slicelen) < 0) {
return -1;
}
Expand Down
1 change: 1 addition & 0 deletions Modules/_sqlite/blob.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ typedef struct
pysqlite_Connection* connection;
sqlite3_blob *blob;
int offset;
int length;

PyObject* in_weakreflist; /* List of weak references */
} pysqlite_Blob;
Expand Down
0