8000 Fix import · arangodb/arangojs@024edac · GitHub
[go: up one dir, main page]
Skip to content

Commit 024edac

Browse files
plumapluma
committed
Fix import
Fixes #461. Fixes #496.
1 parent 7aff85d commit 024edac

File tree

6 files changed

+256
-64
lines changed

6 files changed

+256
-64
lines changed

‎CHANGELOG.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
1414
This may result in type signatures that are incompatible with TypeScript 2
1515
being added in future releases (including patch releases).
1616

17+
- Reimplemented `collection.import`
18+
19+
The previous implementation was broken. The new implementation should be backwards-compatible
20+
in cases where it previously wasn't broken but is more flexible and also handles buffers.
21+
1722
## [6.5.1] - 2018-08-15
1823

1924
### Fixed

‎docs/Drivers/JS/Reference/Collection/BulkImport.md

Lines changed: 76 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -11,45 +11,90 @@ Bulk imports the given _data_ into the collection.
1111

1212
**Arguments**
1313

14-
* **data**: `Array<Array<any>> | Array<Object>`
15-
16-
The data to import. This can be an array of documents:
17-
18-
```js
19-
[
20-
{key1: value1, key2: value2}, // document 1
21-
{key1: value1, key2: value2}, // document 2
22-
...
23-
]
24-
```
25-
26-
Or it can be an array of value arrays following an array of keys.
27-
28-
```js
29-
[
30-
['key1', 'key2'], // key names
31-
[value1, value2], // document 1
32-
[value1, value2], // document 2
33-
...
34-
]
35-
```
36-
37-
* **opts**: `Object` (optional) If _opts_ is set, it must be an object with any
14+
- **data**: `Array | Buffer | string`
15+
16+
The data to import. Depending on the _type_ option this can be any of the
17+
following:
18+
19+
For type `"documents"` or `"auto"`:
20+
21+
- an array of documents, e.g.
22+
23+
```js
24+
[{ _key: "banana", color: "yellow" }, { _key: "peach", color: "pink" }];
25+
```
26+
27+
- a string or buffer containing one JSON document per line, e.g.
28+
29+
```json
30+
{"_key":"banana","color":"yellow"}
31+
{"_key":"peach","color":"pink"}
32+
```
33+
34+
For type `"array"` or `"auto"`:
35+
36+
- a string or buffer containing a JSON array of documents, e.g.
37+
38+
```json
39+
[
40+
{ "_key": "banana", "color": "yellow" },
41+
{ "_key": "peach", "color": "pink" }
42+
]
43+
```
44+
45+
For type `null`:
46+
47+
- an array containing an array of keys followed by arrays of values, e.g.
48+
49+
```js
50+
[["_key", "color"], ["banana", "yellow"], ["peach", "pink"]];
51+
```
52+
53+
- a string or buffer containing a JSON array of keys followed by
54+
one JSON array of values per line, e.g.
55+
56+
```json
57+
["_key", "color"][("banana", "yellow")][("peach", "pink")]
58+
```
59+
60+
- **opts**: `Object` (optional) If _opts_ is set, it must be an object with any
3861
of the following properties:
3962

40-
* **waitForSync**: `boolean` (Default: `false`)
63+
- **type**: `string | null` (Default: `"auto"`)
64+
65+
Indicates which format the data uses.
66+
Can be `"documents"`, `"array"` or `"auto"`.
67+
Use `null` to explicitly set no type.
68+
69+
- **fromPrefix**: `string` (optional)
70+
71+
Prefix to prepend to `_from` attributes.
72+
73+
- **toPrefix**: `string` (optional)
74+
75+
Prefix to prepend to `_to` attributes.
76+
77+
- **overwrite**: `boolean` (Default: `false`)
78+
79+
If set to `true`, the collection is truncated before the data is imported.
80+
81+
- **waitForSync**: `boolean` (Default: `false`)
4182

4283
Wait until the documents have been synced to disk.
4384

44-
* **details**: `boolean` (Default: `false`)
85+
- **onDuplicate**: `string` (Default: `"error"`)
4586

46-
Whether the response should contain additional details about documents that
47-
could not be imported.false\*.
87+
Controls behavior when a unique constraint is violated.
88+
Can be `"error"`, `"update"`, `"replace"` or `"ignore"`.
4889

49-
* **type**: `string` (Default: `"auto"`)
90+
- **complete**: `boolean` (Default: `false`)
5091

51-
Indicates which format the data uses. Can be `"documents"`, `"array"` or
52-
`"auto"`.
92+
If set to `true`, the import will abort if any error occurs.
93+
94+
- **details**: `boolean` (Default: `false`)
95+
96+
Whether the response should contain additional details about documents that
97+
could not be imported.
5398

5499
If _data_ is a JavaScript array, it will be transmitted as a line-delimited JSON
55100
stream. If _opts.type_ is set to `"array"`, it will be transmitted as regular

‎src/collection.ts

Lines changed: 34 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,27 @@ export type IndexHandle =
2020
id?: string;
2121
};
2222

23+
export interface ImportOptions {
24+
type?: null | "auto" | "documents" | "array";
25+
fromPrefix?: string;
26+
toPrefix?: string;
27+
overwrite?: boolean;
28+
waitForSync?: boolean;
29+
onDuplicate?: "error" | "update" | "replace" | "ignore";
30+
complete?: boolean;
31+
details?: boolean;
32+
}
33+
34+
export interface ImportResult {
35+
error: false;
36+
created: number;
37+
errors: number;
38+
empty: number;
39+
updated: number;
40+
ignored: number;
41+
details?: string[];
42+
}
43+
2344
export function isArangoCollection(
2445
collection: any
2546
): collection is ArangoCollection {
@@ -500,14 +521,24 @@ export abstract class BaseCollection implements ArangoCollection {
500521
);
501522
}
502523

503-
import(data: any, opts?: any) {
524+
import(
525+
data: Buffer | Blob | string | any[],
526+
{ type = "auto", ...opts }: ImportOptions = {}
527+
): Promise<ImportResult> {
528+
if (Array.isArray(data)) {
529+
data = data.map(line => JSON.stringify(line)).join("\r\n") + "\r\n";
530+
}
504531
return this._connection.request(
505532
{
506533
method: "POST",
507534
path: "/_api/import",
508535
body: data,
509-
isJsonStream: Boolean(!opts || opts.type !== "array"),
510-
qs: { type: "auto", ...opts, collection: this.name }
536+
isBinary: true,
537+
qs: {
538+
type: type === null ? undefined : type,
539+
...opts,
540+
collection: this.name
541+
}
511542
},
512543
res => res.body
513544
);

‎src/connection.ts

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,6 @@ export type RequestOptions = {
4242
body?: any;
4343
expectBinary?: boolean;
4444
isBinary?: boolean;
45-
isJsonStream?: boolean;
4645
headers?: { [key: string]: string };
4746
absolutePath?: boolean;
4847
basePath?: string;
@@ -272,7 +271,6 @@ export class Connection {
272271
body,
273272
expectBinary = false,
274273
isBinary = false,
275-
isJsonStream = false,
276274
headers,
277275
...urlInfo
278276
}: RequestOptions,
@@ -284,14 +282,8 @@ export class Connection {
284282
contentType = "application/octet-stream";
285283
} else if (body) {
286284
if (typeof body === "object") {
287-
if (isJsonStream) {
288-
body =
289-
body.map((obj: any) => JSON.stringify(obj)).join("\r\n") + "\r\n";
290-
contentType = "application/x-ldjson";
291-
} else {
292-
body = JSON.stringify(body);
293-
contentType = "application/json";
294-
}
285+
body = JSON.stringify(body);
286+
contentType = "application/json";
295287
} else {
296288
body = String(body);
297289
}

‎src/test/13-bulk-imports.ts

Lines changed: 138 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import { expect } from "chai";
22
import { Database } from "../arangojs";
3-
import { DocumentCollection } from "../collection";
3+
import { DocumentCollection, ImportOptions } from "../collection";
44

55
describe("Bulk imports", function() {
66
// create database takes 11s in a standard cluster
@@ -28,25 +28,144 @@ describe("Bulk imports", function() {
2828
db.close();
2929
}
3030
});
31-
beforeEach(done => {
32-
collection
33-
.truncate()
34-
.then(() => done())
35-
.catch(done);
36-
});
3731
describe("collection.import", () => {
38-
it("should import new documents", done => {
39-
const data = [{}, {}, {}];
40-
collection
41-
.import(data)
42-
.then(info => {
43-
expect(info).to.have.property("created", data.length);
44-
expect(info).to.have.property("updated", 0);
45-
expect(info).to.have.property("ignored", 0);
46-
expect(info).to.have.property("empty", 0);
47-
})
48-
.then(() => done())
49-
.catch(done);
32+
describe("with type null", () => {
33+
it("should accept tuples array", async () => {
34+
const data = [
35+
["_key", "data"],
36+
["ta1", "banana"],
37+
["ta2", "peach"],
38+
["ta3", "apricot"]
39+
];
40+
const info = await collection.import(data, { type: null });
41+
expect(info).to.eql({
42+
error: false,
43+
created: 3,
44+
errors: 0,
45+
empty: 0,
46+
updated: 0,
47+
ignored: 0
48+
});
49+
});
50+
it("should accept tuples string", async () => {
51+
const data =
52+
'["_key", "data"]\r\n["ts1", "banana"]\r\n["ts2", "peach"]\r\n["ts3", "apricot"]\r\n';
53+
const info = await collection.import(data, { type: null });
54+
expect(info).to.eql({
55+
error: false,
56+
created: 3,
57+
errors: 0,
58+
empty: 0,
59+
updated: 0,
60+
ignored: 0
61+
});
62+
});
63+
it("should accept tuples buffer", async () => {
64+
const data = Buffer.from(
65+
'["_key", "data"]\r\n["tb1", "banana"]\r\n["tb2", "peach"]\r\n["tb3", "apricot"]\r\n'
66+
);
67+
const info = await collection.import(data, { type: null });
68+
expect(info).to.eql({
69+
error: false,
70+
created: 3,
71+
errors: 0,
72+
empty: 0,
73+
updated: 0,
74+
ignored: 0
75+
});
76+
});
5077
});
78+
for (const type of [
79+
undefined,
80+
"auto",
81+
"documents"
82+
] as ImportOptions["type"][]) {
83+
describe(`with type ${JSON.stringify(type)}`, () => {
84+
it("should accept documents array", async () => {
85+
const data = [
86+
{ _key: `da1-${type}`, data: "banana" },
87+
{ _key: `da2-${type}`, data: "peach" },
88+
{ _key: `da3-${type}`, data: "apricot" }
89+
];
90+
const info = await collection.import(data, { type });
91+
expect(info).to.eql({
92+
error: false,
93+
created: 3,
94+
errors: 0,
95+
empty: 0,
96+
updated: 0,
97+
ignored: 0
98+
});
99+
});
100+
it("should accept documents string", async () => {
101+
const data = `{"_key": "ds1-${type}", "data": "banana"}\r\n{"_key": "ds2-${type}", "data": "peach"}\r\n{"_key": "ds3-${type}", "data": "apricot"}\r\n`;
102+
const info = await collection.import(data, { type });
103+
expect(info).to.eql({
104+
error: false,
105+
created: 3,
106+
errors: 0,
107+
empty: 0,
108+
updated: 0,
109+
ignored: 0
110+
});
111+
});
112+
it("should accept documents buffer", async () => {
113+
const data = Buffer.from(
114+
`{"_key": "db1-${type}", "data": "banana"}\r\n{"_key": "db2-${type}", "data": "peach"}\r\n{"_key": "db3-${type}", "data": "apricot"}\r\n`
115+
);
116+
const info = await collection.import(data, { type });
117+
expect(info).to.eql({
118+
error: false,
119+
created: 3,
120+
errors: 0,
121+
empty: 0,
122+
updated: 0,
123+
ignored: 0
124+
});
125+
});
126+
});
127+
}
128+
for (const type of [
129+
undefined,
130+
"auto",
131+
"array"
132+
] as ImportOptions["type"][]) {
133+
describe(`with type ${JSON.stringify(type)}`, () => {
134+
it("should accept JSON string", async () => {
135+
const data = JSON.stringify([
136+
{ _key: `js1-${String(type)}`, data: "banana" },
137+
{ _key: `js2-${String(type)}`, data: "peach" },
138+
{ _key: `js3-${String(type)}`, data: "apricot" }
139+
]);
140+
const info = await collection.import(data, { type });
141+
expect(info).to.eql({
142+
error: false,
143+
created: 3,
144+
errors: 0,
145+
empty: 0,
146+
updated: 0,
147+
ignored: 0
148+
});
149+
});
150+
it("should accept JSON buffer", async () => {
151+
const data = Buffer.from(
152+
JSON.stringify([
153+
{ _key: `jb1-${String(type)}`, data: "banana" },
154+
{ _key: `jb2-${String(type)}`, data: "peach" },
155+
{ _key: `jb3-${String(type)}`, data: "apricot" }
156+
])
157+
);
158+
const info = await collection.import(data, { type });
159+
expect(info).to.eql({
160+
error: false,
161+
created: 3,
162+
errors: 0,
163+
empty: 0,
164+
updated: 0,
165+
ignored: 0
166+
});
167+
});
168+
});
169+
}
51170
});
52171
});

0 commit comments

Comments
 (0)
0