8000 Fix vector index restoration (#21770) · arangodb/arangodb@c666194 · GitHub
[go: up one dir, main page]
Skip to content

Commit c666194

Browse files
jbajicjbajic
authored
Fix vector index restoration (#21770)
* Fix vector restore * Fix restore * Fix tests * Add arm exception
1 parent 6a8c799 commit c666194

File tree

4 files changed

+121
-4
lines changed

4 files changed

+121
-4
lines changed

‎CHANGELOG

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
devel
22
-----
33

4+
* Fix arangorestore failing to restore the dump containing a
5+
vector index.
6+
47
* FE-575: adds debounce and loading state for node start search.
58

69
* Fix BTS-2086: Fixes missing reschedule of license checks on single-server

‎client-tools/Restore/RestoreFeature.cpp

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1808,18 +1808,35 @@ arangodb::Result RestoreFeature::RestoreMainJob::restoreIndexes(
18081808
}
18091809
}
18101810
}
1811-
indexes = newIndexes.slice();
1811+
1812+
VPackBuilder rewrittenParametersBuilder;
1813+
TRI_ASSERT(parameters.isObject())
1814+
<< "The parameters is expected to be an object!";
1815+
{
1816+
VPackObjectBuilder guard(&rewrittenParametersBuilder);
1817+
VPackObjectIterator it(parameters);
1818+
for (VPackObjectIterator it(parameters); it.valid(); it.next()) {
1819+
auto const key = it.key();
1820+
if (key.toString() == "indexes") {
1821+
continue;
1822+
}
1823+
rewrittenParametersBuilder.add(key);
1824+
rewrittenParametersBuilder.add(it.value());
1825+
}
1826+
rewrittenParametersBuilder.add("indexes", newIndexes.slice());
1827+
}
1828+
auto rewrittenParameters = rewrittenParametersBuilder.slice();
18121829

18131830
// re-create indexes
1814-
if (indexes.length() > 0) {
1831+
if (rewrittenParameters.get("indexes").length() > 0) {
18151832
// we actually have indexes
18161833

18171834
if (options.progress) {
18181835
LOG_TOPIC("d88c6", INFO, Logger::RESTORE)
18191836
<< "# Creating indexes for collection '" << collectionName << "'...";
18201837
}
18211838

1822-
result = sendRestoreIndexes(client, parameters);
1839+
result = sendRestoreIndexes(client, rewrittenParameters);
18231840

18241841
if (result.fail()) {
18251842
LOG_TOPIC("db937", WARN, Logger::RESTORE)

‎js/client/modules/@arangodb/testsuites/aql.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,7 @@ function shellClient (options) {
156156
// increase timeouts after which servers count as BAD/FAILED.
157157
// we want this to ensure that in an overload situation we do not
158158
// get random failedLeader / failedFollower jobs during our tests.
159-
let moreOptions = { "agency.supervision-ok-threshold" : "15", "agency.supervision-grace-period" : "30" };
159+
let moreOptions = { "agency.supervision-ok-threshold" : "15", "agency.supervision-grace-period" : "30", "experimental-vector-index": true };
160160
let rc = new trs.runLocalInArangoshRunner(opts, 'shell_client', moreOptions).run(testCases);
161161
options.cleanup = options.cleanup && opts.cleanup;
162162
return rc;

‎tests/js/client/shell/shell-restore-integration.js

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ const pu = require('@arangodb/testutils/process-utils');
3333
const db = arangodb.db;
3434
const isCluster = require("internal").isCluster();
3535
const { executeExternalAndWaitWithSanitizer } = require('@arangodb/test-helper');
36+
const { versionHas } = require("@arangodb/test-helper");
3637
const dbs = [{"name": "maçã", "id": "9999994", "isUnicode": true}, {
3738
"name": "cachorro",
3839
"id": "9999995",
@@ -1359,6 +1360,102 @@ function restoreIntegrationSuite() {
13591360
};
13601361
}
13611362

1363+
function restoreIntegrationVectorSuite() {
1364+
'use strict';
1365+
const cn = 'UnitTestsVectorIndexRestore';
1366+
const arangorestore = pu.ARANGORESTORE_BIN;
1367+
1368+
assertTrue(fs.isFile(arangorestore), "arangorestore not found!");
1369+
1370+
let addConnectionArgs = function (args) {
1371+
let endpoint = arango.getEndpoint().replace(/\+vpp/, '').replace(/^http:/, 'tcp:').replace(/^https:/, 'ssl:').replace(/^h2:/, 'tcp:');
1372+
args.push('--server.endpoint');
1373+
args.push(endpoint);
1374+
if (args.indexOf("--all-databases") === -1 && args.indexOf("--server.database") === -1) {
1375+
args.push('--server.database');
1376+
args.push(arango.getDatabaseName());
1377+
}
1378+
args.push('--server.username');
1379+
args.push(arango.connectedUser());
1380+
};
1381+
1382+
let runRestore = function (path, args, rc) {
1383+
args.push('--input-directory');
1384+
args.push(path);
1385+
addConnectionArgs(args);
1386+
1387+
const actualRc = executeExternalAndWaitWithSanitizer(arangorestore, args, 'shell-restore-integration');
1388+
assertTrue(actualRc.hasOwnProperty("exit"), actualRc);
1389+
assertEqual(rc, actualRc.exit, actualRc);
1390+
};
1391+
1392+
return {
1393+
1394+
setUp: function () {
1395+
db._drop(cn);
1396+
},
1397+
1398+
tearDown: function () {
1399+
db._drop(cn);
1400+
db._databases().forEach((database) => {
1401+
if (database !== "_system") {
1402+
db._dropDatabase(database);
1403+
}
1404+
});
1405+
},
1406+
1407+
testRestoreVectorIndex: function () {
1408+
let path = fs.getTempFile();
1409+
fs.makeDirectory(path);
1410+
let fn = fs.join(path, cn + ".structure.json");
1411+
fs.write(fn, JSON.stringify({
1412+
indexes: [],
1413+
parameters: {
1414+
indexes: [
1415+
{id: "0", fields: ["_key"], type: "primary", unique: true},
1416+
{id: "95", fields: ["vector"], type: "vector", params: {dimension: 4, nLists:4, metric: "l2"}},
1417+
{id: "295", fields: ["value"], type: "persistent", sparse: true},
1418+
],
1419+
name: cn,
1420+
numberOfShards: 3,
1421+
type: 2
1422+
}
1423+
}));
1424+
let data = [];
1425+
for (let i = 0; i < 1000; ++i) {
1426+
data.push({_key: "test" + i, value: i, vector: [0, i / 10, i / 100, i / 1000]});
1427+
}
1428+
createCollectionDataFile(data, path, cn, /*split*/ false);
1429+
1430+
let args = ['--collection', cn, '--import-data', 'true'];
1431+
runRestore(path, args, 0);
1432+
1433+
let c = db._collection(cn);
1434+
let indexes = c.indexes();
1435+
assertEqual(3, indexes.length);
1436+
assertEqual("primary", indexes[0].type);
1437+
assertEqual(["_key"], indexes[0].fields);
1438+
assertEqual("vector", indexes[1].type);
1439+
assertEqual(["vector"], indexes[1].fields);
1440+
assertEqual("persistent", indexes[2].type);
1441+
assertEqual(["value"], indexes[2].fields);
1442+
1443+
// test if the vector index works
1444+
for (let i = 0; i < 100; ++i) {
1445+
c.insert({value: i, vector: [0, 0, 0, 0]});
1446+
}
1447+
let result = db._query("FOR doc IN " + cn + " LET dist = APPROX_NEAR_L2(doc.vector, [0, 0, 0, 0]) SORT dist LIMIT 10 RETURN doc").toArray();
1448+
assertEqual(10, result.length);
1449+
fs.removeDirectoryRecursive(path, true);
1450+
}
1451+
};
1452+
}
1453+
1454+
13621455
jsunity.run(restoreIntegrationSuite);
13631456

1457+
if 4AB7 (!versionHas("arm")) {
1458+
jsunity.run(restoreIntegrationVectorSuite);
1459+
}
1460+
13641461
return jsunity.done();

0 commit comments

Comments
 (0)
0