8000 Backup with view (#10386) · arangodb/arangodb@ebf3296 · GitHub
[go: up one dir, main page]
Skip to content

Commit ebf3296

Browse files
Lars Maierneunhoef
Lars Maier
authored and
neunhoef
committed
Backup with view (#10386)
* Commit and store view related data on create backup. * Fixes to upgrade on env-var. * Fixed CMakeLists.txt. * Changes to allow restore from 3.5 to 3.6. * Remove unnecessary code introduced in merge. * CHANGELOG:
1 parent caffa6a commit ebf3296

File tree

10 files changed

+79
-36
lines changed

10 files changed

+79
-36
lines changed

‎CHANGELOG

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
devel
22
-----
3+
4+
* Include ArangoSearch data in hotbackups.
5+
6+
* Allow to restore 3.5 hotbackups in 3.6.
7+
38
* Fixed ArangoSearch index removes being discarded on commiting consolidation results with
49
pending removes after some segments under consolidation were already committed
510

‎CMakeLists.txt

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1032,7 +1032,7 @@ else ()
10321032
else ()
10331033
set (OPENSSL_VERSION_PATTERN ".*OPENSSL_${TARGET_OS}[ ]+\"([^\"a-z]*).*")
10341034
endif ()
1035-
string(REGEX MATCH
1035+
string(REGEX MATCH
10361036
"${OPENSSL_VERSION_PATTERN}"
10371037
ARANGODB_REQUIRED_OPENSSL_VERSION
10381038
"${ARANGODB_VERSIONS_CONTENT}")
@@ -1384,3 +1384,6 @@ message(STATUS "building for git revision: ${ARANGODB_BUILD_REPOSITORY}")
13841384
# message(STATUS "${_variableName}=${${_variableName}}")
13851385
# endforeach ()
13861386
# endif ()
1387+
1388+
add_custom_target(arangodb
1389+
DEPENDS arangod arangosh arangodump arangoexport arangobackup arangoimport arangorestore)

‎arangod/CMakeLists.txt

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -873,6 +873,13 @@ target_link_libraries(arango_rocksdb arango_indexes)
873873
target_link_libraries(arango_rocksdb arango_storage_engine_common)
874874
target_link_libraries(arango_rocksdb boost_boost)
875875

876+
if (USE_ENTERPRISE)
877+
# this is required for hotbackup. Views need to be flushed.
878+
target_include_directories(arango_rocksdb
879+
PUBLIC ${IRESEARCH_INCLUDE}
880+
)
881+
endif()
882+
876883
target_link_libraries(arango_storage_engine arango_cluster_engine)
877884
target_link_libraries(arango_storage_engine arango_cluster_methods)
878885
target_link_libraries(arango_storage_engine arango_mmfiles)

‎arangod/Cluster/ClusterMethods.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3011,7 +3011,8 @@ arangodb::Result hotRestoreCoordinator(ClusterFeature& feature, VPackSlice const
30113011
using arangodb::methods::VersionResult;
30123012
#ifdef USE_ENTERPRISE
30133013
// Will never be called in community
3014-
if (!RocksDBHotBackup::versionTestRestore(meta._version)) {
3014+
bool autoUpgradeNeeded; // not actually used
3015+
if (!RocksDBHotBackup::versionTestRestore(meta._version, autoUpgradeNeeded)) {
30153016
return arangodb::Result(TRI_ERROR_HOT_RESTORE_INTERNAL,
30163017
"Version mismatch");
30173018
}

‎arangod/IResearch/IResearchLink.cpp

Lines changed: 25 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -109,31 +109,6 @@ struct LinkTrxState final : public arangodb::TransactionState::Cookie {
109109
}
110110
};
111111

112-
////////////////////////////////////////////////////////////////////////////////
113-
/// @brief compute the data path to user for iresearch data store
114-
/// get base path from DatabaseServerFeature (similar to MMFilesEngine)
115-
/// the path is hardcoded to reside under:
116-
/// <DatabasePath>/<IResearchLink::type()>-<link id>
117-
/// similar to the data path calculation for collections
118-
////////////////////////////////////////////////////////////////////////////////
119-
irs::utf8_path getPersistedPath(arangodb::DatabasePathFeature const& dbPathFeature,
120-
arangodb::iresearch::IResearchLink const& link) {
121-
irs::utf8_path dataPath(dbPathFeature.directory());
122-
static const std::string subPath("databases");
123-
static const std::string dbPath("database-");
124-
125-
dataPath /= subPath;
126-
dataPath /= dbPath;
127-
dataPath += std::to_string(link.collection().vocbase().id());
128-
dataPath /= arangodb::iresearch::DATA_SOURCE_TYPE.name();
129-
dataPath += "-";
130-
dataPath += std::to_string(link.collection().id()); // has to be 'id' since this can be a per-shard collection
131-
dataPath += "_";
132-
dataPath += std::to_string(link.id());
133-
134-
return dataPath;
135-
}
136-
137112
////////////////////////////////////////////////////////////////////////////////
138113
/// @brief inserts ArangoDB document into an IResearch data store
139114
////////////////////////////////////////////////////////////////////////////////
@@ -241,6 +216,31 @@ bool readTick(irs::bytes_ref const& payload, TRI_voc_tick_t& tick) noexcept {
241216
namespace arangodb {
242217
namespace iresearch {
243218

219+
////////////////////////////////////////////////////////////////////////////////
220+
/// @brief compute the data path to user for iresearch data store
221+
/// get base path from DatabaseServerFeature (similar to MMFilesEngine)
222+
/// the path is hardcoded to reside under:
223+
/// <DatabasePath>/<IResearchLink::type()>-<link id>
224+
/// similar to the data path calculation for collections
225+
////////////////////////////////////////////////////////////////////////////////
226+
irs::utf8_path getPersistedPath(arangodb::DatabasePathFeature const& dbPathFeature,
227+
arangodb::iresearch::IResearchLink const& link) {
228+
irs::utf8_path dataPath(dbPathFeature.directory());
229+
static const std::string subPath("databases");
230+
static const std::string dbPath("database-");
231+
232+
dataPath /= subPath;
233+
dataPath /= dbPath;
234+
dataPath += std::to_string(link.collection().vocbase().id());
235+
dataPath /= arangodb::iresearch::DATA_SOURCE_TYPE.name();
236+
dataPath += "-";
237+
dataPath += std::to_string(link.collection().id()); // has to be 'id' since this can be a per-shard collection
238+
dataPath += "_";
239+
dataPath += std::to_string(link.id());
240+
241+
return dataPath;
242+
}
243+
244244
IResearchLink::IResearchLink(
245245
TRI_idx_iid_t iid,
246246
LogicalCollection& collection)

‎arangod/IResearch/IResearchLink.h

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,11 @@
2929
#include "store/directory.hpp"
3030
#include "utils/utf8_path.hpp"
3131

32+
#include "Indexes/Index.h"
3233
#include "IResearchLinkMeta.h"
3334
#include "IResearchViewMeta.h"
3435
#include "IResearchVPackComparer.h"
35-
#include "Indexes/Index.h"
36+
#include "RestServer/DatabasePathFeature.h"
3637
#include "Transaction/Status.h"
3738

3839
namespace arangodb {
@@ -333,6 +334,9 @@ class IResearchLink {
333334
bool _createdInRecovery; // link was created based on recovery marker
334335
}; // IResearchLink
335336

337+
irs::utf8_path getPersistedPath(arangodb::DatabasePathFeature const& dbPathFeature,
338+
arangodb::iresearch::IResearchLink const& link);
339+
336340
} // namespace iresearch
337341
} // namespace arangodb
338342

‎arangod/RestServer/UpgradeFeature.cpp

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,13 @@
2828
#include "ApplicationFeatures/SupervisorFeature.h"
2929
#include "Basics/application-exit.h"
3030
#include "Basics/ScopeGuard.h"
31+
#include "Basics/StaticStrings.h"
32+
#include "Cluster/ClusterFeature.h"
3133
#include "Cluster/ClusterFeature.h"
3234
#include "Cluster/ServerState.h"
35+
#ifdef USE_ENTERPRISE
36+
#include "Enterprise/StorageEngine/HotBackupFeature.h"
37+
#endif
3338
#include "FeaturePhases/AqlFeaturePhase.h"
3439
#include "GeneralServer/AuthenticationFeature.h"
3540
#include "Logger/LogMacros.h"
@@ -85,10 +90,8 @@ void UpgradeFeature::collectOptions(std::shared_ptr<ProgramOptions> options) {
8590
extern std::function<int()> * restartAction;
8691

8792
#ifndef _WIN32
88-
static std::string const UPGRADE_ENV = "ARANGODB_UPGRADE_DURING_RESTORE";
89-
9093
static int upgradeRestart() {
91-
unsetenv(UPGRADE_ENV.c_str());
94+
unsetenv(StaticStrings::UpgradeEnvName.c_str());
9295
return 0;
9396
}
9497
#endif
@@ -103,11 +106,15 @@ void UpgradeFeature::validateOptions(std::shared_ptr<ProgramOptions> options) {
103106
// variable at runtime and then does a restore. After the restart (with
104107
// the old data) the database upgrade is run and another restart is
105108
// happening afterwards with the environment variable being cleared.
106-
char* upgrade = getenv(UPGRADE_ENV.c_str());
109+
char* upgrade = getenv(StaticStrings::UpgradeEnvName.c_str());
107110
if (upgrade != nullptr) {
108111
_upgrade = true;
109112
restartAction = new std::function<int()>();
110113
*restartAction = upgradeRestart;
114+
LOG_TOPIC("fdeae", INFO, Logger::STARTUP)
115+
<< "Detected environment variable " << StaticStrings::UpgradeEnvName
116+
<< " with value " << upgrade
117+
<< " will perform database auto-upgrade and immediately restart.";
111118
}
112119
#endif
113120
if (_upgrade && !_upgradeCheck) {
@@ -150,6 +157,11 @@ void UpgradeFeature::validateOptions(std::shared_ptr<ProgramOptions> options) {
150157

151158
DatabaseFeature& database = server().getFeature<DatabaseFeature>();
152159
database.enableUpgrade();
160+
161+
#ifdef USE_ENTERPRISE
162+
HotBackupFeature& hotBackupFeature = server().getFeature<HotBackupFeature>();
163+
hotBackupFeature.forceDisable();
164+
#endif
153165
}
154166

155167
void UpgradeFeature::prepare() {

‎lib/Basics/StaticStrings.cpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -275,3 +275,7 @@ std::string const StaticStrings::RebootId("rebootId");
275275

276276
std::string const StaticStrings::New("new");
277277
std::string const StaticStrings::Old("old");
278+
std::string const StaticStrings::UpgradeEnvName(
279+
"ARANGODB_UPGRADE_DURING_RESTORE");
280+
std::string const StaticStrings::BackupToDeleteName("DIRECTORY_TO_DELETE");
281+
std::string const StaticStrings::BackupSearchToDeleteName("DIRECTORY_TO_DELETE_SEARCH");

‎lib/Basics/StaticStrings.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -251,6 +251,9 @@ class StaticStrings {
251251
static std::string const RebootId;
252252
static std::string const New;
253253
static std::string const Old;
254+
static std::string const UpgradeEnvName;
255+
static std::string const BackupToDeleteName;
256+
static std::string const BackupSearchToDeleteName;
254257
};
255258
} // namespace arangodb
256259

‎tests/RocksDBEngine/HotBackupTest.cpp

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,7 @@ class RocksDBHotBackupRestoreTest : public RocksDBHotBackupRestore {
217217
arangodb::RandomGenerator::initialize(arangodb::RandomGenerator::RandomType::MERSENNE);
218218
}
219219

220+
220221
_id = TRI_GetTempPath();
221222
_id += TRI_DIR_SEPARATOR_CHAR;
222223
_id += "arangotest-";
@@ -330,14 +331,16 @@ class RocksDBHotBackupRestoreTest : public RocksDBHotBackupRestore {
330331
pathname += "backups";
331332
pathname += TRI_DIR_SEPARATOR_CHAR;
332333
pathname += _idRestore;
334+
pathname += TRI_DIR_SEPARATOR_CHAR;
335+
pathname += "engine_rocksdb";
333336
retVal = TRI_CreateRecursiveDirectory(pathname.c_str(), systemError,
334337
systemErrorStr);
335338

336339
EXPECT_EQ(TRI_ERROR_NO_ERROR, retVal);
337340

341+
writeFile(pathname.c_str(), "../META", "{\"version\":\"" ARANGODB_VERSION "\", \"datetime\":\"xxx\", \"id\":\"xxx\"}");
338342
writeFile(pathname.c_str(), "MANIFEST-000003", "manifest info");
339343
writeFile(pathname.c_str(), "CURRENT", "MANIFEST-000003\n");
340-
writeFile(pathname.c_str(), "META", "{\"version\":\"" ARANGODB_VERSION "\", \"datetime\":\"xxx\", \"id\":\"xxx\"}");
341344
writeFile(pathname.c_str(), "IDENTITY", "huh?");
342345
writeFile(pathname.c_str(), "000111.sst", "raw data 1");
343346
writeFile(pathname.c_str(), "000111.sha.e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855.hash", "");
@@ -362,7 +365,7 @@ class RocksDBHotBackupRestoreTest : public RocksDBHotBackupRestore {
362365

363366
/// @brief test
364367
TEST(RocksDBHotBackupRestoreDirectories, test_createRestoringDirectory) {
365-
std::string restoringDir, tempname;
368+
std::string fullRestoringDir, restoringDir, restoringSearchDir, tempname;
366369
bool retBool;
367370

368371
VPackBuilder report;
@@ -371,7 +374,7 @@ TEST(RocksDBHotBackupRestoreDirectories, test_createRestoringDirectory) {
371374
RocksDBHotBackupRestoreTest testee(feature, VPackSlice(), report);
372375
testee.createHotDirectory();
373376

374-
retBool = testee.createRestoringDirectory(restoringDir);
377+
retBool = testee.createRestoringDirectories(fullRestoringDir, restoringDir, restoringSearchDir);
375378

376379
// spot check files in restoring dir
377380
EXPECT_TRUE( retBool );
@@ -391,7 +394,8 @@ TEST(RocksDBHotBackupRestoreDirectories, test_createRestoringDirectory) {
391394
EXPECT_TRUE( TRI_IsRegularFile(tempname.c_str()) ); // looks same as hard link
392395

393396
// verify still present in originating dir
394-
restoringDir = testee.rebuildPath(testee.getDirectoryRestore());
397+
restoringDir = testee.rebuildPath(testee.getDirectoryRestore() +
398+
TRI_DIR_SEPARATOR_CHAR + "engine_rocksdb");
395399
EXPECT_TRUE( TRI_ExistsFile(restoringDir.c_str()) );
396400
EXPECT_TRUE( TRI_IsDirectory(restoringDir.c_str()) );
397401
tempname = restoringDir + TRI_DIR_SEPARATOR_CHAR + "MANIFEST-000003";

0 commit comments

Comments
 (0)
0