8000 add progress reporting for RocksDB .sst file checksum computation (#2… · 0ArtemBabaev/arangodb@fdc3655 · GitHub
[go: up one dir, main page]
Skip to content

Commit fdc3655

Browse files
jsteemannjsteemann
authored
add progress reporting for RocksDB .sst file checksum computation (arangodb#20640)
* add progress reporting for RocksDB .sst file checksum computation * fix logic error
1 parent 706fa5e commit fdc3655

File tree

4 files changed

+49
-8
lines changed

4 files changed

+49
-8
lines changed

‎CHANGELOG

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
devel
22
-----
33

4+
* Add startup progress reporting for RocksDB .sst file checksum computation.
5+
46
* Upgrade OpenSSL to 3.2.1.
57

68
* Remove arangosync binary.

‎arangod/RocksDBEngine/RocksDBChecksumEnv.cpp

Lines changed: 45 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,10 @@ bool ChecksumHelper::isBlobFile(std::string_view fileName) noexcept {
8686
return fileName.ends_with(".blob");
8787
}
8888

89+
bool ChecksumHelper::isHashFile(std::string_view fileName) noexcept {
90+
return fileName.ends_with(".hash");
91+
}
92+
8993
bool ChecksumHelper::writeShaFile(std::string const& fileName,
9094
std::string const& checksum) {
9195
TRI_ASSERT(isSstFile(fileName) || isBlobFile(fileName));
@@ -154,26 +158,25 @@ void ChecksumHelper::checkMissingShaFiles() {
154158

155159
// check file extension
156160
auto isInteresting = [](std::string_view name) noexcept -> bool {
157-
return name.ends_with(".sst") || name.ends_with(".blob") ||
158-
name.ends_with(".hash");
161+
return isSstFile(name) || isBlobFile(name) || isHashFile(name);
159162
};
160163

161164
if (!isInteresting(lhs) || !isInteresting(rhs)) {
162165
// we are dealing with a non-interesting file type
163166
return lhs < rhs;
164167
}
165168

166-
if (lhs.ends_with(".hash")) {
169+
if (isHashFile(lhs)) {
167170
// cannot have 2 hash files for the same prefix
168-
TRI_ASSERT(!rhs.ends_with(".hash"));
171+
TRI_ASSERT(!isHashFile(rhs));
169172

170173
// prefixes of lhs and rhs are identical - .hash files should be
171174
// sorted first (before .sst or .blob files)
172175
return true;
173176
}
174-
if (rhs.ends_with(".hash")) {
177+
if (isHashFile(rhs)) {
175178
// cannot have 2 hash files for the same prefix
176-
TRI_ASSERT(!lhs.ends_with(".hash"));
179+
TRI_ASSERT(!isHashFile(lhs));
177180

178181
// prefixes of lhs and rhs are identical - .hash files should be
179182
// sorted first (before .sst or .blob files)
@@ -185,6 +188,9 @@ void ChecksumHelper::checkMissingShaFiles() {
185188
return lhs < rhs;
186189
});
187190

191+
// input files for which we need to produce hash files
192+
std::vector<std::string> toProduce;
193+
188194
for (auto it = fileList.begin(); it != fileList.end(); ++it) {
189195
if (it->size() < 5) {
190196
// filename is too short and does not matter
@@ -229,7 +235,16 @@ void ChecksumHelper::checkMissingShaFiles() {
229235
} else if (isSstFile(*it) || isBlobFile(*it)) {
230236
// we have a .sst or .blob file which was not preceeded by a .hash file.
231237
// this means we need to recalculate the sha hash for it!
232-
std::string tempPath = basics::FileUtils::buildFilename(_rootPath, *it);
238+
toProduce.emplace_back(basics::FileUtils::buildFilename(_rootPath, *it));
239+
}
240+
}
241+
242+
if (!toProduce.empty()) {
243+
LOG_TOPIC("ff71d", INFO, arangodb::Logger::ENGINES)
244+
<< "calculating SHA256 checksums for " << toProduce.size()
245+
<< " RocksDB .sst file(s)";
246+
size_t produced = 0;
247+
for (auto const& tempPath : toProduce) {
233248
LOG_TOPIC("d6c86", DEBUG, arangodb::Logger::ENGINES)
234249
<< "checkMissingShaFiles: Computing checksum for " << tempPath;
235250
auto checksumCalc = ChecksumCalculator();
@@ -241,6 +256,29 @@ void ChecksumHelper::checkMissingShaFiles() {
241256
checksumCalc.computeFinalChecksum();
242257
writeShaFile(tempPath, checksumCalc.getChecksum());
243258
}
259+
260+
produced++;
261+
// progress reporting - we are only interested in very rough progress
262+
// so that we don't spam that startup log too much. we intentionally
263+
// report only every 100 .sst files, so in most restart situations
264+
// there will be no progress reporting. progress reporting will become
265+
// visible however if there are 100s or 1000s of hashes to compute.
266+
// this situation should only happen when upgrading from Community
267+
// Edition to Enterprise Edition or so.
268+
if (produced != toProduce.size() && (produced % 100 == 0)) {
269+
int progress =
270+
static_cast<int>(static_cast<double>(produced) /
271+
static_cast<double>(toProduce.size()) * 100.0);
272+
LOG_TOPIC("cf86b", INFO, arangodb::Logger::ENGINES)
273+
<< "calculated " << produced << "/" << toProduce.size()
274+
<< " checksums (" << progress << "% of files)...";
275+
}
276+
}
277+
278+
if (toProduce.size() >= 10) {
279+
// only report end if there was some noteworthy amount of work to do
280+
LOG_TOPIC("96bbd", INFO, arangodb::Logger::ENGINES)
281+
<< "finished calculating SHA256 checksums for RocksDB .sst files";
244282
}
245283
}
246284
}

‎arangod/RocksDBEngine/RocksDBChecksumEnv.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ class ChecksumHelper {
6060

6161
[[nodiscard]] static bool isSstFile(std::string_view fileName) noexcept;
6262
[[nodiscard]] static bool isBlobFile(std::string_view fileName) noexcept;
63+
[[nodiscard]] static bool isHashFile(std::string_view fileName) noexcept;
6364

6465
void checkMissingShaFiles();
6566

‎lib/Basics/files.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1103,7 +1103,7 @@ bool TRI_ProcessFile(
< 6084 code>11031103

11041104
auto guard = scopeGuard([&fd]() noexcept { TRI_CLOSE(fd); });
11051105

1106-
char buffer[4096];
1106+
char buffer[16384];
11071107

11081108
while (true) {
11091109
TRI_read_return_t n = TRI_READ(fd, &buffer[0], sizeof(buffer));

0 commit comments

Comments
 (0)
0