8000 Fix OpenMP failure on ARM by jbajic · Pull Request #21772 · arangodb/arangodb · GitHub
[go: up one dir, main page]

Skip to content

Fix OpenMP failure on ARM #21772

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 50 commits into
base: devel 8000
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
50 commits
Select commit Hold shift + click to select a range
6e76df9
Reduce OMP threading
jbajic Apr 3, 2025
dcae2c5
Disable kmp affinity
jbajic Apr 3, 2025
9627409
Disable on openmp
jbajic Apr 3, 2025
bcab65f
Disable KMP_AFFINITY and LIBOMP_AFFINITY
jbajic Apr 4, 2025
9699c84
Reduce omp number of threads
jbajic Apr 4, 2025
8a6e620
Print cpu stats
jbajic Apr 4, 2025
d5b8823
Use KMP_DEVICE_THREAD_LIMIT
jbajic Apr 4, 2025
bf0c5a1
More verbose
jbajic Apr 4, 2025
794011e
Check restore env
jbajic Apr 14, 2025
d565324
Update env variables
jbajic Apr 14, 2025
f5cec0b
Merge branch ' 8000 ;devel' into fix/arm-failure
jbajic Apr 14, 2025
76ba626
Enable tests
jbajic Apr 14, 2025
b884a99
Enable vector index tests
jbajic Apr 14, 2025
9d81206
Clear string
jbajic Apr 14, 2025
02fd177
Print omp variables
jbajic Apr 14, 2025
83d1504
Fix dependency
jbajic Apr 14, 2025
cc938a4
Add flags only for arm
jbajic Apr 15, 2025
d03933e
Fix arch var
jbajic Apr 15, 2025
1870ad1
Refactor
jbajic Apr 16, 2025
8661867
Merge branch 'devel' into fix/arm-failure
jbajic May 12, 2025
8360527
Fix
jbajic May 12, 2025
abfa5d5
Disable kmp
jbajic May 12, 2025
0f0d8d1
Try testing libomp
jbajic May 13, 2025
4c941ae
Clone arangod for testing
jbajic May 13, 2025
13457e1
Clone arango
jbajic May 13, 2025
14e555b
Fix path
jbajic May 13, 2025
5741da6
Fix
jbajic May 13, 2025
25468b4
Remove verbose
jbajic May 13, 2025
045c48f
Remove test changes
jbajic May 13, 2025
141d9b5
Check env
jbajic May 13, 2025
309ee83
Merge branch 'devel' into fix/arm-failure
jbajic May 13, 2025
9530ce1
Test env
jbajic May 14, 2025
0314bac
Always set env
jbajic May 15, 2025
7674b4c
Disable dynamic
jbajic May 15, 2025
9de436f
Add preload
jbajic May 15, 2025
141d338
Define cpu topology
jbajic May 19, 2025
74d1034
Test out sample
jbajic May 19, 2025
f4f6231
Try out libgomp
jbajic May 19, 2025
109b270
Use libgomp
jbajic May 20, 2025
dda4ffb
Enable flags
jbajic May 20, 2025
920e989
Fix arm
jbajic May 20, 2025
95230ad
Fix typo
jbajic May 20, 2025
397f315
Fix
jbajic May 20, 2025
6c958c0
Clean CMakeLists
jbajic May 20, 2025
47d4e4e
Remove redundant
jbajic May 20, 2025
1e698ea
Merge branch 'devel' into bug-fix/fix-openmp-arm-failure
jbajic May 20, 2025
62e81f5
Remove dumpEnv
jbajic May 20, 2025
e8b6613
Remove print
jbajic May 20, 2025
bf6df78
Add comment
jbajic May 20, 2025
9fe9e3d
Merge branch 'devel' into bug-fix/fix-openmp-arm-failure
jbajic May 20, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 12 additions & 9 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1034,26 +1034,29 @@ if(STATIC_EXECUTABLES)
if(ARCH_AMD64)
set(LAPACK_LIBRARIES "/usr/lib/x86_64-linux-gnu/lapack/liblapack.a" CACHE PATH "LAPACK static library path")
set(BLAS_LIBRARIES "/usr/lib/x86_64-linux-gnu/libblas.a" CACHE PATH "BLAS static library path")
set(OpenMP_gomp_LIBRARY "/usr/lib/gcc/x86_64-linux-gnu/13/libgomp.a")
else()
set(LAPACK_LIBRARIES "/usr/lib/aarch64-linux-gnu/lapack/liblapack.a" CACHE PATH "LAPACK static library path")
set(BLAS_LIBRARIES "/usr/lib/aarch64-linux-gnu/libblas.a" CACHE PATH "BLAS static library path")
set(OpenMP_gomp_LIBRARY "/usr/lib/gcc/aarch64-linux-gnu/13/libgomp.a")
endif()
message("LAPACK: ${LAPACK_LIBRARIES} and BLAS: to ${BLAS_LIBRARIES}")

find_library(LIBOMP omp PATHS /opt/omp NO_DEFAULT_PATH)

if(LIBOMP)
message(STATUS "Using hardcoded paths '-fopenmp=libomp -L/opt/omp -llapack -lgfortran'")
else()
message(FATAL_ERROR "libomp.a not found in /opt/omp")
endif()
# Setting this is necessary because we want to bind to gcc static library
# since the LLVM OpenMP implementation fails in our CI
# This is related to this issue (https://github.com/llvm/llvm-project/issues/137136)
set(OpenMP_C_FLAGS "-fopenmp=libgomp")
set(OpenMP_CXX_FLAGS "-fopenmp=libgomp")
set(OpenMP_C_LIB_NAMES "gomp;pthread")
set(OpenMP_CXX_LIB_NAMES "gomp;pthread")
set(OpenMP_pthread_LIBRARY "pthread")

# TODO avoid hard coded paths
set(FAISS_EXE_LINKER_FLAGS "-fopenmp=libomp -L/opt/omp -llapack -lgfortran")
set(FAISS_EXE_LINKER_FLAGS "-fopenmp=libgomp -L/opt/omp -llapack -lgfortran")
else()
set(BLA_STATIC Off)
set(BLA_VENDOR OpenBLAS)
find_package(OpenMP)
find_package(OpenMP REQUIRED)
find_package(LAPACK)
find_package(BLAS)

Expand Down
9 changes: 0 additions & 9 deletions js/client/modules/@arangodb/testutils/client-tools.js
Original file line number Diff line number Diff line change
Expand Up @@ -551,15 +551,6 @@ function rtaMakedata(options, instanceManager, writeReadClean, msg, logFile, mor
if (addArgs !== undefined) {
args = Object.assign(args, addArgs);
}
// TODO: vector index broken on circleci-ARM
if (versionHas("arm")) {
let skipOffset = moreargv.findIndex(i => {return i === '--skip';});
if (skipOffset >= 0) {
moreargv[skipOffset + 1] += ',107';
} else {
moreargv = ['--skip', '107_'];
}
}

let argv = toArgv(args);
argv = argv.concat(['--', options.makedataDB],
Expand Down
8 changes: 0 additions & 8 deletions lib/ApplicationFeatures/ProcessEnvironmentFeature.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -22,18 +22,11 @@
////////////////////////////////////////////////////////////////////////////////

#include "ApplicationFeatures/ProcessEnvironmentFeature.h"
#include "Basics/ArangoGlobalContext.h"
#include "CrashHandler/CrashHandler.h"
#include "Basics/FileUtils.h"
#include "Basics/StringUtils.h"
#include "Basics/Thread.h"
#include "Basics/files.h"
#include "Logger/LogMacros.h"
#include "Logger/Logger.h"
#include "Logger/LoggerStream.h"
#include "ProgramOptions/Parameters.h"
#include "ProgramOptions/ProgramOptions.h"
#include "ProgramOptions/Section.h"

using namespace arangodb::options;
extern char** environ;
Expand All @@ -51,7 +44,6 @@ void ProcessEnvironmentFeature::collectOptions(

void ProcessEnvironmentFeature::prepare() {
if (_dumpEnv) {
// Maybe Replace with TRI_ASSERT, I leave that to you
if (environ == nullptr) {
return;
}
Expand Down
2 changes: 1 addition & 1 deletion lib/ApplicationFeatures/ProcessEnvironmentFeature.h
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ class ProcessEnvironmentFeature final
void prepare() override final;

private:
bool _dumpEnv;
bool _dumpEnv{false};
};
#endif
} // namespace arangodb
Original file line number Diff line number Diff line change
Expand Up @@ -263,8 +263,7 @@ function VectorIndexInvalidDefinitionInAgencyTest() {
};
}

if (!versionHas("arm")) {
jsunity.run(VectorIndexCorrectDefinitionInAgencyTest);
jsunity.run(VectorIndexInvalidDefinitionInAgencyTest);
}
jsunity.run(VectorIndexCorrectDefinitionInAgencyTest);
jsunity.run(VectorIndexInvalidDefinitionInAgencyTest);

return jsunity.done();
7 changes: 3 additions & 4 deletions tests/js/client/aql/vector/aql-vector-create-and-remove.js
A3E2
Original file line number Diff line number Diff line change
Expand Up @@ -321,8 +321,7 @@ function VectorIndexTestCreationWithVectors() {
}


if (!versionHas("arm")) {
jsunity.run(VectorIndexCreateAndRemoveTestSuite);
jsunity.run(VectorIndexTestCreationWithVectors);
}
jsunity.run(VectorIndexCreateAndRemoveTestSuite);
jsunity.run(VectorIndexTestCreationWithVectors);

return jsunity.done();
9 changes: 4 additions & 5 deletions tests/js/client/aql/vector/aql-vector-full-count.js
Original file line number Diff line number Diff line change
Expand Up @@ -417,9 +417,8 @@ function VectorIndexFullCountCollectionWithSmallAmountOfDocs() {
};
}

if (!versionHas("arm")) {
jsunity.run(VectorIndexFullCountTestSuite);
jsunity.run(VectorIndexFullCountWithNotEnoughNListsTestSuite);
jsunity.run(VectorIndexFullCountCollectionWithSmallAmountOfDocs);
}
jsunity.run(VectorIndexFullCountTestSuite);
jsunity.run(VectorIndexFullCountWithNotEnoughNListsTestSuite);
jsunity.run(VectorIndexFullCountCollectionWithSmallAmountOfDocs);

return jsunity.done();
5 changes: 2 additions & 3 deletions tests/js/client/aql/vector/aql-vector-nprobe.js
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,6 @@ function VectorIndexL2NprobeTestSuite() {
};
}

if (!versionHas("arm")) {
jsunity.run(VectorIndexL2NprobeTestSuite);
}
jsunity.run(VectorIndexL2NprobeTestSuite);

return jsunity.done();
9 changes: 4 additions & 5 deletions tests/js/client/aql/vector/aql-vector.js
Original file line number Diff line number Diff line change
Expand Up @@ -884,10 +884,9 @@ function MultipleVectorIndexesOnField() {
};
}

if (!versionHas("arm")) {
jsunity.run(VectorIndexL2TestSuite);
jsunity.run(VectorIndexCosineTestSuite);
jsunity.run(MultipleVectorIndexesOnField);
}
jsunity.run(VectorIndexL2TestSuite);
jsunity.run(VectorIndexCosineTestSuite);
jsunity.run(MultipleVectorIndexesOnField);

return jsunity.done();

0