From 5f8692e8eab657a5cab09cf8fef6fec7e860ee4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Plewa?= Date: Tue, 2 Jul 2024 13:00:40 +0200 Subject: [PATCH 01/52] optimize qemu workflow --- .github/workflows/pr_push.yml | 6 +-- .github/workflows/qemu.yml | 69 +++++++++++++++-------------------- scripts/qemu/run-build.sh | 23 +----------- scripts/qemu/run-tests.sh | 26 +++++++++++++ scripts/qemu/start_qemu.sh | 31 ++++++++++++++++ 5 files changed, 91 insertions(+), 64 deletions(-) create mode 100755 scripts/qemu/run-tests.sh create mode 100755 scripts/qemu/start_qemu.sh diff --git a/.github/workflows/pr_push.yml b/.github/workflows/pr_push.yml index b69da6b89..628f773f9 100644 --- a/.github/workflows/pr_push.yml +++ b/.github/workflows/pr_push.yml @@ -200,6 +200,9 @@ jobs: Sanitizers: needs: [Spellcheck, FastBuild, CodeStyle] uses: ./.github/workflows/sanitizers.yml + Qemu: + needs: [Spellcheck, FastBuild, CodeStyle] + uses: ./.github/workflows/qemu.yml Benchmarks: needs: [Build] uses: ./.github/workflows/benchmarks.yml @@ -209,9 +212,6 @@ jobs: GPU: needs: [Build] uses: ./.github/workflows/gpu.yml - Qemu: - needs: [Build] - uses: ./.github/workflows/qemu.yml Valgrind: needs: [Build] uses: ./.github/workflows/valgrind.yml diff --git a/.github/workflows/qemu.yml b/.github/workflows/qemu.yml index c7a92feae..946a7edd2 100644 --- a/.github/workflows/qemu.yml +++ b/.github/workflows/qemu.yml @@ -4,7 +4,7 @@ name: Qemu on: workflow_call env: - CI_BRANCH : "${{ github.head_ref || github.ref_name }}" + CI_BRANCH: "${{ github.head_ref || github.ref_name }}" permissions: contents: read @@ -13,19 +13,6 @@ jobs: qemu-build: name: Qemu runs-on: ubuntu-22.04 - strategy: - matrix: - config: [{name: 'default', hmat: 'on'}, - { name: 'sock_2_var1', hmat: 'off'}, - { name: 'sock_2_var1_hmat', hmat: 'on'}, - { name: 'sock_2_var2', hmat: 'off'}, - { name: 'sock_2_var2_hmat', hmat: 'on'}, - { name: 'sock_2_var3', hmat: 'off'}, - { name: 'sock_2_var3_hmat', hmat: 'on'}, - { name: 'sock_4_var1', hmat: 'off'}, - { name: 'sock_4_var1_hmat', hmat: 'on'}, - { name: 'sock_4_var2', hmat: 'off'}, - { name: 'sock_4_var2_hmat', hmat: 'on'}] steps: - name: Checkout @@ -83,33 +70,10 @@ jobs: run: wget https://cloud-images.ubuntu.com/releases/lunar/release/ubuntu-23.04-server-cloudimg-amd64.img - name: Resize image run: qemu-img resize ./ubuntu-23.04-server-cloudimg-amd64.img +4G - - name: Print qemu args - run: | - # notice: should be sedded, but it hides status - python3 scripts/qemu/qemu_config.py scripts/qemu/configs/${{matrix.config.name}}.xml - - name: Run qemu - run: | - sudo qemu-system-x86_64 \ - -drive file=./ubuntu-23.04-server-cloudimg-amd64.img,format=qcow2,index=0,media=disk,id=hd \ - -cdrom ./ubuntu-cloud-init.iso \ - -machine q35,usb=off,hmat=${{matrix.config.hmat}} \ - -enable-kvm \ - -net nic -net user,hostfwd=tcp::2222-:22 \ - $(echo `python3 scripts/qemu/qemu_config.py scripts/qemu/configs/${{matrix.config.name}}.xml | sed s/''\''/'/g`) \ - -daemonize -display none - - name: Run ssh keyscan - run: | - set +e - ssh-keyscan -p 2222 -H 127.0.0.1 >> ~/.ssh/known_hosts - while [ $? -ne 0 ] - do - echo "Trying to connect..." - ps -aux | grep qemu - sleep 5 - ssh-keyscan -p 2222 -H 127.0.0.1 >> ~/.ssh/known_hosts - done - - name: Run build on qemu + - name: Build run: | + scripts/qemu/start_qemu.sh scripts/qemu/configs/default.xml + if [ ${{ github.event_name }} = 'pull_request' ]; then CI_REPO="${{ github.event.pull_request.head.repo.full_name }}" else @@ -117,4 +81,29 @@ jobs: fi scp -P 2222 ${{github.workspace}}/scripts/qemu/run-build.sh cxltest@127.0.0.1:/home/cxltest + scp -P 2222 ${{github.workspace}}/scripts/qemu/run-tests.sh cxltest@127.0.0.1:/home/cxltest ssh cxltest@127.0.0.1 -p 2222 -t "bash /home/cxltest/run-build.sh https://github.com/$CI_REPO ${{env.CI_BRANCH}}" + + ssh cxltest@127.0.0.1 -p 2222 -t "sudo shutdown -h now" + + - name: Run tests + run: | + for config_file in scripts/qemu/configs/*.xml; do + config_name=$(basename $config_file .xml) + + echo testing $config_name + while ps -aux | grep qemu-system-x86_64 | grep -q -v grep; do + echo "Waiting for QEMU to shut down..." + sleep 5 + done + scripts/qemu/start_qemu.sh $config_file + + if [ ${{ github.event_name }} = 'pull_request' ]; then + CI_REPO="${{ github.event.pull_request.head.repo.full_name }}" + else + CI_REPO="$GITHUB_REPOSITORY" + fi + + ssh cxltest@127.0.0.1 -p 2222 -t "bash /home/cxltest/run-tests.sh" + ssh cxltest@127.0.0.1 -p 2222 -t "sudo shutdown -h now" + done diff --git a/scripts/qemu/run-build.sh b/scripts/qemu/run-build.sh index 2abee9766..91c2e4f61 100755 --- a/scripts/qemu/run-build.sh +++ b/scripts/qemu/run-build.sh @@ -3,7 +3,6 @@ # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -set -x set -e repo=$1 @@ -12,11 +11,6 @@ branch=$2 echo password | sudo -Sk apt update echo password | sudo -Sk apt install -y git cmake gcc g++ numactl libnuma-dev libhwloc-dev libjemalloc-dev libtbb-dev pkg-config valgrind hwloc -# Set ptrace value for IPC test -echo password | sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope" - -numactl -H - git clone $repo umf cd umf git checkout $branch @@ -31,20 +25,7 @@ cmake .. \ -DUMF_DEVELOPER_MODE=ON \ -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON \ -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON \ - -DUMF_BUILD_EXAMPLES=ON + -DUMF_BUILD_EXAMPLES=ON \ + -DUMF_TESTS_FAIL_ON_SKIP=ON make -j $(nproc) - -# Drop caches, restores free memory on NUMA nodes -echo password | sudo sync; -echo password | sudo sh -c "/usr/bin/echo 3 > /proc/sys/vm/drop_caches" - -ctest --verbose - -# run tests bound to a numa node -numactl -N 0 ctest --output-on-failure -numactl -N 1 ctest --output-on-failure - -# run tests under valgrind -echo "Running tests under valgrind memcheck ..." -../test/test_valgrind.sh .. . memcheck diff --git a/scripts/qemu/run-tests.sh b/scripts/qemu/run-tests.sh new file mode 100755 index 000000000..00fdfb8fd --- /dev/null +++ b/scripts/qemu/run-tests.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set -e + +# Drop caches, restores free memory on NUMA nodes +echo password | sudo sync; +echo password | sudo sh -c "/usr/bin/echo 3 > /proc/sys/vm/drop_caches" +# Set ptrace value for IPC test +echo password | sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope" + +numactl -H + +cd umf/build +ctest --verbose + +# run tests bound to a numa node +numactl -N 0 ctest --output-on-failure +numactl -N 1 ctest --output-on-failure + +# run tests under valgrind +echo "Running tests under valgrind memcheck ..." +../test/test_valgrind.sh .. . memcheck + diff --git a/scripts/qemu/start_qemu.sh b/scripts/qemu/start_qemu.sh new file mode 100755 index 000000000..0962dd98a --- /dev/null +++ b/scripts/qemu/start_qemu.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set -x +set -e + +config_file=$1 + +python3 scripts/qemu/qemu_config.py $config_file + +if grep -q '' "$config_file"; then + hmat="on" +else + hmat="off" +fi + +sudo qemu-system-x86_64 \ + -drive file=./ubuntu-23.04-server-cloudimg-amd64.img,format=qcow2,index=0,media=disk,id=hd \ + -cdrom ./ubuntu-cloud-init.iso \ + -machine q35,usb=off,hmat=$hmat \ + -enable-kvm \ + -net nic -net user,hostfwd=tcp::2222-:22 \ + $(python3 scripts/qemu/qemu_config.py $config_file | sed s/''\''/'/g) \ + -daemonize -display none + +until ssh-keyscan -p 2222 -H 127.0.0.1 >> ~/.ssh/known_hosts 2>/dev/null; do + echo "Waiting for SSH..." + sleep 1 +done From 5c3b4da690491ac18cdbfdd9934e74e698ea2758 Mon Sep 17 00:00:00 2001 From: "Vinogradov, Sergei" Date: Wed, 3 Jul 2024 19:04:58 +0200 Subject: [PATCH 02/52] Add umfPoolGetIPCHandleSize API --- include/umf/ipc.h | 8 ++++++++ src/ipc.c | 44 ++++++++++++++++++++++++++++++++++++-------- src/libumf.def.in | 1 + src/libumf.map | 1 + test/ipcFixtures.hpp | 7 +++++++ 5 files changed, 53 insertions(+), 8 deletions(-) diff --git a/include/umf/ipc.h b/include/umf/ipc.h index d7bfe597a..ffe38bfc8 100644 --- a/include/umf/ipc.h +++ b/include/umf/ipc.h @@ -19,6 +19,14 @@ extern "C" { typedef struct umf_ipc_data_t *umf_ipc_handle_t; +/// +/// @brief Returns the size of IPC handles for the specified pool. +/// @param hPool [in] Pool handle +/// @param size [out] size of IPC handle in bytes. +/// @return UMF_RESULT_SUCCESS on success or appropriate error code on failure. +umf_result_t umfPoolGetIPCHandleSize(umf_memory_pool_handle_t hPool, + size_t *size); + /// /// @brief Creates an IPC handle for the specified UMF allocation. /// @param ptr pointer to the allocated memory. diff --git a/src/ipc.c b/src/ipc.c index d5e786177..62845148e 100644 --- a/src/ipc.c +++ b/src/ipc.c @@ -19,6 +19,36 @@ #include "utils_common.h" #include "utils_log.h" +umf_result_t umfPoolGetIPCHandleSize(umf_memory_pool_handle_t hPool, + size_t *size) { + umf_result_t ret = UMF_RESULT_SUCCESS; + if (hPool == NULL) { + LOG_ERR("pool handle is NULL."); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + if (size == NULL) { + LOG_ERR("size is NULL."); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + + // We cannot use umfPoolGetMemoryProvider function because it returns + // upstream provider but we need tracking one + umf_memory_provider_handle_t hProvider = hPool->provider; + assert(hProvider); + + size_t providerIPCHandleSize; + ret = umfMemoryProviderGetIPCHandleSize(hProvider, &providerIPCHandleSize); + if (ret != UMF_RESULT_SUCCESS) { + LOG_ERR("cannot get IPC handle size."); + return ret; + } + + *size = sizeof(umf_ipc_data_t) + providerIPCHandleSize; + + return ret; +} + umf_result_t umfGetIPCHandle(const void *ptr, umf_ipc_handle_t *umfIPCHandle, size_t *size) { size_t ipcHandleSize = 0; @@ -29,25 +59,23 @@ umf_result_t umfGetIPCHandle(const void *ptr, umf_ipc_handle_t *umfIPCHandle, return ret; } - // We cannot use umfPoolGetMemoryProvider function because it returns - // upstream provider but we need tracking one - umf_memory_provider_handle_t provider = allocInfo.pool->provider; - assert(provider); - - size_t providerIPCHandleSize; - ret = umfMemoryProviderGetIPCHandleSize(provider, &providerIPCHandleSize); + ret = umfPoolGetIPCHandleSize(allocInfo.pool, &ipcHandleSize); if (ret != UMF_RESULT_SUCCESS) { LOG_ERR("cannot get IPC handle size."); return ret; } - ipcHandleSize = sizeof(umf_ipc_data_t) + providerIPCHandleSize; umf_ipc_data_t *ipcData = umf_ba_global_alloc(ipcHandleSize); if (!ipcData) { LOG_ERR("failed to allocate ipcData"); return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; } + // We cannot use umfPoolGetMemoryProvider function because it returns + // upstream provider but we need tracking one + umf_memory_provider_handle_t provider = allocInfo.pool->provider; + assert(provider); + ret = umfMemoryProviderGetIPCHandle(provider, allocInfo.base, allocInfo.baseSize, (void *)ipcData->providerIpcData); diff --git a/src/libumf.def.in b/src/libumf.def.in index 6d8ac3d03..8ee99e024 100644 --- a/src/libumf.def.in +++ b/src/libumf.def.in @@ -50,6 +50,7 @@ EXPORTS umfPoolCreateFromMemspace umfPoolDestroy umfPoolFree + umfPoolGetIPCHandleSize umfPoolGetLastAllocationError umfPoolGetMemoryProvider umfPoolMalloc diff --git a/src/libumf.map b/src/libumf.map index 68448daaf..cb09cdb94 100644 --- a/src/libumf.map +++ b/src/libumf.map @@ -50,6 +50,7 @@ UMF_1.0 { umfPoolCreateFromMemspace; umfPoolDestroy; umfPoolFree; + umfPoolGetIPCHandleSize; umfPoolGetLastAllocationError; umfPoolGetMemoryProvider; umfPoolMalloc; diff --git a/test/ipcFixtures.hpp b/test/ipcFixtures.hpp index 090761224..1eb7865e3 100644 --- a/test/ipcFixtures.hpp +++ b/test/ipcFixtures.hpp @@ -112,6 +112,13 @@ struct umfIpcTest : umf_test::test, MemoryAccessor *memAccessor = nullptr; }; +TEST_P(umfIpcTest, GetIPCHandleSize) { + size_t size = 0; + umf_result_t ret = umfPoolGetIPCHandleSize(pool.get(), &size); + EXPECT_EQ(ret, UMF_RESULT_SUCCESS); + EXPECT_GT(size, 0); +} + TEST_P(umfIpcTest, BasicFlow) { constexpr size_t SIZE = 100; std::vector expected_data(SIZE); From 47d6ea1dc7036eb754e871b47115f20535420cce Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Fri, 5 Jul 2024 09:43:35 +0200 Subject: [PATCH 03/52] Run fast builds when Spellcheck and CodeStyle succeed Signed-off-by: Lukasz Dorau --- .github/workflows/fast.yml | 135 +++++++++++++++++++++++++++++++++ .github/workflows/pr_push.yml | 137 ++-------------------------------- 2 files changed, 142 insertions(+), 130 deletions(-) create mode 100644 .github/workflows/fast.yml diff --git a/.github/workflows/fast.yml b/.github/workflows/fast.yml new file mode 100644 index 000000000..9674c5820 --- /dev/null +++ b/.github/workflows/fast.yml @@ -0,0 +1,135 @@ +# Fast builds +name: FastBuild + +on: workflow_call + +permissions: + contents: read + +jobs: + FastBuild: + name: Fast builds + env: + VCPKG_PATH: "${{github.workspace}}/build/vcpkg/packages/hwloc_x64-windows;${{github.workspace}}/build/vcpkg/packages/tbb_x64-windows;${{github.workspace}}/build/vcpkg/packages/jemalloc_x64-windows" + strategy: + matrix: + include: + - os: windows-latest + disjoint: 'OFF' + build_tests: 'ON' + simple_cmake: 'OFF' + # pure C build (Windows) + - os: windows-latest + disjoint: 'OFF' + # Tests' building is off for a pure C build + build_tests: 'OFF' + simple_cmake: 'OFF' + - os: ubuntu-latest + disjoint: 'ON' + build_tests: 'ON' + # Windows doesn't recognize 'CMAKE_BUILD_TYPE', it uses '--config' param in build command + extra_build_options: '-DCMAKE_BUILD_TYPE=Release -DUMF_BUILD_BENCHMARKS=ON -DUMF_BUILD_BENCHMARKS_MT=ON' + simple_cmake: 'OFF' + # pure C build (Linux) + - os: ubuntu-latest + disjoint: 'OFF' + # Windows doesn't recognize 'CMAKE_BUILD_TYPE', it uses '--config' param in build command + # Tests' building is off for a pure C build + build_tests: 'OFF' + extra_build_options: '-DCMAKE_BUILD_TYPE=Release -DUMF_BUILD_BENCHMARKS=ON' + simple_cmake: 'OFF' + # simplest CMake on ubuntu-latest + - os: ubuntu-latest + disjoint: 'OFF' + build_tests: 'ON' + extra_build_options: '-DCMAKE_BUILD_TYPE=Release' + simple_cmake: 'ON' + # simplest CMake ubuntu-20.04 + - os: ubuntu-20.04 + disjoint: 'OFF' + build_tests: 'ON' + extra_build_options: '-DCMAKE_BUILD_TYPE=Release' + simple_cmake: 'ON' + runs-on: ${{matrix.os}} + + steps: + - name: Checkout repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Initialize vcpkg + if: matrix.os == 'windows-latest' + uses: lukka/run-vcpkg@5e0cab206a5ea620130caf672fce3e4a6b5666a1 # v11.5 + with: + vcpkgGitCommitId: 3dd44b931481d7a8e9ba412621fa810232b66289 + vcpkgDirectory: ${{github.workspace}}/build/vcpkg + vcpkgJsonGlob: '**/vcpkg.json' + + - name: Install dependencies + if: matrix.os == 'windows-latest' + run: vcpkg install + shell: pwsh # Specifies PowerShell as the shell for running the script. + + - name: Install apt packages (ubuntu-latest) + if: matrix.os == 'ubuntu-latest' + run: | + sudo apt-get update + sudo apt-get install -y cmake libjemalloc-dev libhwloc-dev libnuma-dev libtbb-dev + + - name: Install apt packages (ubuntu-20.04) + if: matrix.os == 'ubuntu-20.04' + run: | + sudo apt-get update + sudo apt-get install -y cmake libjemalloc-dev libnuma-dev libtbb-dev + .github/scripts/install_hwloc.sh # install hwloc-2.3.0 instead of hwloc-2.1.0 present in the OS package + + - name: Set ptrace value for IPC test (on Linux only) + if: ${{ matrix.os == 'ubuntu-latest' || matrix.os == 'ubuntu-20.04' }} + run: sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope" + + - name: Configure CMake + if: matrix.simple_cmake == 'OFF' + run: > + cmake + -B ${{github.workspace}}/build + -DCMAKE_PREFIX_PATH="${{env.VCPKG_PATH}}" + -DUMF_FORMAT_CODE_STYLE=OFF + -DUMF_DEVELOPER_MODE=ON + -DUMF_BUILD_LIBUMF_POOL_DISJOINT=${{matrix.disjoint}} + -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON + -DUMF_BUILD_TESTS=${{matrix.build_tests}} + -DUMF_BUILD_EXAMPLES=ON + -DUMF_BUILD_LEVEL_ZERO_PROVIDER=ON + -DUMF_TESTS_FAIL_ON_SKIP=ON + -DUMF_BUILD_SHARED_LIBRARY=ON + ${{matrix.extra_build_options}} + + - name: Configure CMake (simple) + if: matrix.simple_cmake == 'ON' + run: > + cmake + -B ${{github.workspace}}/build + -DUMF_BUILD_SHARED_LIBRARY=ON + -DUMF_TESTS_FAIL_ON_SKIP=ON + ${{matrix.extra_build_options}} + + - name: Build + run: cmake --build ${{github.workspace}}/build --config Release -j + + - name: Run examples + working-directory: ${{github.workspace}}/build + run: ctest --output-on-failure --test-dir examples -C Release + + - name: Run tests + if: matrix.build_tests == 'ON' + working-directory: ${{github.workspace}}/build + run: ctest --output-on-failure --test-dir test -C Release + + - name: check /DEPENDENTLOADFLAG (Windows only) + if: matrix.os == 'windows-latest' + run: ${{github.workspace}}/.github/scripts/check_dll_flags.ps1 ${{github.workspace}}/build/bin/Release/umf.dll + shell: pwsh + + - name: check /DEPENDENTLOADFLAG in umf_proxy.dll + if: matrix.os == 'windows-latest' + run: ${{github.workspace}}/.github/scripts/check_dll_flags.ps1 ${{github.workspace}}/build/src/proxy_lib/Release/umf_proxy.dll + shell: pwsh diff --git a/.github/workflows/pr_push.yml b/.github/workflows/pr_push.yml index cec345662..525e4ac9e 100644 --- a/.github/workflows/pr_push.yml +++ b/.github/workflows/pr_push.yml @@ -15,133 +15,6 @@ permissions: contents: read jobs: - FastBuild: - name: Fast build - env: - VCPKG_PATH: "${{github.workspace}}/build/vcpkg/packages/hwloc_x64-windows;${{github.workspace}}/build/vcpkg/packages/tbb_x64-windows;${{github.workspace}}/build/vcpkg/packages/jemalloc_x64-windows" - strategy: - matrix: - include: - - os: windows-latest - disjoint: 'OFF' - build_tests: 'ON' - simple_cmake: 'OFF' - # pure C build (Windows) - - os: windows-latest - disjoint: 'OFF' - # Tests' building is off for a pure C build - build_tests: 'OFF' - simple_cmake: 'OFF' - - os: ubuntu-latest - disjoint: 'ON' - build_tests: 'ON' - # Windows doesn't recognize 'CMAKE_BUILD_TYPE', it uses '--config' param in build command - extra_build_options: '-DCMAKE_BUILD_TYPE=Release -DUMF_BUILD_BENCHMARKS=ON -DUMF_BUILD_BENCHMARKS_MT=ON' - simple_cmake: 'OFF' - # pure C build (Linux) - - os: ubuntu-latest - disjoint: 'OFF' - # Windows doesn't recognize 'CMAKE_BUILD_TYPE', it uses '--config' param in build command - # Tests' building is off for a pure C build - build_tests: 'OFF' - extra_build_options: '-DCMAKE_BUILD_TYPE=Release -DUMF_BUILD_BENCHMARKS=ON' - simple_cmake: 'OFF' - # simplest CMake on ubuntu-latest - - os: ubuntu-latest - disjoint: 'OFF' - build_tests: 'ON' - extra_build_options: '-DCMAKE_BUILD_TYPE=Release' - simple_cmake: 'ON' - # simplest CMake ubuntu-20.04 - - os: ubuntu-20.04 - disjoint: 'OFF' - build_tests: 'ON' - extra_build_options: '-DCMAKE_BUILD_TYPE=Release' - simple_cmake: 'ON' - runs-on: ${{matrix.os}} - - steps: - - name: Checkout repository - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - - name: Initialize vcpkg - if: matrix.os == 'windows-latest' - uses: lukka/run-vcpkg@5e0cab206a5ea620130caf672fce3e4a6b5666a1 # v11.5 - with: - vcpkgGitCommitId: 3dd44b931481d7a8e9ba412621fa810232b66289 - vcpkgDirectory: ${{github.workspace}}/build/vcpkg - vcpkgJsonGlob: '**/vcpkg.json' - - - name: Install dependencies - if: matrix.os == 'windows-latest' - run: vcpkg install - shell: pwsh # Specifies PowerShell as the shell for running the script. - - - name: Install apt packages (ubuntu-latest) - if: matrix.os == 'ubuntu-latest' - run: | - sudo apt-get update - sudo apt-get install -y cmake libjemalloc-dev libhwloc-dev libnuma-dev libtbb-dev - - - name: Install apt packages (ubuntu-20.04) - if: matrix.os == 'ubuntu-20.04' - run: | - sudo apt-get update - sudo apt-get install -y cmake libjemalloc-dev libnuma-dev libtbb-dev - .github/scripts/install_hwloc.sh # install hwloc-2.3.0 instead of hwloc-2.1.0 present in the OS package - - - name: Set ptrace value for IPC test (on Linux only) - if: ${{ matrix.os == 'ubuntu-latest' || matrix.os == 'ubuntu-20.04' }} - run: sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope" - - - name: Configure CMake - if: matrix.simple_cmake == 'OFF' - run: > - cmake - -B ${{github.workspace}}/build - -DCMAKE_PREFIX_PATH="${{env.VCPKG_PATH}}" - -DUMF_FORMAT_CODE_STYLE=OFF - -DUMF_DEVELOPER_MODE=ON - -DUMF_BUILD_LIBUMF_POOL_DISJOINT=${{matrix.disjoint}} - -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON - -DUMF_BUILD_TESTS=${{matrix.build_tests}} - -DUMF_BUILD_EXAMPLES=ON - -DUMF_BUILD_LEVEL_ZERO_PROVIDER=ON - -DUMF_TESTS_FAIL_ON_SKIP=ON - -DUMF_BUILD_SHARED_LIBRARY=ON - ${{matrix.extra_build_options}} - - - name: Configure CMake (simple) - if: matrix.simple_cmake == 'ON' - run: > - cmake - -B ${{github.workspace}}/build - -DUMF_BUILD_SHARED_LIBRARY=ON - -DUMF_TESTS_FAIL_ON_SKIP=ON - ${{matrix.extra_build_options}} - - - name: Build - run: cmake --build ${{github.workspace}}/build --config Release -j - - - name: Run examples - working-directory: ${{github.workspace}}/build - run: ctest --output-on-failure --test-dir examples -C Release - - - name: Run tests - if: matrix.build_tests == 'ON' - working-directory: ${{github.workspace}}/build - run: ctest --output-on-failure --test-dir test -C Release - - - name: check /DEPENDENTLOADFLAG (Windows only) - if: matrix.os == 'windows-latest' - run: ${{github.workspace}}/.github/scripts/check_dll_flags.ps1 ${{github.workspace}}/build/bin/Release/umf.dll - shell: pwsh - - - name: check /DEPENDENTLOADFLAG in umf_proxy.dll - if: matrix.os == 'windows-latest' - run: ${{github.workspace}}/.github/scripts/check_dll_flags.ps1 ${{github.workspace}}/build/src/proxy_lib/Release/umf_proxy.dll - shell: pwsh - CodeStyle: name: Coding style runs-on: ubuntu-latest @@ -197,15 +70,19 @@ jobs: Spellcheck: uses: ./.github/workflows/spellcheck.yml + FastBuild: + name: Fast builds + needs: [Spellcheck, CodeStyle] + uses: ./.github/workflows/fast.yml Build: name: Basic builds - needs: [Spellcheck, FastBuild, CodeStyle] + needs: [FastBuild] uses: ./.github/workflows/basic.yml Sanitizers: - needs: [Spellcheck, FastBuild, CodeStyle] + needs: [FastBuild] uses: ./.github/workflows/sanitizers.yml Qemu: - needs: [Spellcheck, FastBuild, CodeStyle] + needs: [FastBuild] uses: ./.github/workflows/qemu.yml Benchmarks: needs: [Build] From 876d990d4e3f6ce2be18d50d63d22523532a0c93 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Fri, 5 Jul 2024 11:36:07 +0200 Subject: [PATCH 04/52] Add standalone tests of examples Fixes: #427 Signed-off-by: Lukasz Dorau --- examples/basic/CMakeLists.txt | 52 +++++++++++++++ examples/cmake/FindLIBHWLOC.cmake | 75 ++++++++++++++++++++++ examples/cmake/FindLIBUMF.cmake | 30 +++++++++ examples/cmake/FindTBB.cmake | 50 +++++++++++++++ examples/gpu_shared_memory/CMakeLists.txt | 77 +++++++++++++++++++++++ examples/ipc_ipcapi/CMakeLists.txt | 75 ++++++++++++++++++++++ examples/ipc_level_zero/CMakeLists.txt | 76 ++++++++++++++++++++++ src/CMakeLists.txt | 5 ++ test/CMakeLists.txt | 60 ++++++++++++++++++ test/test_examples.sh | 75 ++++++++++++++++++++++ 10 files changed, 575 insertions(+) create mode 100644 examples/basic/CMakeLists.txt create mode 100644 examples/cmake/FindLIBHWLOC.cmake create mode 100644 examples/cmake/FindLIBUMF.cmake create mode 100644 examples/cmake/FindTBB.cmake create mode 100644 examples/gpu_shared_memory/CMakeLists.txt create mode 100644 examples/ipc_ipcapi/CMakeLists.txt create mode 100644 examples/ipc_level_zero/CMakeLists.txt create mode 100755 test/test_examples.sh diff --git a/examples/basic/CMakeLists.txt b/examples/basic/CMakeLists.txt new file mode 100644 index 000000000..ebd1822ab --- /dev/null +++ b/examples/basic/CMakeLists.txt @@ -0,0 +1,52 @@ +# Copyright (C) 2024 Intel Corporation +# Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 3.14.0 FATAL_ERROR) +project(umf_example_basic LANGUAGES C) +enable_testing() + +set(UMF_EXAMPLE_DIR "${CMAKE_SOURCE_DIR}/..") +list(APPEND CMAKE_MODULE_PATH "${UMF_EXAMPLE_DIR}/cmake") +message(STATUS "CMAKE_MODULE_PATH=${CMAKE_MODULE_PATH}") + +find_package(PkgConfig) +pkg_check_modules(LIBUMF libumf) +if(NOT LIBUMF_FOUND) + find_package(LIBUMF REQUIRED libumf) +endif() + +pkg_check_modules(LIBHWLOC hwloc>=2.3.0) +if(NOT LIBHWLOC_FOUND) + find_package(LIBHWLOC 2.3.0 REQUIRED hwloc) +endif() + +pkg_check_modules(TBB tbb) +if(NOT TBB_FOUND) + find_package(TBB REQUIRED tbb) +endif() + +# build the example +set(EXAMPLE_NAME umf_example_basic) +add_executable(${EXAMPLE_NAME} basic.c) +target_include_directories(${EXAMPLE_NAME} PRIVATE ${LIBUMF_INCLUDE_DIRS}) +target_link_directories(${EXAMPLE_NAME} PRIVATE ${LIBHWLOC_LIBRARY_DIRS}) +target_link_libraries(${EXAMPLE_NAME} PRIVATE ${LIBUMF_LIBRARIES} hwloc) + +# an optional part - adds a test of this example +add_test( + NAME ${EXAMPLE_NAME} + COMMAND ${EXAMPLE_NAME} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + +set_tests_properties(${EXAMPLE_NAME} PROPERTIES LABELS "example-standalone") + +if(LINUX) + # set LD_LIBRARY_PATH + set_property( + TEST ${EXAMPLE_NAME} + PROPERTY + ENVIRONMENT_MODIFICATION + "LD_LIBRARY_PATH=path_list_append:${LIBUMF_LIBRARY_DIRS};LD_LIBRARY_PATH=path_list_append:${LIBHWLOC_LIBRARY_DIRS}" + ) +endif() diff --git a/examples/cmake/FindLIBHWLOC.cmake b/examples/cmake/FindLIBHWLOC.cmake new file mode 100644 index 000000000..f4e33417a --- /dev/null +++ b/examples/cmake/FindLIBHWLOC.cmake @@ -0,0 +1,75 @@ +# Copyright (C) 2024 Intel Corporation +# Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +message(STATUS "Checking for module 'libhwloc' using find_library()") + +find_library(LIBHWLOC_LIBRARY NAMES libhwloc hwloc) +set(LIBHWLOC_LIBRARIES ${LIBHWLOC_LIBRARY}) + +get_filename_component(LIBHWLOC_LIB_DIR ${LIBHWLOC_LIBRARIES} DIRECTORY) +set(LIBHWLOC_LIBRARY_DIRS ${LIBHWLOC_LIB_DIR}) + +find_file(LIBHWLOC_HEADER NAMES hwloc.h) +get_filename_component(LIBHWLOC_INCLUDE_DIR ${LIBHWLOC_HEADER} DIRECTORY) +set(LIBHWLOC_INCLUDE_DIRS ${LIBHWLOC_INCLUDE_DIR}) + +set(HWLOC_VERSION_CODE + " +#include +#include +#include \"hwloc.h\" + +void main(int argc, char** argv) { + unsigned LIBHWLOC_API_PATCH = HWLOC_API_VERSION & 0xFF; + unsigned LIBHWLOC_API_MINOR = (HWLOC_API_VERSION >> 8) & 0xFF; + unsigned LIBHWLOC_API_MAJOR = (HWLOC_API_VERSION >> 16) & 0xFF; + printf(\"%d.%d.%d\", LIBHWLOC_API_MAJOR, LIBHWLOC_API_MINOR, LIBHWLOC_API_PATCH); +}") + +set(HWLOC_VERSION_CODE_FILENAME "hwloc_get_version.c") +file(WRITE "${CMAKE_BINARY_DIR}/${HWLOC_VERSION_CODE_FILENAME}" + "${HWLOC_VERSION_CODE}") + +try_run( + HWLOC_RUN_RESULT HWLOC_COMPILE_RESULT ${CMAKE_BINARY_DIR} + "${CMAKE_BINARY_DIR}/${HWLOC_VERSION_CODE_FILENAME}" + CMAKE_FLAGS "-DINCLUDE_DIRECTORIES=${LIBHWLOC_INCLUDE_DIR}" + RUN_OUTPUT_VARIABLE LIBHWLOC_API_VERSION) + +if(WINDOWS) + find_file(LIBHWLOC_DLL NAMES "bin/hwloc-15.dll" "bin/libhwloc-15.dll") + get_filename_component(LIBHWLOC_DLL_DIR ${LIBHWLOC_DLL} DIRECTORY) + set(LIBHWLOC_DLL_DIRS ${LIBHWLOC_DLL_DIR}) +endif() + +if(LIBHWLOC_LIBRARY) + message(STATUS " Found libhwloc using find_library()") + message(STATUS " LIBHWLOC_LIBRARIES = ${LIBHWLOC_LIBRARIES}") + message(STATUS " LIBHWLOC_INCLUDE_DIRS = ${LIBHWLOC_INCLUDE_DIRS}") + message(STATUS " LIBHWLOC_LIBRARY_DIRS = ${LIBHWLOC_LIBRARY_DIRS}") + message(STATUS " LIBHWLOC_API_VERSION = ${LIBHWLOC_API_VERSION}") + if(WINDOWS) + message(STATUS " LIBHWLOC_DLL_DIRS = ${LIBHWLOC_DLL_DIRS}") + endif() + + if(LIBHWLOC_FIND_VERSION) + if(NOT LIBHWLOC_API_VERSION) + message(FATAL_ERROR "Failed to retrieve libhwloc version") + elseif(NOT LIBHWLOC_API_VERSION VERSION_GREATER_EQUAL + LIBHWLOC_FIND_VERSION) + message( + FATAL_ERROR + " Required version: ${LIBHWLOC_FIND_VERSION}, found ${LIBHWLOC_API_VERSION}" + ) + endif() + endif() +else() + set(MSG_NOT_FOUND + "libhwloc NOT found (set CMAKE_PREFIX_PATH to point the location)") + if(LIBHWLOC_FIND_REQUIRED) + message(FATAL_ERROR ${MSG_NOT_FOUND}) + else() + message(WARNING ${MSG_NOT_FOUND}) + endif() +endif() diff --git a/examples/cmake/FindLIBUMF.cmake b/examples/cmake/FindLIBUMF.cmake new file mode 100644 index 000000000..12bdc1823 --- /dev/null +++ b/examples/cmake/FindLIBUMF.cmake @@ -0,0 +1,30 @@ +# Copyright (C) 2024 Intel Corporation +# Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +message(STATUS "Checking for module 'libumf' using find_library()") + +find_library(LIBUMF_LIBRARY NAMES libumf umf) +set(LIBUMF_LIBRARIES ${LIBUMF_LIBRARY}) + +get_filename_component(LIBUMF_LIB_DIR ${LIBUMF_LIBRARIES} DIRECTORY) +set(LIBUMF_LIBRARY_DIRS ${LIBUMF_LIB_DIR}) + +find_file(LIBUMF_HEADER NAMES umf.h) +get_filename_component(LIBUMF_INCLUDE_DIR ${LIBUMF_HEADER} DIRECTORY) +set(LIBUMF_INCLUDE_DIRS ${LIBUMF_INCLUDE_DIR}) + +if(LIBUMF_LIBRARY) + message(STATUS " Found libumf using find_library()") + message(STATUS " LIBUMF_LIBRARIES = ${LIBUMF_LIBRARIES}") + message(STATUS " LIBUMF_INCLUDE_DIRS = ${LIBUMF_INCLUDE_DIRS}") + message(STATUS " LIBUMF_LIBRARY_DIRS = ${LIBUMF_LIBRARY_DIRS}") +else() + set(MSG_NOT_FOUND + "libumf NOT found (set CMAKE_PREFIX_PATH to point the location)") + if(LIBUMF_FIND_REQUIRED) + message(FATAL_ERROR ${MSG_NOT_FOUND}) + else() + message(WARNING ${MSG_NOT_FOUND}) + endif() +endif() diff --git a/examples/cmake/FindTBB.cmake b/examples/cmake/FindTBB.cmake new file mode 100644 index 000000000..8aa6289ef --- /dev/null +++ b/examples/cmake/FindTBB.cmake @@ -0,0 +1,50 @@ +# Copyright (C) 2024 Intel Corporation +# Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +message(STATUS "Checking for module 'tbb' using find_library()") + +find_library(TBB_LIBRARY NAMES libtbbmalloc tbbmalloc) +set(TBB_LIBRARIES ${TBB_LIBRARY}) + +get_filename_component(TBB_LIB_DIR ${TBB_LIBRARIES} DIRECTORY) +set(TBB_LIBRARY_DIRS ${TBB_LIB_DIR}) + +find_file(TBB_HEADER NAMES "tbb/scalable_allocator.h") +if(TBB_HEADER) + get_filename_component(TBB_INCLUDE_DIR_TBB ${TBB_HEADER} DIRECTORY) + get_filename_component(TBB_INCLUDE_DIR ${TBB_INCLUDE_DIR_TBB} DIRECTORY) + set(TBB_INCLUDE_DIRS ${TBB_INCLUDE_DIR}) +else() + set(MSG_NOT_FOUND " header NOT found (set " + "CMAKE_PREFIX_PATH to point the location)") + if(TBB_FIND_REQUIRED) + message(FATAL_ERROR ${MSG_NOT_FOUND}) + else() + message(WARNING ${MSG_NOT_FOUND}) + endif() +endif() + +if(WINDOWS) + find_file(TBB_DLL NAMES "bin/tbbmalloc.dll") + get_filename_component(TBB_DLL_DIR ${TBB_DLL} DIRECTORY) + set(TBB_DLL_DIRS ${TBB_DLL_DIR}) +endif() + +if(TBB_LIBRARY) + message(STATUS " Found tbb using find_library()") + message(STATUS " TBB_LIBRARIES = ${TBB_LIBRARIES}") + message(STATUS " TBB_INCLUDE_DIRS = ${TBB_INCLUDE_DIRS}") + message(STATUS " TBB_LIBRARY_DIRS = ${TBB_LIBRARY_DIRS}") + if(WINDOWS) + message(STATUS " TBB_DLL_DIRS = ${TBB_DLL_DIRS}") + endif() +else() + set(MSG_NOT_FOUND "tbb NOT found (set CMAKE_PREFIX_PATH to point the " + "location)") + if(TBB_FIND_REQUIRED) + message(FATAL_ERROR ${MSG_NOT_FOUND}) + else() + message(WARNING ${MSG_NOT_FOUND}) + endif() +endif() diff --git a/examples/gpu_shared_memory/CMakeLists.txt b/examples/gpu_shared_memory/CMakeLists.txt new file mode 100644 index 000000000..259b47d08 --- /dev/null +++ b/examples/gpu_shared_memory/CMakeLists.txt @@ -0,0 +1,77 @@ +# Copyright (C) 2024 Intel Corporation +# Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 3.14.0 FATAL_ERROR) +project(umf_example_gpu_shared_memory LANGUAGES C) +enable_testing() + +set(UMF_EXAMPLE_DIR "${CMAKE_SOURCE_DIR}/..") +list(APPEND CMAKE_MODULE_PATH "${UMF_EXAMPLE_DIR}/cmake") +message(STATUS "CMAKE_MODULE_PATH=${CMAKE_MODULE_PATH}") + +find_package(PkgConfig) +pkg_check_modules(LIBUMF libumf) +if(NOT LIBUMF_FOUND) + find_package(LIBUMF REQUIRED libumf) +endif() + +pkg_check_modules(LIBHWLOC hwloc>=2.3.0) +if(NOT LIBHWLOC_FOUND) + find_package(LIBHWLOC 2.3.0 REQUIRED hwloc) +endif() + +include(FetchContent) + +set(LEVEL_ZERO_LOADER_REPO "https://github.com/oneapi-src/level-zero.git") +set(LEVEL_ZERO_LOADER_TAG v1.16.1) + +message( + STATUS + "Installing level-zero ${LEVEL_ZERO_LOADER_TAG} from ${LEVEL_ZERO_LOADER_REPO} ..." +) + +FetchContent_Declare( + level-zero-loader + GIT_REPOSITORY ${LEVEL_ZERO_LOADER_REPO} + GIT_TAG ${LEVEL_ZERO_LOADER_TAG} + EXCLUDE_FROM_ALL) + +FetchContent_GetProperties(level-zero-loader) +if(NOT level-zero-loader_POPULATED) + FetchContent_Populate(level-zero-loader) +endif() + +set(LEVEL_ZERO_INCLUDE_DIRS + ${level-zero-loader_SOURCE_DIR}/include + CACHE PATH "Path to Level Zero Headers") +message(STATUS "Level Zero include directory: ${LEVEL_ZERO_INCLUDE_DIRS}") + +# build the example +set(EXAMPLE_NAME umf_example_gpu_shared_memory) +add_executable(${EXAMPLE_NAME} gpu_shared_memory.c) +target_include_directories(${EXAMPLE_NAME} PRIVATE ${LIBUMF_INCLUDE_DIRS} + ${UMF_EXAMPLE_DIR}/common) +target_link_directories(${EXAMPLE_NAME} PRIVATE ${LIBUMF_LIBRARY_DIRS} + ${LIBHWLOC_LIBRARY_DIRS}) +target_link_options(${EXAMPLE_NAME} PRIVATE "-Wl,--start-group") +target_link_libraries(${EXAMPLE_NAME} PRIVATE stdc++ libdisjoint_pool.a + ze_loader ${LIBUMF_LIBRARIES}) + +# an optional part - adds a test of this example +add_test( + NAME ${EXAMPLE_NAME} + COMMAND ${EXAMPLE_NAME} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + +set_tests_properties(${EXAMPLE_NAME} PROPERTIES LABELS "example-standalone") + +if(LINUX) + # set LD_LIBRARY_PATH + set_property( + TEST ${EXAMPLE_NAME} + PROPERTY + ENVIRONMENT_MODIFICATION + "LD_LIBRARY_PATH=path_list_append:${LIBUMF_LIBRARY_DIRS};LD_LIBRARY_PATH=path_list_append:${LIBHWLOC_LIBRARY_DIRS}" + ) +endif() diff --git a/examples/ipc_ipcapi/CMakeLists.txt b/examples/ipc_ipcapi/CMakeLists.txt new file mode 100644 index 000000000..41fae5899 --- /dev/null +++ b/examples/ipc_ipcapi/CMakeLists.txt @@ -0,0 +1,75 @@ +# Copyright (C) 2024 Intel Corporation +# Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 3.14.0 FATAL_ERROR) +project(umf_example_ipc_ipcapi LANGUAGES C) +enable_testing() + +set(UMF_EXAMPLE_DIR "${CMAKE_SOURCE_DIR}/..") +list(APPEND CMAKE_MODULE_PATH "${UMF_EXAMPLE_DIR}/cmake") +message(STATUS "CMAKE_MODULE_PATH=${CMAKE_MODULE_PATH}") + +find_package(PkgConfig) +pkg_check_modules(LIBUMF libumf) +if(NOT LIBUMF_FOUND) + find_package(LIBUMF REQUIRED libumf) +endif() + +pkg_check_modules(LIBHWLOC hwloc>=2.3.0) +if(NOT LIBHWLOC_FOUND) + find_package(LIBHWLOC 2.3.0 REQUIRED hwloc) +endif() + +pkg_check_modules(TBB tbb) +if(NOT TBB_FOUND) + find_package(TBB REQUIRED tbb) +endif() + +# build the example +function(build_umf_ipc_example name) + set(BASE_NAME ${name}) + set(EXAMPLE_NAME umf_example_${BASE_NAME}) + + foreach(loop_var IN ITEMS "producer" "consumer") + set(EX_NAME ${EXAMPLE_NAME}_${loop_var}) + add_executable(${EX_NAME} ${BASE_NAME}_${loop_var}.c) + target_include_directories(${EX_NAME} PRIVATE ${LIBUMF_INCLUDE_DIRS}) + target_link_directories(${EX_NAME} PRIVATE ${LIBHWLOC_LIBRARY_DIRS}) + target_link_libraries(${EX_NAME} PRIVATE ${LIBUMF_LIBRARIES} hwloc) + endforeach(loop_var) +endfunction() + +# an optional part - adds a test of this example +function(add_test_for_umf_ipc_example script) + set(EXAMPLE_NAME umf_example_${script}) + + file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/${script}.sh + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) + + add_test( + NAME ${EXAMPLE_NAME} + COMMAND ${script}.sh + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + + if(LINUX) + set_property( + TEST ${EXAMPLE_NAME} + PROPERTY + ENVIRONMENT_MODIFICATION + "LD_LIBRARY_PATH=path_list_append:${LIBUMF_LIBRARY_DIRS};LD_LIBRARY_PATH=path_list_append:${LIBHWLOC_LIBRARY_DIRS}" + ) + endif() + + set_tests_properties(${EXAMPLE_NAME} PROPERTIES LABELS "example-standalone") + if(NOT UMF_TESTS_FAIL_ON_SKIP) + set_tests_properties(${EXAMPLE_NAME} PROPERTIES SKIP_RETURN_CODE 125) + endif() +endfunction() + +# build the example +build_umf_ipc_example(ipc_ipcapi) + +# an optional part - adds a test of this example +add_test_for_umf_ipc_example(ipc_ipcapi_anon_fd) +add_test_for_umf_ipc_example(ipc_ipcapi_shm) diff --git a/examples/ipc_level_zero/CMakeLists.txt b/examples/ipc_level_zero/CMakeLists.txt new file mode 100644 index 000000000..e38adf25f --- /dev/null +++ b/examples/ipc_level_zero/CMakeLists.txt @@ -0,0 +1,76 @@ +# Copyright (C) 2024 Intel Corporation +# Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +cmake_minimum_required(VERSION 3.14.0 FATAL_ERROR) +project(umf_example_ipc_level_zero LANGUAGES C) +enable_testing() + +set(UMF_EXAMPLE_DIR "${CMAKE_SOURCE_DIR}/..") +list(APPEND CMAKE_MODULE_PATH "${UMF_EXAMPLE_DIR}/cmake") +message(STATUS "CMAKE_MODULE_PATH=${CMAKE_MODULE_PATH}") + +find_package(PkgConfig) +pkg_check_modules(LIBUMF libumf) +if(NOT LIBUMF_FOUND) + find_package(LIBUMF REQUIRED libumf) +endif() + +pkg_check_modules(LIBHWLOC hwloc>=2.3.0) +if(NOT LIBHWLOC_FOUND) + find_package(LIBHWLOC 2.3.0 REQUIRED hwloc) +endif() + +include(FetchContent) + +set(LEVEL_ZERO_LOADER_REPO "https://github.com/oneapi-src/level-zero.git") +set(LEVEL_ZERO_LOADER_TAG v1.16.1) + +message( + STATUS + "Installing level-zero ${LEVEL_ZERO_LOADER_TAG} from ${LEVEL_ZERO_LOADER_REPO} ..." +) + +FetchContent_Declare( + level-zero-loader + GIT_REPOSITORY ${LEVEL_ZERO_LOADER_REPO} + GIT_TAG ${LEVEL_ZERO_LOADER_TAG} + EXCLUDE_FROM_ALL) + +FetchContent_GetProperties(level-zero-loader) +if(NOT level-zero-loader_POPULATED) + FetchContent_Populate(level-zero-loader) +endif() + +set(LEVEL_ZERO_INCLUDE_DIRS + ${level-zero-loader_SOURCE_DIR}/include + CACHE PATH "Path to Level Zero Headers") +message(STATUS "Level Zero include directory: ${LEVEL_ZERO_INCLUDE_DIRS}") + +# build the example +set(EXAMPLE_NAME umf_example_ipc_level_zero) +add_executable(${EXAMPLE_NAME} ipc_level_zero.c) +target_include_directories(${EXAMPLE_NAME} PRIVATE ${LIBUMF_INCLUDE_DIRS} + ${UMF_EXAMPLE_DIR}/common) +target_link_directories(${EXAMPLE_NAME} PRIVATE ${LIBUMF_LIBRARY_DIRS} + ${LIBHWLOC_LIBRARY_DIRS}) +target_link_options(${EXAMPLE_NAME} PRIVATE "-Wl,--start-group") +target_link_libraries(${EXAMPLE_NAME} PRIVATE stdc++ libdisjoint_pool.a + ze_loader ${LIBUMF_LIBRARIES}) + +# an optional part - adds a test of this example +add_test( + NAME ${EXAMPLE_NAME} + COMMAND ${EXAMPLE_NAME} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + +set_tests_properties(${EXAMPLE_NAME} PROPERTIES LABELS "example-standalone") + +if(LINUX) + set_property( + TEST ${EXAMPLE_NAME} + PROPERTY + ENVIRONMENT_MODIFICATION + "LD_LIBRARY_PATH=path_list_append:${LIBUMF_LIBRARY_DIRS};LD_LIBRARY_PATH=path_list_append:${LIBHWLOC_LIBRARY_DIRS}" + ) +endif() diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4019deeeb..bf72c992b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -10,6 +10,11 @@ if(UMF_BUILD_LEVEL_ZERO_PROVIDER) set(LEVEL_ZERO_LOADER_REPO "https://github.com/oneapi-src/level-zero.git") set(LEVEL_ZERO_LOADER_TAG v1.16.1) + message( + STATUS + "Installing level-zero ${LEVEL_ZERO_LOADER_TAG} from ${LEVEL_ZERO_LOADER_REPO} ..." + ) + FetchContent_Declare( level-zero-loader GIT_REPOSITORY ${LEVEL_ZERO_LOADER_REPO} diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index bebcd842e..c005d8038 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -375,3 +375,63 @@ if(LINUX PROPERTY ENVIRONMENT_MODIFICATION "LD_LIBRARY_PATH=path_list_append:${CMAKE_BINARY_DIR}/lib") endif() + +# Tests of examples as standalone projects. TODO: enable this for Windows (maybe +# replace test_examples.sh with CMake script?) +if(LINUX + AND UMF_BUILD_SHARED_LIBRARY + AND NOT + (USE_ASAN + OR USE_UBSAN + OR USE_TSAN + OR USE_MSAN)) + set(EXAMPLES "") + if(UMF_POOL_SCALABLE_ENABLED) + set(EXAMPLES ${EXAMPLES} basic) + else() + message( + STATUS + "The basic example requires TBB to be installed and added to the default library search path - skipping" + ) + endif() + + if(UMF_BUILD_GPU_EXAMPLES + AND UMF_BUILD_LIBUMF_POOL_DISJOINT + AND UMF_BUILD_LEVEL_ZERO_PROVIDER) + set(EXAMPLES ${EXAMPLES} gpu_shared_memory) + else() + message( + STATUS + "GPU shared memory example requires UMF_BUILD_GPU_EXAMPLES, " + "UMF_BUILD_LEVEL_ZERO_PROVIDER and UMF_BUILD_LIBUMF_POOL_DISJOINT " + "to be turned ON - skipping") + endif() + + if(UMF_BUILD_GPU_EXAMPLES + AND UMF_BUILD_LIBUMF_POOL_DISJOINT + AND UMF_BUILD_LEVEL_ZERO_PROVIDER) + set(EXAMPLES ${EXAMPLES} ipc_level_zero) + else() + message( + STATUS + "IPC Level 0 example requires UMF_BUILD_GPU_EXAMPLES, UMF_BUILD_LEVEL_ZERO_PROVIDER and UMF_BUILD_LIBUMF_POOL_DISJOINT to be turned ON - skipping" + ) + endif() + + if(LINUX AND UMF_POOL_SCALABLE_ENABLED) + set(EXAMPLES ${EXAMPLES} ipc_ipcapi) + else() + message( + STATUS + "IPC examples with UMF pool API are supported on Linux with TBB installed only - skipping" + ) + endif() + + add_test( + NAME umf_standalone_examples + COMMAND + ${UMF_CMAKE_SOURCE_DIR}/test/test_examples.sh + ${UMF_CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR} + ${CMAKE_BINARY_DIR}/install ${EXAMPLES} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) +endif() diff --git a/test/test_examples.sh b/test/test_examples.sh new file mode 100755 index 000000000..54b834527 --- /dev/null +++ b/test/test_examples.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# Copyright (C) 2024 Intel Corporation +# Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +set -e + +WORKSPACE=$1 +BUILD_DIR=$2 +INSTALL_DIR=$3 + +echo "Running: $0 $*" + +function print_usage() { + echo "$(basename $0) - test all examples standalone" + echo "Usage: $(basename $0) " +} + +if [ "$3" == "" ]; then + print_usage + echo -e "Error: too few arguments\n" + exit 1 +fi + +if [ "$4" == "" ]; then + print_usage + echo "No examples to run!" + exit 0 +fi + +if [ ! -f $WORKSPACE/README.md ]; then + echo -e "error: incorrect : $WORKSPACE\n" + print_usage + exit 1 +fi + +WORKSPACE=$(realpath $WORKSPACE) +BUILD_DIR=$(realpath $BUILD_DIR) +INSTALL_DIR=$(realpath $INSTALL_DIR) + +shift 3 +EXAMPLES="$*" +echo "Examples to run: $EXAMPLES" +echo + +cd ${BUILD_DIR} +echo "DIR=$(pwd)" + +set -x +cmake .. -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" +make -j$(nproc) install +set +x + +for ex in $EXAMPLES; do + SRC_DIR="${WORKSPACE}/examples/$ex" + BLD_DIR="${BUILD_DIR}/examples-standalone/$ex" + + if [ ! -d $SRC_DIR ]; then + echo "Example does not exist: $ex ($SRC_DIR)" + exit 1 + fi + + echo + echo "Building and running the example: $ex" + echo + + set -x + rm -rf $BLD_DIR + mkdir -p $BLD_DIR + cd $BLD_DIR + CMAKE_PREFIX_PATH="$INSTALL_DIR" cmake $SRC_DIR + make -j$(nproc) + ctest --output-on-failure + set +x +done From ec20ca81ec662da3903bff673763a58c315d3e7a Mon Sep 17 00:00:00 2001 From: Rafal Rudnicki Date: Fri, 5 Jul 2024 15:21:54 +0200 Subject: [PATCH 05/52] add rc file for windows --- src/CMakeLists.txt | 5 ++++ src/libumf.rc.in | 47 +++++++++++++++++++++++++++++++++++ src/proxy_lib/CMakeLists.txt | 5 ++++ src/proxy_lib/proxy_lib.rc.in | 47 +++++++++++++++++++++++++++++++++++ 4 files changed, 104 insertions(+) create mode 100644 src/libumf.rc.in create mode 100644 src/proxy_lib/proxy_lib.rc.in diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4019deeeb..751792db4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -106,6 +106,11 @@ if(LINUX) set(UMF_LIBS ${UMF_LIBS} dl rt) # librt for shm_open() elseif(WINDOWS) set(UMF_SOURCES ${UMF_SOURCES} ${UMF_SOURCES_WINDOWS}) + + # Add resource file needed for Windows to fill metadata in binary files + configure_file("${CMAKE_CURRENT_SOURCE_DIR}/libumf.rc.in" + "${CMAKE_CURRENT_BINARY_DIR}/libumf.rc" IMMEDIATE @ONLY) + set(UMF_SOURCES ${UMF_SOURCES} ${CMAKE_CURRENT_BINARY_DIR}/libumf.rc) elseif(MACOSX) set(UMF_SOURCES ${UMF_SOURCES} ${UMF_SOURCES_MACOSX}) endif() diff --git a/src/libumf.rc.in b/src/libumf.rc.in new file mode 100644 index 000000000..f4ddd3653 --- /dev/null +++ b/src/libumf.rc.in @@ -0,0 +1,47 @@ +// Copyright (c) 2024 Intel Corporation +// +// Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// + +#include + +#include "umf/base.h" + +#define UMF_VERNUMBERS @CMAKE_PROJECT_VERSION_MAJOR@,@CMAKE_PROJECT_VERSION_MINOR@,@CMAKE_PROJECT_VERSION_PATCH@,0 +#define UMF_VERSION "@CMAKE_PROJECT_VERSION_MAJOR@.@CMAKE_PROJECT_VERSION_MINOR@" + +VS_VERSION_INFO VERSIONINFO + FILEVERSION UMF_VERNUMBERS + PRODUCTVERSION UMF_VERNUMBERS + FILEFLAGSMASK VS_FFI_FILEFLAGSMASK +#ifdef _DEBUG + FILEFLAGS VS_FF_DEBUG +#else + FILEFLAGS 0 +#endif + FILEOS VOS__WINDOWS32 + FILETYPE VFT_DLL + FILESUBTYPE 0 +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904b0" // U.S. English, Unicode (0x04b0 == 1200) + BEGIN + VALUE "CompanyName", "Intel Corporation\0" + VALUE "FileDescription", "Unified Memory Framework (UMF) library\0" + VALUE "FileVersion", UMF_VERSION "\0" + VALUE "LegalCopyright", "Copyright 2024, Intel Corporation. All rights reserved.\0" + VALUE "LegalTrademarks", "\0" + VALUE "OriginalFilename", "umf.dll\0" + VALUE "ProductName", "Unified Memory Framework (UMF)\0" + VALUE "ProductVersion", UMF_VERSION "\0" + VALUE "PrivateBuild", "\0" + VALUE "SpecialBuild", "\0" + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x409, 1200 + END +END diff --git a/src/proxy_lib/CMakeLists.txt b/src/proxy_lib/CMakeLists.txt index 7b03626c8..af3f8cb60 100644 --- a/src/proxy_lib/CMakeLists.txt +++ b/src/proxy_lib/CMakeLists.txt @@ -16,6 +16,11 @@ if(LINUX) set(PROXY_SOURCES ${PROXY_SOURCES} ${PROXY_SOURCES_LINUX}) elseif(WINDOWS) set(PROXY_SOURCES ${PROXY_SOURCES} ${PROXY_SOURCES_WINDOWS}) + + # Add resource file needed for Windows to fill metadata in binary files + configure_file("${CMAKE_CURRENT_SOURCE_DIR}/proxy_lib.rc.in" + "${CMAKE_CURRENT_BINARY_DIR}/proxy_lib.rc" IMMEDIATE @ONLY) + set(PROXY_SOURCES ${PROXY_SOURCES} ${CMAKE_CURRENT_BINARY_DIR}/proxy_lib.rc) elseif(MACOSX) set(PROXY_SOURCES ${PROXY_SOURCES} ${PROXY_SOURCES_MACOSX}) endif() diff --git a/src/proxy_lib/proxy_lib.rc.in b/src/proxy_lib/proxy_lib.rc.in new file mode 100644 index 000000000..d8f3fec27 --- /dev/null +++ b/src/proxy_lib/proxy_lib.rc.in @@ -0,0 +1,47 @@ +// Copyright (c) 2024 Intel Corporation +// +// Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// + +#include + +#include "umf/base.h" + +#define UMF_VERNUMBERS @CMAKE_PROJECT_VERSION_MAJOR@,@CMAKE_PROJECT_VERSION_MINOR@,@CMAKE_PROJECT_VERSION_PATCH@,0 +#define UMF_VERSION "@CMAKE_PROJECT_VERSION_MAJOR@.@CMAKE_PROJECT_VERSION_MINOR@" + +VS_VERSION_INFO VERSIONINFO + FILEVERSION UMF_VERNUMBERS + PRODUCTVERSION UMF_VERNUMBERS + FILEFLAGSMASK VS_FFI_FILEFLAGSMASK +#ifdef _DEBUG + FILEFLAGS VS_FF_DEBUG +#else + FILEFLAGS 0 +#endif + FILEOS VOS__WINDOWS32 + FILETYPE VFT_DLL + FILESUBTYPE 0 +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904b0" // U.S. English, Unicode (0x04b0 == 1200) + BEGIN + VALUE "CompanyName", "Intel Corporation\0" + VALUE "FileDescription", "Unified Memory Framework (UMF) proxy library\0" + VALUE "FileVersion", UMF_VERSION "\0" + VALUE "LegalCopyright", "Copyright 2024, Intel Corporation. All rights reserved.\0" + VALUE "LegalTrademarks", "\0" + VALUE "OriginalFilename", "umf_proxy.dll\0" + VALUE "ProductName", "Unified Memory Framework (UMF)\0" + VALUE "ProductVersion", UMF_VERSION "\0" + VALUE "PrivateBuild", "\0" + VALUE "SpecialBuild", "\0" + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x409, 1200 + END +END From d2bb638fa82d083ef309bde1cdb6ad022d7801eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Mon, 8 Jul 2024 17:44:03 +0200 Subject: [PATCH 06/52] fix misspells in the repo --- include/umf/memory_pool.h | 2 +- include/umf/memory_pool_ops.h | 2 +- include/umf/mempolicy.h | 4 ++-- scripts/docs_config/examples.rst | 4 ++-- scripts/docs_config/introduction.rst | 2 +- src/ipc.c | 2 +- src/libumf.h | 4 ++-- src/provider/provider_os_memory.c | 2 +- 8 files changed, 11 insertions(+), 11 deletions(-) diff --git a/include/umf/memory_pool.h b/include/umf/memory_pool.h index d47448760..a93d400f9 100644 --- a/include/umf/memory_pool.h +++ b/include/umf/memory_pool.h @@ -147,7 +147,7 @@ umf_result_t umfFree(void *ptr); /// /// * The implementation of this function *should* be lock-free. /// @param hPool specified memory pool handle for which the last allocation error is returned -/// @return Error code desciribng the failure of the last failed allocation operation. +/// @return Error code describing the failure of the last failed allocation operation. /// The value is undefined if the previous allocation was successful. /// umf_result_t umfPoolGetLastAllocationError(umf_memory_pool_handle_t hPool); diff --git a/include/umf/memory_pool_ops.h b/include/umf/memory_pool_ops.h index 9ddb34700..67afdd166 100644 --- a/include/umf/memory_pool_ops.h +++ b/include/umf/memory_pool_ops.h @@ -116,7 +116,7 @@ typedef struct umf_memory_pool_ops_t { /// /// * The implementation of this function *should* be lock-free. /// @param pool pointer to the memory pool for which the last allocation error is returned - /// @return Error code desciribng the failure of the last failed allocation operation. + /// @return Error code describing the failure of the last failed allocation operation. /// The value is undefined if the previous allocation was successful. /// umf_result_t (*get_last_allocation_error)(void *pool); diff --git a/include/umf/mempolicy.h b/include/umf/mempolicy.h index 0bb7ec9db..59ca5bdd7 100644 --- a/include/umf/mempolicy.h +++ b/include/umf/mempolicy.h @@ -22,9 +22,9 @@ typedef const struct umf_mempolicy_t *umf_const_mempolicy_handle_t; typedef enum umf_mempolicy_membind_t { /// Interleave memory from all memory in memspace UMF_MEMPOLICY_INTERLEAVE, - /// Bind memory to namespace + /// Bind memory to memspace UMF_MEMPOLICY_BIND, - /// Prefer memory from namespace but fallback to other memory if not available + /// Prefer memory from memspace but fallback to other memory if not available UMF_MEMPOLICY_PREFERRED, /// Allocation will be split evenly across nodes specified in nodemask. /// umf_mempolicy_split_partition_t can be used to specify different distribution. diff --git a/scripts/docs_config/examples.rst b/scripts/docs_config/examples.rst index fea381cdf..7f83eee11 100644 --- a/scripts/docs_config/examples.rst +++ b/scripts/docs_config/examples.rst @@ -85,7 +85,7 @@ the :any:`umfPoolCreate` function:: umfPoolCreate(pool_ops, provider, pool_params, flags, &pool); The ``pool`` has been created, we can allocate some memory now -with ie. :any:`umfPoolCalloc`:: +with i.e. :any:`umfPoolCalloc`:: size_t num = 1; alloc_size = 128; @@ -169,7 +169,7 @@ to another process it can be opened by the :any:`umfOpenIPCHandle` function. void *mapped_buf = NULL; umf_result = umfOpenIPCHandle(consumer_pool, ipc_handle, &mapped_buf); -The :any:`umfOpenIPCHandle` function requires the memory pool handle and the IPC handle as input parameters. It mapps +The :any:`umfOpenIPCHandle` function requires the memory pool handle and the IPC handle as input parameters. It maps the handle to the current process address space and returns the pointer to the same memory region that was allocated in the producer process. diff --git a/scripts/docs_config/introduction.rst b/scripts/docs_config/introduction.rst index 52c261958..d47439047 100644 --- a/scripts/docs_config/introduction.rst +++ b/scripts/docs_config/introduction.rst @@ -139,7 +139,7 @@ Logging ============ Logging in UMF is handled by logger. There are several levels of logging: *debug*, *info*, *warning*, and *error*. -The level of logging determines what messages will be printed, ie. the level set to *warning* means all messages at levels *warning* and *error* will be printed. +The level of logging determines what messages will be printed, i.e. the level set to *warning* means all messages at levels *warning* and *error* will be printed. By default, there is a guarantee that *error* messages are flushed immediately. One can change this behavior to flush on lower-level messages. diff --git a/src/ipc.c b/src/ipc.c index 62845148e..b266004f3 100644 --- a/src/ipc.c +++ b/src/ipc.c @@ -104,7 +104,7 @@ umf_result_t umfPutIPCHandle(umf_ipc_handle_t umfIPCHandle) { // to upstream memory provider when umfMemoryProviderFree is called. // To support incapsulation we should not take into account // implementation details of tracking memory provider and find the - // approrpiate pool, get memory provider of that pool and call + // appropriate pool, get memory provider of that pool and call // umfMemoryProviderPutIPCHandle(hProvider, // umfIPCHandle->providerIpcData); umf_ba_global_free(umfIPCHandle); diff --git a/src/libumf.h b/src/libumf.h index 0e0fbfb0e..e85890b6c 100644 --- a/src/libumf.h +++ b/src/libumf.h @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023 Intel Corporation + * Copyright (C) 2023-2024 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -14,7 +14,7 @@ extern "C" { #endif -// initializes runtime state needed by the library (needed mostly for static libaries on windows) +// initializes runtime state needed by the library (needed mostly for static libraries on Windows) void libumfInit(void); #ifdef __cplusplus diff --git a/src/provider/provider_os_memory.c b/src/provider/provider_os_memory.c index 6b71caac0..6aaec9681 100644 --- a/src/provider/provider_os_memory.c +++ b/src/provider/provider_os_memory.c @@ -580,7 +580,7 @@ static void os_finalize(void *provider) { static umf_result_t os_get_min_page_size(void *provider, void *ptr, size_t *page_size); -// TODO: this function should be reenabled when CTL is implemented +// TODO: this function should be re-enabled when CTL is implemented #if 0 static void print_numa_nodes(os_memory_provider_t *os_provider, void *addr, size_t size) { From b52c49b52faf666fa70a5eb840e70144a730b4ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Plewa?= Date: Tue, 9 Jul 2024 13:38:02 +0200 Subject: [PATCH 07/52] Remove not needed else clause in os provider MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Łukasz Plewa --- src/provider/provider_os_memory.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/provider/provider_os_memory.c b/src/provider/provider_os_memory.c index 6b71caac0..b52b4bdeb 100644 --- a/src/provider/provider_os_memory.c +++ b/src/provider/provider_os_memory.c @@ -1103,17 +1103,17 @@ static umf_result_t os_allocation_split(void *provider, void *ptr, "descriptor offset map failed (addr=%p)", ptr); return UMF_RESULT_ERROR_UNKNOWN; - } else { - uintptr_t new_key = (uintptr_t)ptr + firstSize; - void *new_value = (void *)((uintptr_t)value + firstSize); - int ret = critnib_insert(os_provider->fd_offset_map, new_key, new_value, - 0 /* update */); - if (ret) { - LOG_ERR("os_allocation_split(): inserting a value to the file " - "descriptor offset map failed (addr=%p, offset=%zu)", - (void *)new_key, (size_t)new_value - 1); - return UMF_RESULT_ERROR_UNKNOWN; - } + } + + uintptr_t new_key = (uintptr_t)ptr + firstSize; + void *new_value = (void *)((uintptr_t)value + firstSize); + int ret = critnib_insert(os_provider->fd_offset_map, new_key, new_value, + 0 /* update */); + if (ret) { + LOG_ERR("os_allocation_split(): inserting a value to the file " + "descriptor offset map failed (addr=%p, offset=%zu)", + (void *)new_key, (size_t)new_value - 1); + return UMF_RESULT_ERROR_UNKNOWN; } return UMF_RESULT_SUCCESS; From 1569a9514349ab7f9d66114c2f9a3d7a986e6377 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Plewa?= Date: Tue, 9 Jul 2024 16:11:24 +0200 Subject: [PATCH 08/52] Enable logger in proxy lib MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fixes: #605 Signed-off-by: Łukasz Plewa --- src/proxy_lib/proxy_lib.c | 1 + src/utils/utils_log.c | 4 ++-- test/utils/utils_log.cpp | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/proxy_lib/proxy_lib.c b/src/proxy_lib/proxy_lib.c index c29bae300..6c3ffa272 100644 --- a/src/proxy_lib/proxy_lib.c +++ b/src/proxy_lib/proxy_lib.c @@ -107,6 +107,7 @@ static __TLS int was_called_from_umfPool = 0; /*****************************************************************************/ void proxy_lib_create_common(void) { + util_log_init(); umf_os_memory_provider_params_t os_params = umfOsMemoryProviderParamsDefault(); umf_result_t umf_result; diff --git a/src/utils/utils_log.c b/src/utils/utils_log.c index 8e68fe899..aebb21d65 100644 --- a/src/utils/utils_log.c +++ b/src/utils/utils_log.c @@ -254,10 +254,10 @@ void util_log_init(void) { memcpy(file, arg, len); file[len] = '\0'; - loggerConfig.output = fopen(file, "w+"); + loggerConfig.output = fopen(file, "a"); if (!loggerConfig.output) { loggerConfig.output = stderr; - LOG_ERR("Cannot open output file %s - logging disabled", file); + LOG_PERR("Cannot open output file %s - logging disabled", file); loggerConfig.output = NULL; return; } diff --git a/test/utils/utils_log.cpp b/test/utils/utils_log.cpp index 159236ab5..235ae7816 100644 --- a/test/utils/utils_log.cpp +++ b/test/utils/utils_log.cpp @@ -14,7 +14,7 @@ int fopen_count = 0; FILE *mock_fopen(const char *filename, const char *mode) { fopen_count++; EXPECT_STREQ(filename, expected_filename.c_str()); - EXPECT_STREQ(mode, "w+"); + EXPECT_STREQ(mode, "a"); return MOCK_FILE_PTR; } From ed8c916876e91a71df0bf28067c6a713f18cbc49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Wed, 10 Jul 2024 13:33:04 +0200 Subject: [PATCH 09/52] [docs] Update links to examples --- scripts/docs_config/examples.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/docs_config/examples.rst b/scripts/docs_config/examples.rst index 7f83eee11..4098583a6 100644 --- a/scripts/docs_config/examples.rst +++ b/scripts/docs_config/examples.rst @@ -111,14 +111,14 @@ Freeing memory is as easy as can be:: GPU shared memory ============================================================================== -You can find the full example code in the `examples/basic/gpu_shared_memory.c`_ file +You can find the full example code in the `examples/gpu_shared_memory/gpu_shared_memory.c`_ file in the UMF repository. TODO IPC example with Level Zero Memory Provider ============================================================================== -The full code of the example is in the `examples/basic/ipc_level_zero.c`_ file in the UMF repository. +The full code of the example is in the `examples/ipc_level_zero/ipc_level_zero.c`_ file in the UMF repository. The example demonstrates how to use UMF :ref:`IPC API `. For demonstration purpose the example uses Level Zero memory provider to instantiate a pool. But the same flow will work with any memory provider that supports IPC capabilities. @@ -193,8 +193,8 @@ function is called on the consumer side. The memory mappings on the consumer sid the :any:`umfCloseIPCHandle` function is called. .. _examples/basic/basic.c: https://github.com/oneapi-src/unified-memory-framework/blob/main/examples/basic/basic.c -.. _examples/basic/gpu_shared_memory.c: https://github.com/oneapi-src/unified-memory-framework/blob/main/examples/basic/gpu_shared_memory.c -.. _examples/basic/ipc_level_zero.c: https://github.com/oneapi-src/unified-memory-framework/blob/main/examples/basic/ipc_level_zero.c +.. _examples/gpu_shared_memory/gpu_shared_memory.c: https://github.com/oneapi-src/unified-memory-framework/blob/main/examples/gpu_shared_memory/gpu_shared_memory.c +.. _examples/ipc_level_zero/ipc_level_zero.c: https://github.com/oneapi-src/unified-memory-framework/blob/main/examples/ipc_level_zero/ipc_level_zero.c .. _README: https://github.com/oneapi-src/unified-memory-framework/blob/main/README.md#memory-pool-managers .. _umf/ipc.h: https://github.com/oneapi-src/unified-memory-framework/blob/main/include/umf/ipc.h .. _provider_os_memory.h: https://github.com/oneapi-src/unified-memory-framework/blob/main/include/umf/providers/provider_os_memory.h From ffb4da9f169d23079f1666d8a2b9afad5981564e Mon Sep 17 00:00:00 2001 From: Rafal Rudnicki Date: Wed, 10 Jul 2024 11:10:26 +0200 Subject: [PATCH 10/52] add Third Party Programs file --- CMakeLists.txt | 1 + test/test_installation.py | 1 + third-party-programs.txt | 391 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 393 insertions(+) create mode 100644 third-party-programs.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 1ef9ba7ba..0957649eb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -519,6 +519,7 @@ endif() # Configure make install/uninstall and packages # --------------------------------------------------------------------------- # install(FILES ${CMAKE_SOURCE_DIR}/LICENSE.TXT + ${CMAKE_SOURCE_DIR}/third-party-programs.txt DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/doc/${PROJECT_NAME}/") install(DIRECTORY examples DESTINATION "${CMAKE_INSTALL_DOCDIR}") diff --git a/test/test_installation.py b/test/test_installation.py index b2a86f430..6acb4a0bc 100644 --- a/test/test_installation.py +++ b/test/test_installation.py @@ -146,6 +146,7 @@ def _create_match_list(self) -> List[str]: examples.insert(0, "share/doc/umf/examples") share.extend(examples) share.append("share/doc/umf/LICENSE.TXT") + share.append("share/doc/umf/third-party-programs.txt") all_files = bin + include + lib + share if platform.system() == "Windows": diff --git a/third-party-programs.txt b/third-party-programs.txt new file mode 100644 index 000000000..4adbd8cf6 --- /dev/null +++ b/third-party-programs.txt @@ -0,0 +1,391 @@ +Unified Memory Framework (UMF) Third Party Programs File + +This file is the "third-party-programs.txt" file specified in the associated +Intel end user license agreement for the Intel software you are licensing. + +The third party programs and their corresponding required notices and/or +license terms are listed below. +_______________________________________________________________________________ + +1. Modified mimaloc new/delete implementation: + + MIT License + + Copyright (c) 2018-2021 Microsoft Corporation, Daan Leijen + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. + +_______________________________________________________________________________ + +2. Portable Hardware Locality (hwloc): + + Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana + University Research and Technology Corporation. All + rights reserved. + Copyright (c) 2004-2005 The University of Tennessee and The University of + Tennessee Research Foundation. All rights reserved. + Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + University of Stuttgart. All rights reserved. + Copyright (c) 2004-2005 The Regents of the University of California. All + rights reserved. + Copyright (c) 2009 CNRS + Copyright (c) 2009-2016 Inria. All rights reserved. + Copyright (c) 2009-2015 Université Bordeaux + Copyright (c) 2009-2015 Cisco Systems, Inc. All rights reserved. + Copyright (c) 2009-2012 Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2010 IBM + Copyright (c) 2010 Jirka Hladky + Copyright (c) 2012 Aleksej Saushev, The NetBSD Foundation + Copyright (c) 2012 Blue Brain Project, EPFL. All rights reserved. + Copyright (c) 2013-2014 University of Wisconsin-La Crosse. All rights + reserved. + Copyright (c) 2015 Research Organization for Information Science and + Technology (RIST). All rights reserved. + Copyright (c) 2015-2016 Intel, Inc. All rights reserved. + + See COPYING in top-level directory. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + 3. The name of the author may not be used to endorse or promote products + derived from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +_______________________________________________________________________________ + +3. jemalloc + + Copyright (C) 2002-present Jason Evans . + All rights reserved. + Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved. + Copyright (C) 2009-present Facebook, Inc. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + 1. Redistributions of source code must retain the above copyright + notice(s), this list of conditions and the following disclaimer. + 2. Redistributions in binary form must reproduce the above copyright + notice(s), this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY + EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY + DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH + DAMAGE. +_______________________________________________________________________________ + +4. ubench + + This is free and unencumbered software released into the public domain. + + Anyone is free to copy, modify, publish, use, compile, sell, or + distribute this software, either in source code form or as a compiled + binary, for any purpose, commercial or non-commercial, and by any + means. + + In jurisdictions that recognize copyright laws, the author or authors + of this software dedicate any and all copyright interest in the + software to the public domain. We make this dedication for the benefit + of the public at large and to the detriment of our heirs and + successors. We intend this dedication to be an overt act of + relinquishment in perpetuity of all present and future rights to this + software under copyright law. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + OTHER DEALINGS IN THE SOFTWARE. + + For more information, please refer to +_______________________________________________________________________________ + +5. Level Zero + + MIT License + + Copyright (C) 2019-2021 Intel Corporation + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. +_______________________________________________________________________________ + +6. The Unified Runtime Project is under the Apache License v2.0 with LLVM + Exceptions: + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + ---- LLVM Exceptions to the Apache 2.0 License ---- + + As an exception, if, as a result of your compiling your source code, + portions of this Software are embedded into an Object form of such source + code, you may redistribute such embedded portions in such Object form + without complying with the conditions of Sections 4(a), 4(b) and 4(d) of + the License. + + In addition, if you combine or link compiled forms of this Software with + software that is licensed under the GPLv2 ("Combined Software") and if a + court of competent jurisdiction determines that the patent provision + (Section 3), the indemnity provision (Section 9) or other Section of the + License conflicts with the conditions of the GPLv2, you may retroactively + and prospectively choose to deem waived or otherwise exclude such + Section(s) of the License, but only in their entirety and only with respect + to the Combined Software. +_______________________________________________________________________________ + +*Other names and brands may be claimed as the property of others. From affb448674fbf4cc1486d76c3025421abb1bd87d Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Mon, 15 Jul 2024 13:47:40 +0200 Subject: [PATCH 11/52] Set CMAKE_INSTALL_PREFIX in all CI builds Set CMAKE_INSTALL_PREFIX in all CI builds. Clean up setting and using of env.BUILD_DIR. Signed-off-by: Lukasz Dorau --- .github/workflows/basic.yml | 28 +++++++++++++--------------- .github/workflows/benchmarks.yml | 6 +++++- .github/workflows/codeql.yml | 11 ++++++++--- .github/workflows/fast.yml | 22 ++++++++++++++-------- .github/workflows/gpu.yml | 18 ++++++++++++------ .github/workflows/proxy_lib.yml | 17 +++++++++++------ .github/workflows/sanitizers.yml | 7 +++++-- 7 files changed, 68 insertions(+), 41 deletions(-) diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml index 1adbffe31..0853fc848 100644 --- a/.github/workflows/basic.yml +++ b/.github/workflows/basic.yml @@ -9,6 +9,8 @@ permissions: env: # for installation testing - it should match with version set in CMake UMF_VERSION: 0.1.0 + BUILD_DIR : "${{github.workspace}}/build" + INSTL_DIR : "${{github.workspace}}/../install-dir" jobs: ubuntu-build: @@ -67,9 +69,6 @@ jobs: shared_library: 'ON' level_zero_provider: 'ON' install_tbb: 'OFF' - env: - BUILD_DIR : "${{github.workspace}}/build/" - INSTL_DIR : "${{github.workspace}}/../install-dir" runs-on: ${{matrix.os}} steps: @@ -131,6 +130,9 @@ jobs: ${{ matrix.compiler.cxx == 'icpx' && '. /opt/intel/oneapi/setvars.sh &&' || ''}} ctest --output-on-failure --test-dir test + - name: Remove the installation directory + run: rm -rf ${{env.INSTL_DIR}} + - name: Test UMF installation and uninstallation # The '--shared-library' parameter is added to the installation test when the UMF is built as a shared library run: > @@ -148,8 +150,6 @@ jobs: name: Windows env: VCPKG_PATH: "${{github.workspace}}/build/vcpkg/packages/hwloc_x64-windows;${{github.workspace}}/build/vcpkg/packages/tbb_x64-windows;${{github.workspace}}/build/vcpkg/packages/jemalloc_x64-windows" - BUILD_DIR : "${{github.workspace}}/build/" - INSTL_DIR : "${{github.workspace}}/../install-dir" strategy: matrix: os: ['windows-2019', 'windows-2022'] @@ -185,7 +185,7 @@ jobs: uses: lukka/run-vcpkg@5e0cab206a5ea620130caf672fce3e4a6b5666a1 # v11.5 with: vcpkgGitCommitId: 3dd44b931481d7a8e9ba412621fa810232b66289 - vcpkgDirectory: ${{github.workspace}}/build/vcpkg + vcpkgDirectory: ${{env.BUILD_DIR}}/vcpkg vcpkgJsonGlob: '**/vcpkg.json' - name: Install dependencies @@ -197,6 +197,7 @@ jobs: cmake -B ${{env.BUILD_DIR}} ${{matrix.toolset}} + -DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}" -DCMAKE_PREFIX_PATH="${{env.VCPKG_PATH}}" -DCMAKE_C_COMPILER=${{matrix.compiler.c}} -DCMAKE_CXX_COMPILER=${{matrix.compiler.cxx}} @@ -230,18 +231,16 @@ jobs: - name: check /DEPENDENTLOADFLAG in umf.dll if: ${{matrix.shared_library == 'ON' && matrix.compiler.cxx == 'cl'}} - run: ${{github.workspace}}/.github/scripts/check_dll_flags.ps1 ${{github.workspace}}/build/bin/${{matrix.build_type}}/umf.dll + run: ${{github.workspace}}/.github/scripts/check_dll_flags.ps1 ${{env.BUILD_DIR}}/bin/${{matrix.build_type}}/umf.dll shell: pwsh - name: check /DEPENDENTLOADFLAG in umf_proxy.dll if: ${{matrix.compiler.cxx == 'cl'}} - run: ${{github.workspace}}/.github/scripts/check_dll_flags.ps1 ${{github.workspace}}/build/src/proxy_lib/${{matrix.build_type}}/umf_proxy.dll + run: ${{github.workspace}}/.github/scripts/check_dll_flags.ps1 ${{env.BUILD_DIR}}/src/proxy_lib/${{matrix.build_type}}/umf_proxy.dll shell: pwsh windows-dynamic_build_hwloc: name: "Windows dynamic UMF + static hwloc" - env: - BUILD_DIR : "${{github.workspace}}/build" strategy: matrix: build_type: [Release] @@ -256,6 +255,7 @@ jobs: run: > cmake -B ${{env.BUILD_DIR}} + -DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}" -DUMF_BUILD_SHARED_LIBRARY=ON -DUMF_BUILD_EXAMPLES=OFF -DUMF_FORMAT_CODE_STYLE=OFF @@ -276,13 +276,11 @@ jobs: # we check umf.dll only here - note that the proxy library is disabled in # this configuration - name: check /DEPENDENTLOADFLAG in umf.dll - run: ${{github.workspace}}/.github/scripts/check_dll_flags.ps1 ${{github.workspace}}/build/bin/${{matrix.build_type}}/umf.dll + run: ${{github.workspace}}/.github/scripts/check_dll_flags.ps1 ${{env.BUILD_DIR}}/bin/${{matrix.build_type}}/umf.dll shell: pwsh windows-static_build_hwloc: name: "Windows static UMF + static hwloc" - env: - BUILD_DIR : "${{github.workspace}}/build" strategy: matrix: build_type: [Release] @@ -297,6 +295,7 @@ jobs: run: > cmake -B ${{env.BUILD_DIR}} + -DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}" -DUMF_BUILD_SHARED_LIBRARY=OFF -DUMF_BUILD_EXAMPLES=OFF -DUMF_FORMAT_CODE_STYLE=OFF @@ -320,8 +319,6 @@ jobs: matrix: os: ['macos-12', 'macos-13'] env: - BUILD_DIR : "${{github.workspace}}/build/" - INSTL_DIR : "${{github.workspace}}/../install-dir" BUILD_TYPE : "Release" runs-on: ${{matrix.os}} @@ -339,6 +336,7 @@ jobs: run: > cmake -B ${{env.BUILD_DIR}} + -DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}" -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DUMF_FORMAT_CODE_STYLE=OFF -DUMF_DEVELOPER_MODE=ON diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 6a418a09c..e6268a0fc 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -6,11 +6,14 @@ on: workflow_call permissions: contents: read +env: + BUILD_DIR : "${{github.workspace}}/build" + INSTL_DIR : "${{github.workspace}}/../install-dir" + jobs: benchmarks: name: Benchmarks env: - BUILD_DIR : "${{github.workspace}}/build/" VCPKG_PATH: "${{github.workspace}}/build/vcpkg/packages/hwloc_x64-windows;${{github.workspace}}/build/vcpkg/packages/tbb_x64-windows;${{github.workspace}}/build/vcpkg/packages/jemalloc_x64-windows" strategy: matrix: @@ -49,6 +52,7 @@ jobs: cmake -B ${{env.BUILD_DIR}} ${{matrix.extra_build_option}} + -DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}" -DCMAKE_PREFIX_PATH="${{env.VCPKG_PATH}}" -DUMF_BUILD_SHARED_LIBRARY=ON -DUMF_BUILD_BENCHMARKS=ON diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 795feb71a..d59a45089 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -18,6 +18,10 @@ concurrency: permissions: contents: read +env: + BUILD_DIR : "${{github.workspace}}/build" + INSTL_DIR : "${{github.workspace}}/../install-dir" + jobs: analyze: name: Analyze @@ -50,7 +54,7 @@ jobs: uses: lukka/run-vcpkg@5e0cab206a5ea620130caf672fce3e4a6b5666a1 # v11.5 with: vcpkgGitCommitId: 3dd44b931481d7a8e9ba412621fa810232b66289 - vcpkgDirectory: ${{github.workspace}}/build/vcpkg + vcpkgDirectory: ${{env.BUILD_DIR}}/vcpkg vcpkgJsonGlob: '**/vcpkg.json' - name: Install dependencies @@ -70,8 +74,9 @@ jobs: - name: Configure CMake run: > cmake - -B ${{github.workspace}}/build + -B ${{env.BUILD_DIR}} ${{matrix.extra_build_option}} + -DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}" -DCMAKE_PREFIX_PATH="${{env.VCPKG_PATH}}" -DUMF_FORMAT_CODE_STYLE=OFF -DUMF_DEVELOPER_MODE=ON @@ -80,7 +85,7 @@ jobs: -DUMF_TESTS_FAIL_ON_SKIP=ON - name: Build - run: cmake --build ${{github.workspace}}/build --config Release -j + run: cmake --build ${{env.BUILD_DIR}} --config Release -j - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@b7bf0a3ed3ecfa44160715d7c442788f65f0f923 # v3.23.2 diff --git a/.github/workflows/fast.yml b/.github/workflows/fast.yml index 9674c5820..76f4f09ab 100644 --- a/.github/workflows/fast.yml +++ b/.github/workflows/fast.yml @@ -6,6 +6,10 @@ on: workflow_call permissions: contents: read +env: + BUILD_DIR : "${{github.workspace}}/build" + INSTL_DIR : "${{github.workspace}}/../install-dir" + jobs: FastBuild: name: Fast builds @@ -61,7 +65,7 @@ jobs: uses: lukka/run-vcpkg@5e0cab206a5ea620130caf672fce3e4a6b5666a1 # v11.5 with: vcpkgGitCommitId: 3dd44b931481d7a8e9ba412621fa810232b66289 - vcpkgDirectory: ${{github.workspace}}/build/vcpkg + vcpkgDirectory: ${{env.BUILD_DIR}}/vcpkg vcpkgJsonGlob: '**/vcpkg.json' - name: Install dependencies @@ -90,7 +94,8 @@ jobs: if: matrix.simple_cmake == 'OFF' run: > cmake - -B ${{github.workspace}}/build + -B ${{env.BUILD_DIR}} + -DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}" -DCMAKE_PREFIX_PATH="${{env.VCPKG_PATH}}" -DUMF_FORMAT_CODE_STYLE=OFF -DUMF_DEVELOPER_MODE=ON @@ -107,29 +112,30 @@ jobs: if: matrix.simple_cmake == 'ON' run: > cmake - -B ${{github.workspace}}/build + -B ${{env.BUILD_DIR}} + -DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}" -DUMF_BUILD_SHARED_LIBRARY=ON -DUMF_TESTS_FAIL_ON_SKIP=ON ${{matrix.extra_build_options}} - name: Build - run: cmake --build ${{github.workspace}}/build --config Release -j + run: cmake --build ${{env.BUILD_DIR}} --config Release -j - name: Run examples - working-directory: ${{github.workspace}}/build + working-directory: ${{env.BUILD_DIR}} run: ctest --output-on-failure --test-dir examples -C Release - name: Run tests if: matrix.build_tests == 'ON' - working-directory: ${{github.workspace}}/build + working-directory: ${{env.BUILD_DIR}} run: ctest --output-on-failure --test-dir test -C Release - name: check /DEPENDENTLOADFLAG (Windows only) if: matrix.os == 'windows-latest' - run: ${{github.workspace}}/.github/scripts/check_dll_flags.ps1 ${{github.workspace}}/build/bin/Release/umf.dll + run: ${{github.workspace}}/.github/scripts/check_dll_flags.ps1 ${{env.BUILD_DIR}}/bin/Release/umf.dll shell: pwsh - name: check /DEPENDENTLOADFLAG in umf_proxy.dll if: matrix.os == 'windows-latest' - run: ${{github.workspace}}/.github/scripts/check_dll_flags.ps1 ${{github.workspace}}/build/src/proxy_lib/Release/umf_proxy.dll + run: ${{github.workspace}}/.github/scripts/check_dll_flags.ps1 ${{env.BUILD_DIR}}/src/proxy_lib/Release/umf_proxy.dll shell: pwsh diff --git a/.github/workflows/gpu.yml b/.github/workflows/gpu.yml index 2b2e79a70..444ebdfe4 100644 --- a/.github/workflows/gpu.yml +++ b/.github/workflows/gpu.yml @@ -8,6 +8,10 @@ on: [workflow_call] permissions: contents: read +env: + BUILD_DIR : "${{github.workspace}}/build" + INSTL_DIR : "${{github.workspace}}/../install-dir" + jobs: gpu: name: Build @@ -42,7 +46,8 @@ jobs: run: > cmake -DCMAKE_PREFIX_PATH="${{env.VCPKG_PATH}}" - -B ${{github.workspace}}/build + -B ${{env.BUILD_DIR}} + -DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}" -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_C_COMPILER=${{matrix.compiler.c}} -DCMAKE_CXX_COMPILER=${{matrix.compiler.cxx}} @@ -62,7 +67,8 @@ jobs: if: matrix.os == 'Ubuntu' run: > cmake - -B ${{github.workspace}}/build + -B ${{env.BUILD_DIR}} + -DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}" -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}} -DCMAKE_C_COMPILER=${{matrix.compiler.c}} -DCMAKE_CXX_COMPILER=${{matrix.compiler.cxx}} @@ -79,16 +85,16 @@ jobs: -DUMF_TESTS_FAIL_ON_SKIP=ON - name: Build UMF - run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}} -j ${{matrix.number_of_processors}} + run: cmake --build ${{env.BUILD_DIR}} --config ${{env.BUILD_TYPE}} -j ${{matrix.number_of_processors}} - name: Run tests - working-directory: ${{github.workspace}}/build + working-directory: ${{env.BUILD_DIR}} run: ctest -C ${{env.BUILD_TYPE}} --output-on-failure --test-dir test - name: Run examples - working-directory: ${{github.workspace}}/build + working-directory: ${{env.BUILD_DIR}} run: ctest --output-on-failure --test-dir examples -C ${{env.BUILD_TYPE}} - name: Run benchmarks - working-directory: ${{github.workspace}}/build + working-directory: ${{env.BUILD_DIR}} run: ctest --output-on-failure --test-dir benchmark -C ${{env.BUILD_TYPE}} --exclude-regex umf-bench-multithreaded diff --git a/.github/workflows/proxy_lib.yml b/.github/workflows/proxy_lib.yml index f77e9656b..43309b153 100644 --- a/.github/workflows/proxy_lib.yml +++ b/.github/workflows/proxy_lib.yml @@ -6,6 +6,10 @@ on: workflow_call permissions: contents: read +env: + BUILD_DIR : "${{github.workspace}}/build" + INSTL_DIR : "${{github.workspace}}/../install-dir" + jobs: proxy-ubuntu: name: Ubuntu @@ -33,7 +37,8 @@ jobs: - name: Configure build run: > cmake - -B ${{github.workspace}}/build + -B ${{env.BUILD_DIR}} + -DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}" -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DCMAKE_C_COMPILER=${{matrix.compiler.c}} -DCMAKE_CXX_COMPILER=${{matrix.compiler.cxx}} @@ -48,20 +53,20 @@ jobs: -DUMF_PROXY_LIB_BASED_ON_POOL=${{matrix.proxy_lib_pool}} - name: Build UMF - run: cmake --build ${{github.workspace}}/build -j $(nproc) + run: cmake --build ${{env.BUILD_DIR}} -j $(nproc) - name: Run "ctest --output-on-failure" with proxy library - working-directory: ${{github.workspace}}/build + working-directory: ${{env.BUILD_DIR}} run: LD_PRELOAD=./lib/libumf_proxy.so ctest --output-on-failure - name: Run "./test/umf_test-memoryPool" with proxy library - working-directory: ${{github.workspace}}/build + working-directory: ${{env.BUILD_DIR}} run: LD_PRELOAD=./lib/libumf_proxy.so ./test/umf_test-memoryPool - name: Run "/usr/bin/ls" with proxy library - working-directory: ${{github.workspace}}/build + working-directory: ${{env.BUILD_DIR}} run: LD_PRELOAD=./lib/libumf_proxy.so /usr/bin/ls - name: Run "/usr/bin/date" with proxy library - working-directory: ${{github.workspace}}/build + working-directory: ${{env.BUILD_DIR}} run: LD_PRELOAD=./lib/libumf_proxy.so /usr/bin/date diff --git a/.github/workflows/sanitizers.yml b/.github/workflows/sanitizers.yml index 19c2f8e96..d3a4cd4b8 100644 --- a/.github/workflows/sanitizers.yml +++ b/.github/workflows/sanitizers.yml @@ -4,7 +4,8 @@ name: Sanitizers on: workflow_call env: - BUILD_DIR : "${{github.workspace}}/build/" + BUILD_DIR : "${{github.workspace}}/build" + INSTL_DIR : "${{github.workspace}}/../install-dir" permissions: contents: read @@ -46,6 +47,7 @@ jobs: ${{ matrix.compiler.cxx == 'icpx' && '. /opt/intel/oneapi/setvars.sh &&' || ''}} cmake -B ${{env.BUILD_DIR}} + -DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}" -DCMAKE_BUILD_TYPE=Debug -DUMF_BUILD_SHARED_LIBRARY=OFF -DCMAKE_C_COMPILER=${{matrix.compiler.c}} @@ -103,7 +105,7 @@ jobs: uses: lukka/run-vcpkg@5e0cab206a5ea620130caf672fce3e4a6b5666a1 # v11.5 with: vcpkgGitCommitId: 3dd44b931481d7a8e9ba412621fa810232b66289 - vcpkgDirectory: ${{github.workspace}}/build/vcpkg + vcpkgDirectory: ${{env.BUILD_DIR}}/vcpkg vcpkgJsonGlob: '**/vcpkg.json' - name: Install dependencies @@ -115,6 +117,7 @@ jobs: run: > cmake -B ${{env.BUILD_DIR}} + -DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}" -DCMAKE_C_COMPILER=${{matrix.compiler.c}} -DCMAKE_CXX_COMPILER=${{matrix.compiler.cxx}} -DCMAKE_PREFIX_PATH="${{env.VCPKG_PATH}}" From d8583b4e3d8742976a36d9e6988f76af76a828b5 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Mon, 15 Jul 2024 15:45:05 +0200 Subject: [PATCH 12/52] Do not run cmake inside ctest tests Signed-off-by: Lukasz Dorau --- test/CMakeLists.txt | 4 ++-- test/test_examples.sh | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index c005d8038..890ac5572 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -431,7 +431,7 @@ if(LINUX NAME umf_standalone_examples COMMAND ${UMF_CMAKE_SOURCE_DIR}/test/test_examples.sh - ${UMF_CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR} - ${CMAKE_BINARY_DIR}/install ${EXAMPLES} + ${UMF_CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR} ${CMAKE_INSTALL_PREFIX} + ${EXAMPLES} WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) endif() diff --git a/test/test_examples.sh b/test/test_examples.sh index 54b834527..9331b1d06 100755 --- a/test/test_examples.sh +++ b/test/test_examples.sh @@ -47,7 +47,6 @@ cd ${BUILD_DIR} echo "DIR=$(pwd)" set -x -cmake .. -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" make -j$(nproc) install set +x From 8ff8d6e0fcf698e2801f50be6ea1478ccb6e586c Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Mon, 15 Jul 2024 16:57:19 +0200 Subject: [PATCH 13/52] Define UMF_SRC_VERSION in the proxy library Define UMF_SRC_VERSION in the proxy library. It enables printing out the source version in util_log_init(). Signed-off-by: Lukasz Dorau --- src/CMakeLists.txt | 2 +- src/proxy_lib/CMakeLists.txt | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4c29d3437..a856ac826 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -81,7 +81,7 @@ set(UMF_SOURCES_WINDOWS libumf_windows.c) # Compile definitions for UMF library. # # TODO: Cleanup the compile definitions across all the CMake files -set(UMF_PRIVATE_COMPILE_DEFINITIONS "-DUMF_SRC_VERSION=${UMF_SRC_VERSION}") +set(UMF_PRIVATE_COMPILE_DEFINITIONS UMF_SRC_VERSION=${UMF_SRC_VERSION}) set(UMF_SOURCES_COMMON_LINUX_MACOSX provider/provider_os_memory.c diff --git a/src/proxy_lib/CMakeLists.txt b/src/proxy_lib/CMakeLists.txt index af3f8cb60..0d36e0258 100644 --- a/src/proxy_lib/CMakeLists.txt +++ b/src/proxy_lib/CMakeLists.txt @@ -39,6 +39,8 @@ add_library(${PROJECT_NAME}::proxy ALIAS umf_proxy) target_link_directories(umf_proxy PRIVATE ${LIBHWLOC_LIBRARY_DIRS}) +target_compile_definitions(umf_proxy PRIVATE UMF_SRC_VERSION=${UMF_SRC_VERSION}) + if(PROXY_LIB_USES_SCALABLE_POOL) target_compile_definitions(umf_proxy PRIVATE PROXY_LIB_USES_SCALABLE_POOL=1) elseif(PROXY_LIB_USES_JEMALLOC_POOL) From 8307f5158c81902871871814e61846983b664e12 Mon Sep 17 00:00:00 2001 From: Krzysztof Filipek Date: Tue, 16 Jul 2024 11:34:09 +0200 Subject: [PATCH 14/52] [CI] Add branch context to the Coverity Since this PR Coverity can also be executed for another branch. Previous solutions always cloned the main branch. --- .github/workflows/coverity.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml index 4219b8832..8bc624f87 100644 --- a/.github/workflows/coverity.yml +++ b/.github/workflows/coverity.yml @@ -22,6 +22,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: ${{ github.ref }} - name: Install apt packages run: | From cda61b04ca459215bebd40ef3aff1713af0d94cc Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Tue, 16 Jul 2024 13:38:08 +0200 Subject: [PATCH 15/52] Disable warning 28251: Inconsistent annotation for _BitScanForward Disable warning 28251: Inconsistent annotation for _BitScanForward in utils_concurrency.h. Signed-off-by: Lukasz Dorau --- src/utils/utils_concurrency.h | 4 ++++ src/utils/utils_windows_intrin.h | 26 ++++++++++++++++++++++++++ src/utils/utils_windows_math.c | 14 +------------- 3 files changed, 31 insertions(+), 13 deletions(-) create mode 100644 src/utils/utils_windows_intrin.h diff --git a/src/utils/utils_concurrency.h b/src/utils/utils_concurrency.h index ad3737f68..dcc67dc42 100644 --- a/src/utils/utils_concurrency.h +++ b/src/utils/utils_concurrency.h @@ -14,6 +14,10 @@ #ifdef _WIN32 #include + +#include "utils_windows_intrin.h" + +#pragma intrinsic(_BitScanForward64) #else #include diff --git a/src/utils/utils_windows_intrin.h b/src/utils/utils_windows_intrin.h new file mode 100644 index 000000000..23b2e5d7b --- /dev/null +++ b/src/utils/utils_windows_intrin.h @@ -0,0 +1,26 @@ +/* + * + * Copyright (C) 2024 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + +#ifndef UMF_UTILS_WINDOWS_INTRIN_H +#define UMF_UTILS_WINDOWS_INTRIN_H 1 + +#ifdef _WIN32 + +// Disable warning 28251: "inconsistent annotation for function" thrown in +// intrin.h, as we do not want to modify this file. +#pragma warning(push) +#pragma warning(disable : 28251) + +#include + +#pragma warning(pop) + +#endif /* _WIN32 */ + +#endif /* UMF_UTILS_WINDOWS_INTRIN_H */ diff --git a/src/utils/utils_windows_math.c b/src/utils/utils_windows_math.c index 81c1490d8..07c4c9978 100644 --- a/src/utils/utils_windows_math.c +++ b/src/utils/utils_windows_math.c @@ -8,19 +8,7 @@ */ #include "utils_math.h" - -// disable warning 28251: "inconsistent annotation for function" thrown in -// intrin.h, as we do not want to modify this file -#if defined(_MSC_VER) -#pragma warning(push) -#pragma warning(disable : 28251) -#endif // _MSC_VER - -#include - -#if defined(_MSC_VER) -#pragma warning(pop) -#endif // _MSC_VER +#include "utils_windows_intrin.h" #pragma intrinsic(_BitScanReverse) From 996273c1153d1ece82ca5f54150e77949f7494bd Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Wed, 17 Jul 2024 16:33:07 +0200 Subject: [PATCH 16/52] Add disable_provider_free parameter to pool jemalloc Add the disable_provider_free parameter to pool jemalloc. It should be set to true if umfMemoryProviderFree() should never be called. This option will be needed by the future memory providers that will not provide the free() op. Signed-off-by: Lukasz Dorau --- include/umf/pools/pool_jemalloc.h | 7 +++++++ src/pool/pool_jemalloc.c | 24 ++++++++++++++++++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/include/umf/pools/pool_jemalloc.h b/include/umf/pools/pool_jemalloc.h index c30df6509..dfd75746b 100644 --- a/include/umf/pools/pool_jemalloc.h +++ b/include/umf/pools/pool_jemalloc.h @@ -14,8 +14,15 @@ extern "C" { #endif +#include #include +/// @brief Configuration of Jemalloc Pool +typedef struct umf_jemalloc_pool_params_t { + /// Set to true if umfMemoryProviderFree() should never be called. + bool disable_provider_free; +} umf_jemalloc_pool_params_t; + umf_memory_pool_ops_t *umfJemallocPoolOps(void); #ifdef __cplusplus diff --git a/src/pool/pool_jemalloc.c b/src/pool/pool_jemalloc.c index ef72ca2f6..094ceeaf7 100644 --- a/src/pool/pool_jemalloc.c +++ b/src/pool/pool_jemalloc.c @@ -37,6 +37,8 @@ typedef struct jemalloc_memory_pool_t { umf_memory_provider_handle_t provider; unsigned int arena_index; // index of jemalloc arena + // set to true if umfMemoryProviderFree() should never be called + bool disable_provider_free; } jemalloc_memory_pool_t; static __TLS umf_result_t TLS_last_allocation_error; @@ -80,7 +82,9 @@ static void *arena_extent_alloc(extent_hooks_t *extent_hooks, void *new_addr, } if (new_addr != NULL && ptr != new_addr) { - umfMemoryProviderFree(pool->provider, ptr, size); + if (!pool->disable_provider_free) { + umfMemoryProviderFree(pool->provider, ptr, size); + } return NULL; } @@ -114,6 +118,10 @@ static void arena_extent_destroy(extent_hooks_t *extent_hooks, void *addr, jemalloc_memory_pool_t *pool = get_pool_by_arena_index(arena_ind); + if (pool->disable_provider_free) { + return; + } + umf_result_t ret; ret = umfMemoryProviderFree(pool->provider, addr, size); if (ret != UMF_RESULT_SUCCESS) { @@ -136,6 +144,10 @@ static bool arena_extent_dalloc(extent_hooks_t *extent_hooks, void *addr, jemalloc_memory_pool_t *pool = get_pool_by_arena_index(arena_ind); + if (pool->disable_provider_free) { + return true; // opt-out from deallocation + } + umf_result_t ret; ret = umfMemoryProviderFree(pool->provider, addr, size); if (ret != UMF_RESULT_SUCCESS) { @@ -388,7 +400,9 @@ static umf_result_t op_initialize(umf_memory_provider_handle_t provider, void *params, void **out_pool) { assert(provider); assert(out_pool); - (void)params; // unused + + umf_jemalloc_pool_params_t *je_params = + (umf_jemalloc_pool_params_t *)params; extent_hooks_t *pHooks = &arena_extent_hooks; size_t unsigned_size = sizeof(unsigned); @@ -402,6 +416,12 @@ static umf_result_t op_initialize(umf_memory_provider_handle_t provider, pool->provider = provider; + if (je_params) { + pool->disable_provider_free = je_params->disable_provider_free; + } else { + pool->disable_provider_free = false; + } + unsigned arena_index; err = je_mallctl("arenas.create", (void *)&arena_index, &unsigned_size, NULL, 0); From a7b17ce2f414b326c2bafc87aa9cb22a7df66b5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Wed, 17 Jul 2024 09:56:57 +0200 Subject: [PATCH 17/52] [CI] Always fetch the whole repo content to assure a git tag (setting the version) is available. --- .github/workflows/bandit.yml | 2 ++ .github/workflows/basic.yml | 10 ++++++++++ .github/workflows/benchmarks.yml | 2 ++ .github/workflows/codeql.yml | 2 ++ .github/workflows/coverity.yml | 1 + .github/workflows/docs.yml | 2 ++ .github/workflows/fast.yml | 2 ++ .github/workflows/gpu.yml | 2 ++ .github/workflows/multi_numa.yml | 2 ++ .github/workflows/nightly.yml | 4 ++++ .github/workflows/pr_push.yml | 4 ++++ .github/workflows/proxy_lib.yml | 2 ++ .github/workflows/qemu.yml | 3 +++ .github/workflows/sanitizers.yml | 4 ++++ .github/workflows/scorecard.yml | 2 ++ .github/workflows/spellcheck.yml | 2 ++ .github/workflows/trivy.yml | 2 ++ .github/workflows/valgrind.yml | 2 ++ 18 files changed, 50 insertions(+) diff --git a/.github/workflows/bandit.yml b/.github/workflows/bandit.yml index 995b5da22..80b383665 100644 --- a/.github/workflows/bandit.yml +++ b/.github/workflows/bandit.yml @@ -24,6 +24,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 - name: Install Bandit run: python3 -m pip install bandit diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml index 0853fc848..31cfba774 100644 --- a/.github/workflows/basic.yml +++ b/.github/workflows/basic.yml @@ -74,6 +74,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 - name: Install apt packages run: | @@ -180,6 +182,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 - name: Initialize vcpkg uses: lukka/run-vcpkg@5e0cab206a5ea620130caf672fce3e4a6b5666a1 # v11.5 @@ -250,6 +254,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 - name: Configure build run: > @@ -290,6 +296,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 - name: Configure build run: > @@ -325,6 +333,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 - name: Install Python requirements run: python3 -m pip install -r third_party/requirements.txt diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index e6268a0fc..de48173bf 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -27,6 +27,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 - name: Install apt packages if: matrix.os == 'ubuntu-latest' diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index d59a45089..b449eb23e 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -43,6 +43,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 - name: Initialize CodeQL uses: github/codeql-action/init@b7bf0a3ed3ecfa44160715d7c442788f65f0f923 # v3.23.2 diff --git a/.github/workflows/coverity.yml b/.github/workflows/coverity.yml index 8bc624f87..f6fe2ad26 100644 --- a/.github/workflows/coverity.yml +++ b/.github/workflows/coverity.yml @@ -24,6 +24,7 @@ jobs: uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: ref: ${{ github.ref }} + fetch-depth: 0 - name: Install apt packages run: | diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index c768746fe..4fdd89766 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -21,6 +21,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 - name: Install doxygen run: | diff --git a/.github/workflows/fast.yml b/.github/workflows/fast.yml index 76f4f09ab..3bcfcce7d 100644 --- a/.github/workflows/fast.yml +++ b/.github/workflows/fast.yml @@ -59,6 +59,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 - name: Initialize vcpkg if: matrix.os == 'windows-latest' diff --git a/.github/workflows/gpu.yml b/.github/workflows/gpu.yml index 444ebdfe4..3024b9f7e 100644 --- a/.github/workflows/gpu.yml +++ b/.github/workflows/gpu.yml @@ -36,6 +36,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 - name: Get information about platform if: matrix.os == 'Ubuntu' diff --git a/.github/workflows/multi_numa.yml b/.github/workflows/multi_numa.yml index 4b8079c61..a9433018e 100644 --- a/.github/workflows/multi_numa.yml +++ b/.github/workflows/multi_numa.yml @@ -20,6 +20,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 - name: Get information about platform run: .github/scripts/get_system_info.sh diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index c03cd0d0e..28a07c3a9 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -24,6 +24,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 - name: Install apt packages run: | @@ -59,6 +61,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 - name: Install apt packages run: | diff --git a/.github/workflows/pr_push.yml b/.github/workflows/pr_push.yml index 525e4ac9e..c35664a56 100644 --- a/.github/workflows/pr_push.yml +++ b/.github/workflows/pr_push.yml @@ -22,6 +22,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 - name: Install apt packages run: | @@ -55,6 +57,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 - name: Install doxygen run: | diff --git a/.github/workflows/proxy_lib.yml b/.github/workflows/proxy_lib.yml index 43309b153..8d73569f0 100644 --- a/.github/workflows/proxy_lib.yml +++ b/.github/workflows/proxy_lib.yml @@ -25,6 +25,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 - name: Install apt packages run: | diff --git a/.github/workflows/qemu.yml b/.github/workflows/qemu.yml index 946a7edd2..f8916c7de 100644 --- a/.github/workflows/qemu.yml +++ b/.github/workflows/qemu.yml @@ -17,6 +17,9 @@ jobs: steps: - name: Checkout uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 + - name: Enable KVM run: | echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules diff --git a/.github/workflows/sanitizers.yml b/.github/workflows/sanitizers.yml index d3a4cd4b8..06ad492eb 100644 --- a/.github/workflows/sanitizers.yml +++ b/.github/workflows/sanitizers.yml @@ -23,6 +23,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 - name: Install apt packages run: | @@ -88,6 +90,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 # Use the latest MSVC toolset available, when compiling UMF with ASan. # Running binaries compiled with older toolsets results in a diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 5c38a247b..b28bb150e 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -27,6 +27,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 - name: Run analysis uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1 diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/spellcheck.yml index ec2e2d2b1..07265fc17 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/spellcheck.yml @@ -14,6 +14,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 - name: Run a spell check uses: crate-ci/typos@b63f421581dce830bda2f597a678cb7776b41877 # v1.18.2 diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index 2839ef1c1..1c3e63120 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -31,6 +31,8 @@ jobs: steps: - name: Clone the git repo uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 - name: Run Trivy uses: aquasecurity/trivy-action@84384bd6e777ef152729993b8145ea352e9dd3ef # v0.17.0 diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 99552f9a1..0ca9bf779 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -17,6 +17,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 - name: Install apt packages run: | From 30c59d0c88c6c410f24b5b1b540530b8e2b9356c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Fri, 12 Jul 2024 17:58:42 +0200 Subject: [PATCH 18/52] [CMake] Update versioning Properly set version based on our git tags. For CMake's sake we use major.minor.patch For Windows dll metadata we use major.minor.build.revision and additional variables. --- CMakeLists.txt | 35 ++++--- RELEASE_STEPS.md | 2 +- cmake/helpers.cmake | 188 +++++++++++++++++++++++++--------- src/CMakeLists.txt | 2 +- src/libumf.rc.in | 34 ++++-- src/proxy_lib/CMakeLists.txt | 2 +- src/proxy_lib/proxy_lib.rc.in | 34 ++++-- src/utils/CMakeLists.txt | 7 +- src/utils/utils_log.c | 14 ++- test/utils/utils_log.cpp | 1 + 10 files changed, 228 insertions(+), 91 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0957649eb..e84a45097 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,32 +3,35 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception cmake_minimum_required(VERSION 3.14.0 FATAL_ERROR) -project( - umf - VERSION 0.1.0 - LANGUAGES C) - # needed when UMF is used as an external project set(UMF_CMAKE_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) list(APPEND CMAKE_MODULE_PATH "${UMF_CMAKE_SOURCE_DIR}/cmake") -include(${UMF_CMAKE_SOURCE_DIR}/cmake/helpers.cmake) +include(helpers) + +# We use semver aligned version, set via git tags. We parse git output to +# establih the version of UMF to be used in CMake, Win dll's, and within the +# code (e.g. in logger). We have 3-component releases (e.g. 1.5.1) plus release +# candidates and git info. Function below sets all variables related to version. +set_version_variables() +message(STATUS "UMF version: ${UMF_VERSION}") + +# version we set in CMake is abbreviated just to major.minor.patch +project( + umf + VERSION ${UMF_CMAKE_VERSION} + LANGUAGES C) + +if(CMAKE_PROJECT_VERSION_PATCH GREATER 0) + # set extra variable for Windows dll metadata + set(UMF_VERSION_BUGFIX 1) +endif() include(CTest) include(CMakePackageConfigHelpers) include(GNUInstallDirs) find_package(PkgConfig) -# CMAKE_PROJECT_VERSION[_MAJOR|_MINOR|_PATCH] variables are set via 'project' -# command. They cannot contain any "pre-release" part, though. We use custom -# "UMF_SRC_VERSION" to store more accurate (source) version - this var should be -# used, e.g., for creating packages. -set_source_version() -message( - STATUS - "UMF version: ${CMAKE_PROJECT_VERSION} (source version: ${UMF_SRC_VERSION})" -) - # Build Options option(UMF_BUILD_SHARED_LIBRARY "Build UMF as shared library" OFF) option(UMF_BUILD_LEVEL_ZERO_PROVIDER "Build Level Zero memory provider" ON) diff --git a/RELEASE_STEPS.md b/RELEASE_STEPS.md index ad463caf0..e88ca9c2d 100644 --- a/RELEASE_STEPS.md +++ b/RELEASE_STEPS.md @@ -40,7 +40,7 @@ Do changes for a release: - Add an entry to ChangeLog, remember to change the day of the week in the release date - For major releases mention API and ABI compatibility with the previous release - Update project's version in a few places: - - Set the new $VERSION in `project` function in the top-level `CMakeLists.txt` + - For major and minor releases: `UMF_VERSION_CURRENT` in `include/umf/base.h` (the API version) - `release` variable in `scripts/docs_config/conf.py` (for docs) - `UMF_VERSION` variable in `.github/workflows/basic.yml` (for installation test) - For major releases update ABI version in `.map` and `.def` files diff --git a/cmake/helpers.cmake b/cmake/helpers.cmake index f401cfb48..bb9b703d8 100644 --- a/cmake/helpers.cmake +++ b/cmake/helpers.cmake @@ -10,68 +10,158 @@ include(CheckCCompilerFlag) include(CheckCXXCompilerFlag) -# src version shows the current version, as reported by 'git describe', unless -# 'git' is not available, then fall back to the top-level defined version -function(set_source_version) +# This function establishes version variables based on the git describe output. +# If there's no git available in the system, the version will be set to "0.0.0". +# If git reports only a hash, the version will be set to "0.0.0.git.". +# Otherwise we'll use 3-component version: major.minor.patch, just for CMake's +# sake. A few extra variables will be set for Win dll metadata. +# +# Important note: CMake does not support rc or git information. According to +# semver rules, 1.5.1-rc1 should be less than 1.5.1, but it seems hard to +# achieve such comparison in CMake. So, for CMake's sake we only set 3-component +# version in variable "UMF_CMAKE_VERSION", ignoring the rc and git information. +# It's only used to set SOVERSION and creating "umf-config.cmake" file. +# +# For Windows versioning in dll metadata, we use 4-component version plus a few +# additional variables. REVISION has to be an integer and is calculated as: +# REVISION = rc_no * 1000 + git_commit_no (commits count after the last release) +# +# For all other usages (beside CMake and Win dll), we use semver aligned version +# "UMF_VERSION", which is in line with our tags (e.g. "1.5.0-rc2"). +# +# Example parsing of git output: +# cmake-format: off +# +-----------------------+-------+-------+-------+----------+--------+---------+------------+ +# | \ CMake:| Major | Minor | Patch | | | | | +# +-----------------------+-------+-------+-------+----------+--------+---------+------------+ +# | git describe \ Win32:| MAJOR | MINOR | BUILD | REVISION | BUGFIX | PRIVATE | PRERELEASE | +# +-----------------------+-------+-------+-------+----------+--------+---------+------------+ +# | 1.5.0-rc2-0-gb8f7a32 | 1 | 5 | 0 | 2000 | | | true | +# | 1.5.0-rc2 | 1 | 5 | 0 | 2000 | | | true | +# | 1.5.0-rc3-6-gb8f7a32 | 1 | 5 | 0 | 3006 | | true | true | +# | 1.5.0-0-gb8f7a32 | 1 | 5 | 0 | 0 | | | | +# | 1.5.0 | 1 | 5 | 0 | 0 | | | | +# | 1.5.0-6-123345678 | 1 | 5 | 0 | 6 | | true | | +# | 1.5.2-rc1-0-gb8f7a32 | 1 | 5 | 2 | 1000 | true | | true | +# | 1.5.2-rc4-6-gb8f7a32 | 1 | 5 | 2 | 4006 | true | true | true | +# | 1.5.2-0-gb8f7a32 | 1 | 5 | 2 | 0 | true | | | +# | 1.5.2-6-gb8f7a32 | 1 | 5 | 2 | 6 | true | true | | +# | gb8f7a32 | 0 | 0 | 0 | 0 | | true | | +# | ? (no git) | 0 | 0 | 0 | 0 | | true | | +# +-----------------------+-------+-------+-------+----------+--------+---------+------------+ +# cmake-format: on +function(set_version_variables) + # default values + set(UMF_VERSION_PRERELEASE + 0 + PARENT_SCOPE) + set(UMF_VERSION_PRIVATE + 1 + PARENT_SCOPE) + set(UMF_VERSION_BUGFIX + 0 + PARENT_SCOPE) + set(UMF_VERSION_REVISION + 0 + PARENT_SCOPE) + set(UMF_CMAKE_VERSION + "0.0.0" + PARENT_SCOPE) + set(UMF_VERSION + "0.0.0" + PARENT_SCOPE) + execute_process( COMMAND git describe --always OUTPUT_VARIABLE GIT_VERSION WORKING_DIRECTORY ${UMF_CMAKE_SOURCE_DIR} OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) - if(GIT_VERSION) - # 1.5.0 - we're on a tag - string(REGEX MATCHALL "\^([0-9]+\.[0-9]+\.[0-9]+)\$" MATCHES - ${GIT_VERSION}) - if(MATCHES) - set(UMF_SRC_VERSION - "${CMAKE_MATCH_1}" - PARENT_SCOPE) - return() - endif() + if(NOT GIT_VERSION) + # no git or it reported no version. Use default ver: "0.0.0" + return() + endif() - # 1.5.0-rc1 - we're on a RC tag - string(REGEX MATCHALL "\^([0-9]+\.[0-9]+\.[0-9]+\-rc[0-9]+)\$" MATCHES - ${GIT_VERSION}) - if(MATCHES) - set(UMF_SRC_VERSION - "${CMAKE_MATCH_1}" - PARENT_SCOPE) - return() - endif() + # v1.5.0 - we're exactly on a tag -> UMF ver: "1.5.0" + string(REGEX MATCHALL "\^v([0-9]+\.[0-9]+\.[0-9]+)\$" MATCHES + ${GIT_VERSION}) + if(MATCHES) + set(UMF_VERSION + "${CMAKE_MATCH_1}" + PARENT_SCOPE) + set(UMF_CMAKE_VERSION + "${CMAKE_MATCH_1}" + PARENT_SCOPE) + set(UMF_VERSION_PRIVATE + 0 + PARENT_SCOPE) + return() + endif() - # 1.5.0-rc1-19-gb8f78a329 -> 1.5.0-rc1.git19.gb8f78a329 - string(REGEX MATCHALL "([0-9.]*)-rc([0-9]*)-([0-9]*)-([0-9a-g]*)" - MATCHES ${GIT_VERSION}) - if(MATCHES) - set(UMF_SRC_VERSION - "${CMAKE_MATCH_1}-rc${CMAKE_MATCH_2}.git${CMAKE_MATCH_3}.${CMAKE_MATCH_4}" - PARENT_SCOPE) - return() - endif() + # v1.5.0-rc1 - we're on a RC tag -> UMF ver: "1.5.0-rc1" + string(REGEX MATCHALL "\^v([0-9]+\.[0-9]+\.[0-9]+)-rc([0-9]+)\$" MATCHES + ${GIT_VERSION}) + if(MATCHES) + set(UMF_VERSION + "${CMAKE_MATCH_1}-rc${CMAKE_MATCH_2}" + PARENT_SCOPE) + set(UMF_CMAKE_VERSION + "${CMAKE_MATCH_1}" + PARENT_SCOPE) + math(EXPR revision "${CMAKE_MATCH_2} * 1000") + set(UMF_VERSION_REVISION + ${revision} + PARENT_SCOPE) + set(UMF_VERSION_PRERELEASE + 1 + PARENT_SCOPE) + set(UMF_VERSION_PRIVATE + 0 + PARENT_SCOPE) + return() + endif() - # 1.5.0-19-gb8f78a329 -> 1.5.0-git19.gb8f78a329 - string(REGEX MATCHALL "([0-9.]*)-([0-9]*)-([0-9a-g]*)" MATCHES - ${GIT_VERSION}) - if(MATCHES) - set(UMF_SRC_VERSION - "${CMAKE_MATCH_1}-git${CMAKE_MATCH_2}.${CMAKE_MATCH_3}" - PARENT_SCOPE) - return() - endif() + # v1.5.0-rc1-19-gb8f7a32 -> UMF ver: "1.5.0-rc1.git19.gb8f7a32" + string(REGEX MATCHALL "v([0-9.]*)-rc([0-9]*)-([0-9]*)-([0-9a-g]*)" MATCHES + ${GIT_VERSION}) + if(MATCHES) + set(UMF_VERSION + "${CMAKE_MATCH_1}-rc${CMAKE_MATCH_2}.git${CMAKE_MATCH_3}.${CMAKE_MATCH_4}" + PARENT_SCOPE) + set(UMF_CMAKE_VERSION + "${CMAKE_MATCH_1}" + PARENT_SCOPE) + math(EXPR revision "${CMAKE_MATCH_2} * 1000 + ${CMAKE_MATCH_3}") + set(UMF_VERSION_REVISION + ${revision} + PARENT_SCOPE) + set(UMF_VERSION_PRERELEASE + 1 + PARENT_SCOPE) + return() + endif() - # no full version is available (e.g. only a hash commit) or a pattern - # was not recognized - set(UMF_SRC_VERSION - "${CMAKE_PROJECT_VERSION}.git.${GIT_VERSION}" + # v1.5.0-19-gb8f7a32 -> UMF ver: "1.5.0-git19.gb8f7a32" + string(REGEX MATCHALL "v([0-9.]*)-([0-9]*)-([0-9a-g]*)" MATCHES + ${GIT_VERSION}) + if(MATCHES) + set(UMF_VERSION + "${CMAKE_MATCH_1}-git${CMAKE_MATCH_2}.${CMAKE_MATCH_3}" + PARENT_SCOPE) + set(UMF_CMAKE_VERSION + "${CMAKE_MATCH_1}" PARENT_SCOPE) - else() - # git reported no version. Use version set up in the top-level CMake - # with a "devel" suffix - set(UMF_SRC_VERSION - "${CMAKE_PROJECT_VERSION}-devel" + set(UMF_VERSION_REVISION + ${CMAKE_MATCH_2} PARENT_SCOPE) + return() endif() + + # no full version is available (e.g. only a hash commit) or a pattern was + # not recognized -> UMF ver: "0.0.0.git." + set(UMF_VERSION + "0.0.0.git.${GIT_VERSION}" + PARENT_SCOPE) endfunction() # Sets ${ret} to version of program specified by ${name} in major.minor format diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a856ac826..c4acf46c6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -81,7 +81,7 @@ set(UMF_SOURCES_WINDOWS libumf_windows.c) # Compile definitions for UMF library. # # TODO: Cleanup the compile definitions across all the CMake files -set(UMF_PRIVATE_COMPILE_DEFINITIONS UMF_SRC_VERSION=${UMF_SRC_VERSION}) +set(UMF_PRIVATE_COMPILE_DEFINITIONS UMF_VERSION=${UMF_VERSION}) set(UMF_SOURCES_COMMON_LINUX_MACOSX provider/provider_os_memory.c diff --git a/src/libumf.rc.in b/src/libumf.rc.in index f4ddd3653..9d0677f6d 100644 --- a/src/libumf.rc.in +++ b/src/libumf.rc.in @@ -8,18 +8,38 @@ #include "umf/base.h" -#define UMF_VERNUMBERS @CMAKE_PROJECT_VERSION_MAJOR@,@CMAKE_PROJECT_VERSION_MINOR@,@CMAKE_PROJECT_VERSION_PATCH@,0 -#define UMF_VERSION "@CMAKE_PROJECT_VERSION_MAJOR@.@CMAKE_PROJECT_VERSION_MINOR@" +#define UMF_VERNUMBERS @CMAKE_PROJECT_VERSION_MAJOR@,@CMAKE_PROJECT_VERSION_MINOR@,@CMAKE_PROJECT_VERSION_PATCH@,@UMF_VERSION_REVISION@ +#define UMF_VERSION "@UMF_VERSION@" + +#ifdef _DEBUG +#define VERSION_DEBUG VS_FF_DEBUG +#else +#define VERSION_DEBUG 0 +#endif + +#if @UMF_VERSION_PRERELEASE@ +#define VERSION_PRERELEASE VS_FF_PRERELEASE +#else +#define VERSION_PRERELEASE 0 +#endif + +#if @UMF_VERSION_PRIVATE@ +#define VERSION_PRIVATE VS_FF_PRIVATEBUILD +#else +#define VERSION_PRIVATE 0 +#endif + +#if @UMF_VERSION_BUGFIX@ +#define VERSION_PATCHED VS_FF_PATCHED +#else +#define VERSION_PATCHED 0 +#endif VS_VERSION_INFO VERSIONINFO FILEVERSION UMF_VERNUMBERS PRODUCTVERSION UMF_VERNUMBERS FILEFLAGSMASK VS_FFI_FILEFLAGSMASK -#ifdef _DEBUG - FILEFLAGS VS_FF_DEBUG -#else - FILEFLAGS 0 -#endif + FILEFLAGS (VERSION_DEBUG | VERSION_PRIVATE | VERSION_PRERELEASE | VERSION_PATCHED) FILEOS VOS__WINDOWS32 FILETYPE VFT_DLL FILESUBTYPE 0 diff --git a/src/proxy_lib/CMakeLists.txt b/src/proxy_lib/CMakeLists.txt index 0d36e0258..6b276ebfd 100644 --- a/src/proxy_lib/CMakeLists.txt +++ b/src/proxy_lib/CMakeLists.txt @@ -39,7 +39,7 @@ add_library(${PROJECT_NAME}::proxy ALIAS umf_proxy) target_link_directories(umf_proxy PRIVATE ${LIBHWLOC_LIBRARY_DIRS}) -target_compile_definitions(umf_proxy PRIVATE UMF_SRC_VERSION=${UMF_SRC_VERSION}) +target_compile_definitions(umf_proxy PRIVATE UMF_VERSION=${UMF_VERSION}) if(PROXY_LIB_USES_SCALABLE_POOL) target_compile_definitions(umf_proxy PRIVATE PROXY_LIB_USES_SCALABLE_POOL=1) diff --git a/src/proxy_lib/proxy_lib.rc.in b/src/proxy_lib/proxy_lib.rc.in index d8f3fec27..66910afc4 100644 --- a/src/proxy_lib/proxy_lib.rc.in +++ b/src/proxy_lib/proxy_lib.rc.in @@ -8,18 +8,38 @@ #include "umf/base.h" -#define UMF_VERNUMBERS @CMAKE_PROJECT_VERSION_MAJOR@,@CMAKE_PROJECT_VERSION_MINOR@,@CMAKE_PROJECT_VERSION_PATCH@,0 -#define UMF_VERSION "@CMAKE_PROJECT_VERSION_MAJOR@.@CMAKE_PROJECT_VERSION_MINOR@" +#define UMF_VERNUMBERS @CMAKE_PROJECT_VERSION_MAJOR@,@CMAKE_PROJECT_VERSION_MINOR@,@CMAKE_PROJECT_VERSION_PATCH@,@UMF_VERSION_REVISION@ +#define UMF_VERSION "@UMF_VERSION@" + +#ifdef _DEBUG +#define VERSION_DEBUG VS_FF_DEBUG +#else +#define VERSION_DEBUG 0 +#endif + +#if @UMF_VERSION_PRERELEASE@ +#define VERSION_PRERELEASE VS_FF_PRERELEASE +#else +#define VERSION_PRERELEASE 0 +#endif + +#if @UMF_VERSION_PRIVATE@ +#define VERSION_PRIVATE VS_FF_PRIVATEBUILD +#else +#define VERSION_PRIVATE 0 +#endif + +#if @UMF_VERSION_BUGFIX@ +#define VERSION_PATCHED VS_FF_PATCHED +#else +#define VERSION_PATCHED 0 +#endif VS_VERSION_INFO VERSIONINFO FILEVERSION UMF_VERNUMBERS PRODUCTVERSION UMF_VERNUMBERS FILEFLAGSMASK VS_FFI_FILEFLAGSMASK -#ifdef _DEBUG - FILEFLAGS VS_FF_DEBUG -#else - FILEFLAGS 0 -#endif + FILEFLAGS (VERSION_DEBUG | VERSION_PRIVATE | VERSION_PRERELEASE | VERSION_PATCHED) FILEOS VOS__WINDOWS32 FILETYPE VFT_DLL FILESUBTYPE 0 diff --git a/src/utils/CMakeLists.txt b/src/utils/CMakeLists.txt index 051cdbc35..c8f9d8f05 100644 --- a/src/utils/CMakeLists.txt +++ b/src/utils/CMakeLists.txt @@ -52,5 +52,10 @@ target_include_directories( $) if(USE_VALGRIND) - target_compile_definitions(umf_utils INTERFACE UMF_VG_ENABLED=1) + set(UMF_UTILS_INTERFACE_DEFS "UMF_VG_ENABLED=1") endif() +if(WINDOWS) + set(UMF_UTILS_INTERFACE_DEFS ${UMF_UTILS_INTERFACE_DEFS} + "UMF_VERSION=${UMF_VERSION}") +endif() +target_compile_definitions(umf_utils INTERFACE ${UMF_UTILS_INTERFACE_DEFS}) diff --git a/src/utils/utils_log.c b/src/utils/utils_log.c index d8e8023c6..ca16014f0 100644 --- a/src/utils/utils_log.c +++ b/src/utils/utils_log.c @@ -305,20 +305,18 @@ void util_log_init(void) { loggerConfig.flushLevel = LOG_FATAL; } -#ifdef UMF_SRC_VERSION +#ifdef UMF_VERSION // convert a define to a C string #define STR_(X) #X #define STR(X) STR_(X) -#define STR_UMF_SRC_VERSION "src version: " STR(UMF_SRC_VERSION) ", " -#else /* !UMF_SRC_VERSION */ -#define STR_UMF_SRC_VERSION "" -#endif /* !UMF_SRC_VERSION */ +#define STR_UMF_VERSION "UMF version: " STR(UMF_VERSION) ", " +#else /* !UMF_VERSION */ +#error "UMF_VERSION not defined!" +#endif /* !UMF_VERSION */ - int umf_ver = umfGetCurrentVersion(); LOG_INFO( - "Logger enabled (umf version: %i.%i, " STR_UMF_SRC_VERSION + "Logger enabled (" STR_UMF_VERSION "level: %s, flush: %s, pid: %s, timestamp: %s)", - UMF_MAJOR_VERSION(umf_ver), UMF_MINOR_VERSION(umf_ver), level_to_str(loggerConfig.level), level_to_str(loggerConfig.flushLevel), bool_to_str(loggerConfig.pid), bool_to_str(loggerConfig.timestamp)); } diff --git a/test/utils/utils_log.cpp b/test/utils/utils_log.cpp index 235ae7816..3e899e685 100644 --- a/test/utils/utils_log.cpp +++ b/test/utils/utils_log.cpp @@ -107,6 +107,7 @@ const char *env_variable = ""; #define strerror_s(A, B, C) mock_strerror_windows(A, B, C) //getenv returns 'char *' not 'const char *' so we need explicit cast to drop const #define getenv(X) strstr(X, "UMF_LOG") ? (char *)env_variable : getenv(X) +#define UMF_VERSION "test version" #include "utils/utils_log.c" #undef util_env_var #undef fopen From e8eb45df895adfa967efbe25a94370979091cf4e Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Thu, 18 Jul 2024 10:08:44 +0200 Subject: [PATCH 19/52] [CMake] Fix compile definitions a little make UMF_VERSION a common umf definition, to be used everywhere. --- src/CMakeLists.txt | 16 ++++++++-------- src/pool/CMakeLists.txt | 3 ++- src/proxy_lib/CMakeLists.txt | 2 +- src/utils/CMakeLists.txt | 8 ++++---- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c4acf46c6..c261ac6ed 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -4,6 +4,11 @@ include(${UMF_CMAKE_SOURCE_DIR}/cmake/helpers.cmake) +# Compile definitions for UMF library. +# +# TODO: Cleanup the compile definitions across all the CMake files +set(UMF_COMMON_COMPILE_DEFINITIONS UMF_VERSION=${UMF_VERSION}) + if(UMF_BUILD_LEVEL_ZERO_PROVIDER) include(FetchContent) @@ -78,11 +83,6 @@ set(UMF_SOURCES_MACOSX libumf_linux.c) set(UMF_SOURCES_WINDOWS libumf_windows.c) -# Compile definitions for UMF library. -# -# TODO: Cleanup the compile definitions across all the CMake files -set(UMF_PRIVATE_COMPILE_DEFINITIONS UMF_VERSION=${UMF_VERSION}) - set(UMF_SOURCES_COMMON_LINUX_MACOSX provider/provider_os_memory.c provider/provider_os_memory_posix.c @@ -128,8 +128,8 @@ if(UMF_BUILD_SHARED_LIBRARY) LIBS ${UMF_LIBS} hwloc LINUX_MAP_FILE ${CMAKE_CURRENT_SOURCE_DIR}/libumf.map WINDOWS_DEF_FILE ${CMAKE_CURRENT_BINARY_DIR}/libumf.def) - set(UMF_PRIVATE_COMPILE_DEFINITIONS ${UMF_PRIVATE_COMPILE_DEFINITIONS} - "UMF_SHARED_LIBRARY") + set(UMF_COMMON_COMPILE_DEFINITIONS ${UMF_COMMON_COMPILE_DEFINITIONS} + "UMF_SHARED_LIBRARY") set_target_properties( umf PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_UMF_OUTPUT_DIRECTORY} @@ -149,7 +149,7 @@ endif() target_link_directories(umf PRIVATE ${UMF_PRIVATE_LIBRARY_DIRS}) -target_compile_definitions(umf PRIVATE ${UMF_PRIVATE_COMPILE_DEFINITIONS}) +target_compile_definitions(umf PRIVATE ${UMF_COMMON_COMPILE_DEFINITIONS}) if(UMF_BUILD_LEVEL_ZERO_PROVIDER) target_sources(umf PRIVATE provider/provider_level_zero.c) diff --git a/src/pool/CMakeLists.txt b/src/pool/CMakeLists.txt index 787cc1ea9..bdd196b04 100644 --- a/src/pool/CMakeLists.txt +++ b/src/pool/CMakeLists.txt @@ -4,10 +4,11 @@ if(UMF_BUILD_SHARED_LIBRARY) set(POOL_EXTRA_SRCS ${BA_SOURCES}) - set(POOL_COMPILE_DEFINITIONS UMF_SHARED_LIBRARY) set(POOL_EXTRA_LIBS $) endif() +set(POOL_COMPILE_DEFINITIONS ${UMF_COMMON_COMPILE_DEFINITIONS}) + # libumf_pool_disjoint if(UMF_BUILD_LIBUMF_POOL_DISJOINT) add_umf_library( diff --git a/src/proxy_lib/CMakeLists.txt b/src/proxy_lib/CMakeLists.txt index 6b276ebfd..379a454d0 100644 --- a/src/proxy_lib/CMakeLists.txt +++ b/src/proxy_lib/CMakeLists.txt @@ -39,7 +39,7 @@ add_library(${PROJECT_NAME}::proxy ALIAS umf_proxy) target_link_directories(umf_proxy PRIVATE ${LIBHWLOC_LIBRARY_DIRS}) -target_compile_definitions(umf_proxy PRIVATE UMF_VERSION=${UMF_VERSION}) +target_compile_definitions(umf_proxy PRIVATE ${UMF_COMMON_COMPILE_DEFINITIONS}) if(PROXY_LIB_USES_SCALABLE_POOL) target_compile_definitions(umf_proxy PRIVATE PROXY_LIB_USES_SCALABLE_POOL=1) diff --git a/src/utils/CMakeLists.txt b/src/utils/CMakeLists.txt index c8f9d8f05..78cd0e129 100644 --- a/src/utils/CMakeLists.txt +++ b/src/utils/CMakeLists.txt @@ -54,8 +54,8 @@ target_include_directories( if(USE_VALGRIND) set(UMF_UTILS_INTERFACE_DEFS "UMF_VG_ENABLED=1") endif() -if(WINDOWS) - set(UMF_UTILS_INTERFACE_DEFS ${UMF_UTILS_INTERFACE_DEFS} - "UMF_VERSION=${UMF_VERSION}") -endif() + +set(UMF_UTILS_INTERFACE_DEFS ${UMF_UTILS_INTERFACE_DEFS} + ${UMF_COMMON_COMPILE_DEFINITIONS}) + target_compile_definitions(umf_utils INTERFACE ${UMF_UTILS_INTERFACE_DEFS}) From f1cb2bcb8f181ba225251b4a74185d9d8db1eb31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Wed, 17 Jul 2024 16:20:36 +0200 Subject: [PATCH 20/52] [CI] Print the dll's metadata in fast build, to check correctness --- .github/workflows/fast.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/fast.yml b/.github/workflows/fast.yml index 3bcfcce7d..a42f0b694 100644 --- a/.github/workflows/fast.yml +++ b/.github/workflows/fast.yml @@ -141,3 +141,11 @@ jobs: if: matrix.os == 'windows-latest' run: ${{github.workspace}}/.github/scripts/check_dll_flags.ps1 ${{env.BUILD_DIR}}/src/proxy_lib/Release/umf_proxy.dll shell: pwsh + + # TODO: We could add some script to verify metadata of dll's (selected fields, perhaps) + # ref. https://superuser.com/questions/381276/what-are-some-nice-command-line-ways-to-inspect-dll-exe-details + - name: Print metadata of our dll's + if: matrix.os == 'windows-latest' + run: | + get-command ${{github.workspace}}/build/bin/Release/umf.dll | format-list + get-command ${{github.workspace}}/build/src/proxy_lib/Release/umf_proxy.dll | format-list From c70ab7d6309456e8758f5386d75649bae20d119b Mon Sep 17 00:00:00 2001 From: Krzysztof Swiecicki Date: Thu, 18 Jul 2024 08:14:15 +0000 Subject: [PATCH 21/52] Remove allocsSpreadAcrossAllNumaNodes test case This test case is removed because it provides too little value compared to the cost to setup this test in appropriate environment. It was the cause of sporadic fails on UR CI. --- test/memspaces/memspace_host_all.cpp | 88 ---------------------------- 1 file changed, 88 deletions(-) diff --git a/test/memspaces/memspace_host_all.cpp b/test/memspaces/memspace_host_all.cpp index 8d11aca0b..3b0825eb5 100644 --- a/test/memspaces/memspace_host_all.cpp +++ b/test/memspaces/memspace_host_all.cpp @@ -152,91 +152,3 @@ TEST_F(memspaceHostAllProviderTest, hostAllDefaults) { UT_ASSERTeq(ret, UMF_RESULT_SUCCESS); umfMemoryProviderDestroy(hProvider); } - -TEST_F(memspaceHostAllProviderTest, allocsSpreadAcrossAllNumaNodes) { - // This testcase is unsuitable for TSan. -#ifdef __SANITIZE_THREAD__ - GTEST_SKIP(); -#endif - - // Arbitrary allocation size, should be big enough to avoid unnecessarily - // prolonging the test execution. - size_t size = SIZE_4M; - size_t alignment = 0; - // Unallocated memory space that has to be left in an attempt to avoid OOM - // killer - 512MB. - size_t remainingSpace = SIZE_4M * 128; - - long long numaCombinedFreeSize = 0; - // Gather free size of all numa nodes. - for (auto &id : nodeIds) { - long long numaFreeSize = 0; - long long numaSize = numa_node_size64(id, &numaFreeSize); - UT_ASSERTne(numaSize, -1); - UT_ASSERT(numaFreeSize >= (long long)(remainingSpace + size)); - - numaCombinedFreeSize += numaFreeSize; - } - - umf_result_t umf_ret = UMF_RESULT_SUCCESS; - // Create allocations until all the NUMA nodes until there's space only for - // one allocation. - std::vector allocs; - std::unordered_set allocNodeIds; - while (numaCombinedFreeSize >= (long long)(remainingSpace + size)) { - void *ptr = nullptr; - umf_ret = umfMemoryProviderAlloc(hProvider, size, alignment, &ptr); - if (umf_ret != UMF_RESULT_SUCCESS) { - UT_ASSERTeq(umf_ret, UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC); - const char *msg = nullptr; - int32_t err = 0; - umfMemoryProviderGetLastNativeError(hProvider, &msg, &err); - // In this scenario, 'UMF_OS_RESULT_ERROR_ALLOC_FAILED' indicates OOM. - UT_ASSERTeq(err, UMF_OS_RESULT_ERROR_ALLOC_FAILED); - break; - } - - UT_ASSERTne(ptr, nullptr); - // Access the allocation, so that all the pages associated with it are - // allocated on available NUMA nodes. - memset(ptr, 0xFF, size); - - int mode = -1; - std::vector boundNodeIds; - size_t allocNodeId = SIZE_MAX; - getAllocationPolicy(ptr, maxNodeId, mode, boundNodeIds, allocNodeId); - - // In case of 'HOST ALL' memspace, the default set of nodes (that - // contains all available nodes) is used but get_mempolicy() would - // return an empty set of nodes. - UT_ASSERTeq(mode, MPOL_DEFAULT); - UT_ASSERTeq(boundNodeIds.size(), 0); - - // Confirm that the memory is allocated on one of the nodes in - // 'HOST ALL' memspace. - auto it = std::find(nodeIds.begin(), nodeIds.end(), allocNodeId); - UT_ASSERT(it != nodeIds.end()); - - allocs.push_back(ptr); - allocNodeIds.insert(allocNodeId); - - numaCombinedFreeSize -= size; - } - - UT_ASSERT(allocs.size() >= nodeIds.size()); - for (auto &ptr : allocs) { - umf_ret = umfMemoryProviderFree(hProvider, ptr, size); - UT_ASSERTeq(umf_ret, UMF_RESULT_SUCCESS); - } - - // TODO: we want to enable this check only when tests are running under QEMU. - // Otherwise it might sporadically fail on a real system where other processes - // occupied all memory from a aparticular NUMA node. -#if 0 - // Confirm that all the NUMA nodes bound to 'HOST ALL' memspace were exhausted. - for (auto &id : nodeIds) { - auto it = std::find(allocNodeIds.begin(), allocNodeIds.end(), id); - UT_ASSERT(it != allocNodeIds.end()); - } -#endif -} From 246f051fe237a99085df63cedd9ef5f03b5a3ef3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Thu, 18 Jul 2024 16:19:53 +0200 Subject: [PATCH 22/52] 0.9.0-rc1 release --- .github/workflows/basic.yml | 2 +- ChangeLog | 8 ++++++++ README.md | 8 -------- scripts/docs_config/conf.py | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml index 31cfba774..b5ce1e948 100644 --- a/.github/workflows/basic.yml +++ b/.github/workflows/basic.yml @@ -8,7 +8,7 @@ permissions: env: # for installation testing - it should match with version set in CMake - UMF_VERSION: 0.1.0 + UMF_VERSION: 0.9.0 BUILD_DIR : "${{github.workspace}}/build" INSTL_DIR : "${{github.workspace}}/../install-dir" diff --git a/ChangeLog b/ChangeLog index b2b9158bf..78c1cc7ef 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,11 @@ +Thu Jul 18 2024 Łukasz Stolarczuk + + * Version 0.9.0-rc1 + + This is a pre-release version. When 0.9.0 will be ready (with no rc status), + it will aim to be the first complete release of the UMF project. + With this release we don't yet guarantee a fully stable API. + Thu Jul 04 2024 Łukasz Stolarczuk * Version 0.1.0 diff --git a/README.md b/README.md index 65c29b76e..808965c5f 100644 --- a/README.md +++ b/README.md @@ -15,14 +15,6 @@ The Unified Memory Framework (UMF) is a library for constructing allocators and memory pools. It also contains broadly useful abstractions and utilities for memory management. UMF allows users to manage multiple memory pools characterized by different attributes, allowing certain allocation types to be isolated from others and allocated using different hardware resources as required. -**⚠️ Work-In-Progress disclaimer:** -> Please note that this project is **pre-production software**, it should not be considered complete or fully functional. -> It has not been fully tested yet (including security testing). It is not recommended to be used in production -> as part of a larger system. Note that this warning is temporary - we plan to release a stable version within six months. -> This project is not eligible for Intel® Bug Bounty Program. -> -> The API is not yet stable, may change without notice, and will not maintain backward compatibility. - ## Usage For a quick introduction to UMF usage, please see diff --git a/scripts/docs_config/conf.py b/scripts/docs_config/conf.py index 306225b88..e0cf028a8 100644 --- a/scripts/docs_config/conf.py +++ b/scripts/docs_config/conf.py @@ -22,7 +22,7 @@ author = "Intel" # The full version, including alpha/beta/rc tags -release = "0.1.0" +release = "0.9.0-rc1" # -- General configuration --------------------------------------------------- From 633da0de5dd65924bf5858c80b97213b95dcca03 Mon Sep 17 00:00:00 2001 From: Adam Date: Thu, 11 Jul 2024 16:04:49 +0200 Subject: [PATCH 23/52] Fix asan's user-poisoning flags bug It add utils_annotate_memory_defined() (which does unpoison on memory region) after all mmap(). This should be unnecessary change but pairs of mmap/munmap do not reset asan's user-poisoning flags, leading to invalid error reports. This bug is describe here - 81619: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81619 --- .github/workflows/sanitizers.yml | 4 +--- src/base_alloc/base_alloc_global.c | 2 -- src/base_alloc/base_alloc_linux.c | 9 +++++++-- src/provider/provider_os_memory_posix.c | 14 ++++++++++++-- 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/.github/workflows/sanitizers.yml b/.github/workflows/sanitizers.yml index 06ad492eb..4b4b37aaf 100644 --- a/.github/workflows/sanitizers.yml +++ b/.github/workflows/sanitizers.yml @@ -72,9 +72,7 @@ jobs: working-directory: ${{env.BUILD_DIR}} run: > ${{ matrix.compiler.cxx == 'icpx' && '. /opt/intel/oneapi/setvars.sh &&' || ''}} - GTEST_FILTER="-*umfProviderTest.alloc_page64_align_0*" ctest --output-on-failure - # TO DO: fix umf-provider_os_memory test for sanitizers - # issue 581: https://github.com/oneapi-src/unified-memory-framework/issues/581 + ctest --output-on-failure windows-build: name: cl and clang-cl on Windows diff --git a/src/base_alloc/base_alloc_global.c b/src/base_alloc/base_alloc_global.c index 003e43a03..b5660d440 100644 --- a/src/base_alloc/base_alloc_global.c +++ b/src/base_alloc/base_alloc_global.c @@ -195,14 +195,12 @@ void umf_ba_global_free(void *ptr) { int ac_index = size_to_idx(total_size); if (ac_index >= NUM_ALLOCATION_CLASSES) { - utils_annotate_memory_inaccessible(ptr, total_size); ba_os_free(ptr, total_size); return; } if (!BASE_ALLOC.ac[ac_index]) { // if creating ac failed, memory must have been allocated by os - utils_annotate_memory_inaccessible(ptr, total_size); ba_os_free(ptr, total_size); return; } diff --git a/src/base_alloc/base_alloc_linux.c b/src/base_alloc/base_alloc_linux.c index 8d07d5ab6..3e5456b2c 100644 --- a/src/base_alloc/base_alloc_linux.c +++ b/src/base_alloc/base_alloc_linux.c @@ -19,8 +19,13 @@ static UTIL_ONCE_FLAG Page_size_is_initialized = UTIL_ONCE_FLAG_INIT; static size_t Page_size; void *ba_os_alloc(size_t size) { - return mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, - -1, 0); + void *ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + // this should be unnecessary but pairs of mmap/munmap do not reset + // asan's user-poisoning flags, leading to invalid error reports + // Bug 81619: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81619 + utils_annotate_memory_defined(ptr, size); + return ptr; } void ba_os_free(void *ptr, size_t size) { diff --git a/src/provider/provider_os_memory_posix.c b/src/provider/provider_os_memory_posix.c index f7040c3f0..9308f6a18 100644 --- a/src/provider/provider_os_memory_posix.c +++ b/src/provider/provider_os_memory_posix.c @@ -16,6 +16,7 @@ #include "provider_os_memory_internal.h" #include "utils_log.h" +#include "utils_sanitizers.h" // maximum value of the off_t type #define OFF_T_MAX \ @@ -74,11 +75,20 @@ void *os_mmap(void *hint_addr, size_t length, int prot, int flag, int fd, if (ptr == MAP_FAILED) { return NULL; } - + // this should be unnecessary but pairs of mmap/munmap do not reset + // asan's user-poisoning flags, leading to invalid error reports + // Bug 81619: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81619 + utils_annotate_memory_defined(ptr, length); return ptr; } -int os_munmap(void *addr, size_t length) { return munmap(addr, length); } +int os_munmap(void *addr, size_t length) { + // this should be unnecessary but pairs of mmap/munmap do not reset + // asan's user-poisoning flags, leading to invalid error reports + // Bug 81619: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81619 + utils_annotate_memory_defined(addr, length); + return munmap(addr, length); +} size_t os_get_page_size(void) { return sysconf(_SC_PAGE_SIZE); } From 066c8c5ca6c7414c2af2430e8bcd372437842c96 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Fri, 19 Jul 2024 09:25:49 +0200 Subject: [PATCH 24/52] Fix warnings RC4005: 'UMF_VERSION' : redefinition Fix the warnings: umf\build\src\libumf.rc(12): warning RC4005: 'UMF_VERSION' : redefinition umf\build\src\proxy_lib\proxy_lib.rc(12): warning RC4005: 'UMF_VERSION' : redefinition Signed-off-by: Lukasz Dorau --- src/libumf.rc.in | 6 +++--- src/proxy_lib/proxy_lib.rc.in | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/libumf.rc.in b/src/libumf.rc.in index 9d0677f6d..3915e0a10 100644 --- a/src/libumf.rc.in +++ b/src/libumf.rc.in @@ -9,7 +9,7 @@ #include "umf/base.h" #define UMF_VERNUMBERS @CMAKE_PROJECT_VERSION_MAJOR@,@CMAKE_PROJECT_VERSION_MINOR@,@CMAKE_PROJECT_VERSION_PATCH@,@UMF_VERSION_REVISION@ -#define UMF_VERSION "@UMF_VERSION@" +#define _UMF_VERSION "@UMF_VERSION@" #ifdef _DEBUG #define VERSION_DEBUG VS_FF_DEBUG @@ -50,12 +50,12 @@ BEGIN BEGIN VALUE "CompanyName", "Intel Corporation\0" VALUE "FileDescription", "Unified Memory Framework (UMF) library\0" - VALUE "FileVersion", UMF_VERSION "\0" + VALUE "FileVersion", _UMF_VERSION "\0" VALUE "LegalCopyright", "Copyright 2024, Intel Corporation. All rights reserved.\0" VALUE "LegalTrademarks", "\0" VALUE "OriginalFilename", "umf.dll\0" VALUE "ProductName", "Unified Memory Framework (UMF)\0" - VALUE "ProductVersion", UMF_VERSION "\0" + VALUE "ProductVersion", _UMF_VERSION "\0" VALUE "PrivateBuild", "\0" VALUE "SpecialBuild", "\0" END diff --git a/src/proxy_lib/proxy_lib.rc.in b/src/proxy_lib/proxy_lib.rc.in index 66910afc4..29c8b0482 100644 --- a/src/proxy_lib/proxy_lib.rc.in +++ b/src/proxy_lib/proxy_lib.rc.in @@ -9,7 +9,7 @@ #include "umf/base.h" #define UMF_VERNUMBERS @CMAKE_PROJECT_VERSION_MAJOR@,@CMAKE_PROJECT_VERSION_MINOR@,@CMAKE_PROJECT_VERSION_PATCH@,@UMF_VERSION_REVISION@ -#define UMF_VERSION "@UMF_VERSION@" +#define _UMF_VERSION "@UMF_VERSION@" #ifdef _DEBUG #define VERSION_DEBUG VS_FF_DEBUG @@ -50,12 +50,12 @@ BEGIN BEGIN VALUE "CompanyName", "Intel Corporation\0" VALUE "FileDescription", "Unified Memory Framework (UMF) proxy library\0" - VALUE "FileVersion", UMF_VERSION "\0" + VALUE "FileVersion", _UMF_VERSION "\0" VALUE "LegalCopyright", "Copyright 2024, Intel Corporation. All rights reserved.\0" VALUE "LegalTrademarks", "\0" VALUE "OriginalFilename", "umf_proxy.dll\0" VALUE "ProductName", "Unified Memory Framework (UMF)\0" - VALUE "ProductVersion", UMF_VERSION "\0" + VALUE "ProductVersion", _UMF_VERSION "\0" VALUE "PrivateBuild", "\0" VALUE "SpecialBuild", "\0" END From 34ad4abb192556235e29193890c9d67e83442081 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Fri, 19 Jul 2024 13:23:51 +0200 Subject: [PATCH 25/52] Add lock for updating file size in OS memory provider Add lock for updating file size in OS memory provider, because umfMemoryProviderAlloc() has to be MT-safe. Signed-off-by: Lukasz Dorau --- src/provider/provider_os_memory.c | 31 +++++++++++++++++----- src/provider/provider_os_memory_internal.h | 3 +++ 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/src/provider/provider_os_memory.c b/src/provider/provider_os_memory.c index 4bec6d5fd..38cb3ff4a 100644 --- a/src/provider/provider_os_memory.c +++ b/src/provider/provider_os_memory.c @@ -525,6 +525,14 @@ static umf_result_t os_initialize(void *params, void **provider) { goto err_destroy_bitmaps; } + if (os_provider->fd > 0) { + if (util_mutex_init(&os_provider->lock_fd) == NULL) { + LOG_ERR("initializing the file size lock failed"); + ret = UMF_RESULT_ERROR_UNKNOWN; + goto err_destroy_bitmaps; + } + } + os_provider->nodeset_str_buf = umf_ba_global_alloc(NODESET_STR_BUF_LEN); if (!os_provider->nodeset_str_buf) { LOG_INFO("allocating memory for printing NUMA nodes failed"); @@ -562,6 +570,10 @@ static void os_finalize(void *provider) { os_memory_provider_t *os_provider = provider; + if (os_provider->fd > 0) { + util_mutex_destroy_not_free(&os_provider->lock_fd); + } + critnib_delete(os_provider->fd_offset_map); free_bitmaps(os_provider); @@ -624,8 +636,8 @@ static inline void assert_is_page_aligned(uintptr_t ptr, size_t page_size) { static int os_mmap_aligned(void *hint_addr, size_t length, size_t alignment, size_t page_size, int prot, int flag, int fd, - size_t max_fd_size, void **out_addr, - size_t *fd_size) { + size_t max_fd_size, os_mutex_t *lock_fd, + void **out_addr, size_t *fd_size) { assert(out_addr); size_t extended_length = length; @@ -641,13 +653,20 @@ static int os_mmap_aligned(void *hint_addr, size_t length, size_t alignment, size_t fd_offset = 0; if (fd > 0) { + if (util_mutex_lock(lock_fd)) { + LOG_ERR("locking file size failed"); + return -1; + } + if (*fd_size + extended_length > max_fd_size) { + util_mutex_unlock(lock_fd); LOG_ERR("cannot grow a file size beyond %zu", max_fd_size); return -1; } fd_offset = *fd_size; *fd_size += extended_length; + util_mutex_unlock(lock_fd); } void *ptr = os_mmap(hint_addr, extended_length, prot, flag, fd, fd_offset); @@ -899,11 +918,11 @@ static umf_result_t os_alloc(void *provider, size_t size, size_t alignment, errno = 0; ret = os_mmap_aligned(NULL, size, alignment, page_size, os_provider->protection, os_provider->visibility, - os_provider->fd, os_provider->max_size_fd, &addr, - &os_provider->size_fd); + os_provider->fd, os_provider->max_size_fd, + &os_provider->lock_fd, &addr, &os_provider->size_fd); if (ret) { - os_store_last_native_error(UMF_OS_RESULT_ERROR_ALLOC_FAILED, errno); - LOG_PERR("memory allocation failed"); + os_store_last_native_error(UMF_OS_RESULT_ERROR_ALLOC_FAILED, 0); + LOG_ERR("memory allocation failed"); return UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC; } diff --git a/src/provider/provider_os_memory_internal.h b/src/provider/provider_os_memory_internal.h index 68750c6d1..81d729d27 100644 --- a/src/provider/provider_os_memory_internal.h +++ b/src/provider/provider_os_memory_internal.h @@ -13,6 +13,7 @@ #include "critnib.h" #include "umf_hwloc.h" #include "utils_common.h" +#include "utils_concurrency.h" #ifdef __cplusplus extern "C" { @@ -33,6 +34,8 @@ typedef struct os_memory_provider_t { int fd; // file descriptor for memory mapping size_t size_fd; // size of file used for memory mapping size_t max_size_fd; // maximum size of file used for memory mapping + os_mutex_t lock_fd; // lock for updating file size + // A critnib map storing (ptr, fd_offset + 1) pairs. We add 1 to fd_offset // in order to be able to store fd_offset equal 0, because // critnib_get() returns value or NULL, so a value cannot equal 0. From cf6bb11f27de5ef6ec0ff4feadf49bd3ebe46f7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Mon, 22 Jul 2024 12:09:53 +0200 Subject: [PATCH 26/52] [CMake] Fix helpers module include use full path, to omit potential conflicts in names with other modules, e.g. in other projects, which can use the same module name. --- CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e84a45097..f3ae1292b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,8 @@ cmake_minimum_required(VERSION 3.14.0 FATAL_ERROR) set(UMF_CMAKE_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) list(APPEND CMAKE_MODULE_PATH "${UMF_CMAKE_SOURCE_DIR}/cmake") -include(helpers) +# Use full path of the helpers module (to omit potential conflicts with others) +include(${UMF_CMAKE_SOURCE_DIR}/cmake/helpers.cmake) # We use semver aligned version, set via git tags. We parse git output to # establih the version of UMF to be used in CMake, Win dll's, and within the From 9f4cd2a43c6fe68bcdbea4fe9dc965755bb7e290 Mon Sep 17 00:00:00 2001 From: Patryk Kaminski Date: Wed, 24 Jul 2024 11:13:13 +0200 Subject: [PATCH 27/52] Disable memory poisoning in DisjointPool ASan throws an error whenever the memory poison is called for the memory allocated on GPU: "AddressSanitizer: CHECK failed: asan_mapping.h:359 "((AddrIsInMem(p))) != (0)" (0x0, 0x0)". This commit disables poisoning. --- src/pool/pool_disjoint.cpp | 35 +++++++++++++++++++++++++++-------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/src/pool/pool_disjoint.cpp b/src/pool/pool_disjoint.cpp index d48d430ce..a1375a3ee 100644 --- a/src/pool/pool_disjoint.cpp +++ b/src/pool/pool_disjoint.cpp @@ -31,6 +31,27 @@ #include "utils_math.h" #include "utils_sanitizers.h" +// Temporary solution for disabling memory poisoning. This is needed because +// AddressSanitizer does not support memory poisoning for GPU allocations. +// More info: https://github.com/oneapi-src/unified-memory-framework/issues/634 +#ifndef POISON_MEMORY +#define POISON_MEMORY 0 +#endif + +static inline void annotate_memory_inaccessible([[maybe_unused]] void *ptr, + [[maybe_unused]] size_t size) { +#ifdef POISON_MEMORY + utils_annotate_memory_inaccessible(ptr, size); +#endif +} + +static inline void annotate_memory_undefined([[maybe_unused]] void *ptr, + [[maybe_unused]] size_t size) { +#ifdef POISON_MEMORY + utils_annotate_memory_undefined(ptr, size); +#endif +} + typedef struct umf_disjoint_pool_shared_limits_t { size_t MaxSize; std::atomic TotalSize; @@ -400,7 +421,7 @@ static void *memoryProviderAlloc(umf_memory_provider_handle_t hProvider, if (ret != UMF_RESULT_SUCCESS) { throw MemoryProviderError{ret}; } - utils_annotate_memory_inaccessible(ptr, size); + annotate_memory_inaccessible(ptr, size); return ptr; } @@ -822,7 +843,7 @@ void *DisjointPool::AllocImpl::allocate(size_t Size, bool &FromPool) try { FromPool = false; if (Size > getParams().MaxPoolableSize) { Ptr = memoryProviderAlloc(getMemHandle(), Size); - utils_annotate_memory_undefined(Ptr, Size); + annotate_memory_undefined(Ptr, Size); return Ptr; } @@ -839,7 +860,7 @@ void *DisjointPool::AllocImpl::allocate(size_t Size, bool &FromPool) try { } VALGRIND_DO_MEMPOOL_ALLOC(this, Ptr, Size); - utils_annotate_memory_undefined(Ptr, Bucket.getSize()); + annotate_memory_undefined(Ptr, Bucket.getSize()); return Ptr; } catch (MemoryProviderError &e) { @@ -877,7 +898,7 @@ void *DisjointPool::AllocImpl::allocate(size_t Size, size_t Alignment, FromPool = false; if (AlignedSize > getParams().MaxPoolableSize) { Ptr = memoryProviderAlloc(getMemHandle(), Size, Alignment); - utils_annotate_memory_undefined(Ptr, Size); + annotate_memory_undefined(Ptr, Size); return Ptr; } @@ -894,8 +915,7 @@ void *DisjointPool::AllocImpl::allocate(size_t Size, size_t Alignment, } VALGRIND_DO_MEMPOOL_ALLOC(this, AlignPtrUp(Ptr, Alignment), Size); - utils_annotate_memory_undefined(AlignPtrUp(Ptr, Alignment), Size); - + annotate_memory_undefined(AlignPtrUp(Ptr, Alignment), Size); return AlignPtrUp(Ptr, Alignment); } catch (MemoryProviderError &e) { umf::getPoolLastStatusRef() = e.code; @@ -962,8 +982,7 @@ void DisjointPool::AllocImpl::deallocate(void *Ptr, bool &ToPool) { } VALGRIND_DO_MEMPOOL_FREE(this, Ptr); - utils_annotate_memory_inaccessible(Ptr, Bucket.getSize()); - + annotate_memory_inaccessible(Ptr, Bucket.getSize()); if (Bucket.getSize() <= Bucket.ChunkCutOff()) { Bucket.freeChunk(Ptr, Slab, ToPool); } else { From e537977d6433c4fc52f78a33ad3af969f5c30f43 Mon Sep 17 00:00:00 2001 From: Krzysztof Swiecicki Date: Thu, 25 Jul 2024 09:05:10 +0000 Subject: [PATCH 28/52] Fix disjoint pool memory poison macro The poison memory macro definition is always defined, so an #ifdef check is insufficient. --- src/pool/pool_disjoint.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pool/pool_disjoint.cpp b/src/pool/pool_disjoint.cpp index a1375a3ee..edb5fc649 100644 --- a/src/pool/pool_disjoint.cpp +++ b/src/pool/pool_disjoint.cpp @@ -40,14 +40,14 @@ static inline void annotate_memory_inaccessible([[maybe_unused]] void *ptr, [[maybe_unused]] size_t size) { -#ifdef POISON_MEMORY +#if (POISON_MEMORY != 0) utils_annotate_memory_inaccessible(ptr, size); #endif } static inline void annotate_memory_undefined([[maybe_unused]] void *ptr, [[maybe_unused]] size_t size) { -#ifdef POISON_MEMORY +#if (POISON_MEMORY != 0) utils_annotate_memory_undefined(ptr, size); #endif } From 40787308921406b9f935402e80ddf5c1e16f7d6f Mon Sep 17 00:00:00 2001 From: Krzysztof Swiecicki Date: Fri, 26 Jul 2024 09:13:55 +0000 Subject: [PATCH 29/52] Add default switch warning suppression --- cmake/helpers.cmake | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/cmake/helpers.cmake b/cmake/helpers.cmake index bb9b703d8..8899077ef 100644 --- a/cmake/helpers.cmake +++ b/cmake/helpers.cmake @@ -176,7 +176,27 @@ function(get_program_version_major_minor name ret) PARENT_SCOPE) endfunction() +# Checks compiler for given ${flag}, stores the output in C_HAS_${flag} and +# CXX_HAS_${flag} (if compiler supports C++) +function(check_compilers_flag flag) + check_c_compiler_flag("${flag}" "C_HAS_${flag}") + if(CMAKE_CXX_COMPILE_FEATURES) + check_cxx_compiler_flag("${flag}" "CXX_HAS_${flag}") + endif() +endfunction() + +function(check_add_target_compile_options target) + foreach(option ${ARGN}) + check_compilers_flag(${option}) + if(C_HAS_${option} AND CXX_HAS_${option}) + target_compile_options(${target} PRIVATE ${option}) + endif() + endforeach() +endfunction() + function(add_umf_target_compile_options name) + check_add_target_compile_options(${name} "-Wno-covered-switch-default") + if(NOT MSVC) target_compile_options( ${name} From 5f552d8ceeef229f738ab459d3d0cb13deb9f90a Mon Sep 17 00:00:00 2001 From: Krzysztof Swiecicki Date: Fri, 26 Jul 2024 09:14:33 +0000 Subject: [PATCH 30/52] Remove werror compile option from UMF targets --- cmake/helpers.cmake | 1 - 1 file changed, 1 deletion(-) diff --git a/cmake/helpers.cmake b/cmake/helpers.cmake index 8899077ef..29cd774b6 100644 --- a/cmake/helpers.cmake +++ b/cmake/helpers.cmake @@ -203,7 +203,6 @@ function(add_umf_target_compile_options name) PRIVATE -fPIC -Wall -Wextra - -Werror -Wpedantic -Wempty-body -Wunused-parameter From 4e31669c0d4fef632929e69c96d929f8d1fce562 Mon Sep 17 00:00:00 2001 From: Krzysztof Swiecicki Date: Fri, 26 Jul 2024 13:34:20 +0000 Subject: [PATCH 31/52] Remove WX MSVC compiler option --- cmake/helpers.cmake | 1 - 1 file changed, 1 deletion(-) diff --git a/cmake/helpers.cmake b/cmake/helpers.cmake index 29cd774b6..0e226d679 100644 --- a/cmake/helpers.cmake +++ b/cmake/helpers.cmake @@ -232,7 +232,6 @@ function(add_umf_target_compile_options name) /analyze /DYNAMICBASE /W4 - /WX /Gy /GS # disable warning 6326: Potential comparison of a constant From f3a60a63b5f8835cf5a2581343e921bf15404bde Mon Sep 17 00:00:00 2001 From: Igor Chorazewicz Date: Fri, 26 Jul 2024 19:49:41 +0000 Subject: [PATCH 32/52] Add option to disable hwloc --- .github/workflows/basic.yml | 11 +++++- CMakeLists.txt | 27 ++++++++++----- examples/CMakeLists.txt | 2 +- src/CMakeLists.txt | 43 +++++++++++++++++------ src/libumf.c | 6 ++-- test/CMakeLists.txt | 68 +++++++++++++++++++++---------------- 6 files changed, 105 insertions(+), 52 deletions(-) diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml index b5ce1e948..2842191e4 100644 --- a/.github/workflows/basic.yml +++ b/.github/workflows/basic.yml @@ -23,6 +23,7 @@ jobs: shared_library: ['OFF'] level_zero_provider: ['ON'] install_tbb: ['ON'] + disable_hwloc: ['OFF'] include: - os: 'ubuntu-20.04' build_type: Release @@ -69,6 +70,13 @@ jobs: shared_library: 'ON' level_zero_provider: 'ON' install_tbb: 'OFF' + - os: 'ubuntu-22.04' + build_type: Release + compiler: {c: gcc, cxx: g++} + shared_library: 'ON' + level_zero_provider: 'ON' + install_tbb: 'ON' + disable_hwloc: 'ON' runs-on: ${{matrix.os}} steps: @@ -122,6 +130,7 @@ jobs: -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON -DUMF_TESTS_FAIL_ON_SKIP=ON + -DUMF_DISABLE_HWLOC=${{matrix.disable_hwloc}} - name: Build UMF run: cmake --build ${{env.BUILD_DIR}} -j $(nproc) @@ -144,7 +153,7 @@ jobs: --build-type ${{matrix.build_type}} --disjoint-pool --jemalloc-pool - ${{ matrix.install_tbb == 'ON' && '--proxy' || '' }} + ${{ matrix.install_tbb == 'ON' && matrix.disable_hwloc != 'ON' && '--proxy' || '' }} --umf-version ${{env.UMF_VERSION}} ${{ matrix.shared_library == 'ON' && '--shared-library' || '' }} diff --git a/CMakeLists.txt b/CMakeLists.txt index f3ae1292b..6005f9d92 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -61,6 +61,10 @@ option(USE_TSAN "Enable ThreadSanitizer checks" OFF) option(USE_MSAN "Enable MemorySanitizer checks" OFF) option(USE_VALGRIND "Enable Valgrind instrumentation" OFF) option(USE_GCOV "Enable gcov support" OFF) +option( + UMF_DISABLE_HWLOC + "Disable features that requires hwloc (OS provider, memory targets, topolgy discovery)" + OFF) # set UMF_PROXY_LIB_BASED_ON_POOL to one of: SCALABLE or JEMALLOC set(KNOWN_PROXY_LIB_POOLS SCALABLE JEMALLOC) @@ -95,14 +99,17 @@ else() endif() if(NOT UMF_LINK_HWLOC_STATICALLY) - pkg_check_modules(LIBHWLOC hwloc>=2.3.0) - if(NOT LIBHWLOC_FOUND) - find_package(LIBHWLOC 2.3.0 REQUIRED hwloc) + if(NOT UMF_DISABLE_HWLOC) + pkg_check_modules(LIBHWLOC hwloc>=2.3.0) + if(NOT LIBHWLOC_FOUND) + find_package(LIBHWLOC 2.3.0 REQUIRED hwloc) + endif() + + # add PATH to DLL on Windows + set(DLL_PATH_LIST + "${DLL_PATH_LIST};PATH=path_list_append:${LIBHWLOC_LIBRARY_DIRS}/../bin" + ) endif() - # add PATH to DLL on Windows - set(DLL_PATH_LIST - "${DLL_PATH_LIST};PATH=path_list_append:${LIBHWLOC_LIBRARY_DIRS}/../bin" - ) else() if(NOT WINDOWS) message(FATAL_ERROR "hwloc can be statically linked only on Windows") @@ -329,7 +336,11 @@ if(UMF_BUILD_BENCHMARKS) endif() if(UMF_BUILD_EXAMPLES) - add_subdirectory(examples) + if(NOT UMF_DISABLE_HWLOC) + add_subdirectory(examples) + else() + message(WARNING "Examples cannot be build - hwloc disabled") + endif() endif() # Conditional configuration for Level Zero provider diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index ac0b3168c..918dc2809 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -18,7 +18,7 @@ set(EXAMPLE_NAME umf_example_basic) add_umf_executable( NAME ${EXAMPLE_NAME} SRCS basic/basic.c - LIBS umf hwloc) + LIBS umf ${LIBHWLOC_LIBRARIES}) target_include_directories( ${EXAMPLE_NAME} PRIVATE ${UMF_CMAKE_SOURCE_DIR}/src/utils diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c261ac6ed..81a6a84a7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -61,6 +61,8 @@ set(BA_SOURCES ${BA_SOURCES} PARENT_SCOPE) +set(HWLOC_DEPENDENT_SOURCES topology.c) + set(UMF_SOURCES ${BA_SOURCES} libumf.c @@ -74,8 +76,11 @@ set(UMF_SOURCES provider/provider_tracking.c critnib/critnib.c pool/pool_proxy.c - pool/pool_scalable.c - topology.c) + pool/pool_scalable.c) + +if(NOT UMF_DISABLE_HWLOC) + set(UMF_SOURCES ${UMF_SOURCES} ${HWLOC_DEPENDENT_SOURCES}) +endif() set(UMF_SOURCES_LINUX libumf_linux.c) @@ -93,16 +98,22 @@ set(UMF_SOURCES_COMMON_LINUX_MACOSX memspaces/memspace_highest_bandwidth.c memspaces/memspace_lowest_latency.c) -set(UMF_SOURCES_LINUX ${UMF_SOURCES_LINUX} ${UMF_SOURCES_COMMON_LINUX_MACOSX} - provider/provider_os_memory_linux.c) +if(NOT UMF_DISABLE_HWLOC) + set(UMF_SOURCES_LINUX + ${UMF_SOURCES_LINUX} ${UMF_SOURCES_COMMON_LINUX_MACOSX} + provider/provider_os_memory_linux.c) -set(UMF_SOURCES_MACOSX ${UMF_SOURCES_MACOSX} ${UMF_SOURCES_COMMON_LINUX_MACOSX} - provider/provider_os_memory_macosx.c) + set(UMF_SOURCES_MACOSX + ${UMF_SOURCES_MACOSX} ${UMF_SOURCES_COMMON_LINUX_MACOSX} + provider/provider_os_memory_macosx.c) -set(UMF_SOURCES_WINDOWS ${UMF_SOURCES_WINDOWS} provider/provider_os_memory.c - provider/provider_os_memory_windows.c) + set(UMF_SOURCES_WINDOWS + ${UMF_SOURCES_WINDOWS} provider/provider_os_memory.c + provider/provider_os_memory_windows.c) + + set(UMF_LIBS ${UMF_LIBS} ${LIBHWLOC_LIBRARIES}) +endif() -set(UMF_LIBS ${UMF_LIBS} ${LIBHWLOC_LIBRARIES}) set(UMF_PRIVATE_LIBRARY_DIRS ${UMF_PRIVATE_LIBRARY_DIRS} ${LIBHWLOC_LIBRARY_DIRS}) @@ -121,11 +132,14 @@ elseif(MACOSX) endif() if(UMF_BUILD_SHARED_LIBRARY) + if(NOT UMF_DISABLE_HWLOC) + set(HWLOC_LIB hwloc) + endif() add_umf_library( NAME umf TYPE SHARED SRCS ${UMF_SOURCES} - LIBS ${UMF_LIBS} hwloc + LIBS ${UMF_LIBS} ${HWLOC_LIB} LINUX_MAP_FILE ${CMAKE_CURRENT_SOURCE_DIR}/libumf.map WINDOWS_DEF_FILE ${CMAKE_CURRENT_BINARY_DIR}/libumf.def) set(UMF_COMMON_COMPILE_DEFINITIONS ${UMF_COMMON_COMPILE_DEFINITIONS} @@ -143,6 +157,11 @@ else() LIBS ${UMF_LIBS}) endif() +if(UMF_DISABLE_HWLOC) + set(UMF_COMMON_COMPILE_DEFINITIONS ${UMF_COMMON_COMPILE_DEFINITIONS} + UMF_NO_HWLOC=1) +endif() + if(UMF_LINK_HWLOC_STATICALLY) add_dependencies(umf hwloc) endif() @@ -190,6 +209,8 @@ install(TARGETS umf EXPORT ${PROJECT_NAME}-targets) add_subdirectory(pool) -if(UMF_PROXY_LIB_ENABLED AND NOT UMF_LINK_HWLOC_STATICALLY) +if(UMF_PROXY_LIB_ENABLED + AND NOT UMF_LINK_HWLOC_STATICALLY + AND NOT UMF_DISABLE_HWLOC) add_subdirectory(proxy_lib) endif() diff --git a/src/libumf.c b/src/libumf.c index d11fa1637..1d99ab26a 100644 --- a/src/libumf.c +++ b/src/libumf.c @@ -12,8 +12,10 @@ #include "base_alloc_global.h" #include "memspace_internal.h" #include "provider_tracking.h" -#include "topology.h" #include "utils_log.h" +#if !defined(UMF_NO_HWLOC) +#include "topology.h" +#endif umf_memory_tracker_handle_t TRACKER = NULL; @@ -30,7 +32,7 @@ int umfInit(void) { void umfTearDown(void) { if (util_fetch_and_add64(&umfRefCount, -1) == 1) { -#ifndef _WIN32 +#if !defined(_WIN32) && !defined(UMF_NO_HWLOC) umfMemspaceHostAllDestroy(); umfMemspaceHighestCapacityDestroy(); umfMemspaceHighestBandwidthDestroy(); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 890ac5572..9a84f71d6 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -152,26 +152,29 @@ endif() if(UMF_BUILD_LIBUMF_POOL_DISJOINT AND UMF_BUILD_LIBUMF_POOL_JEMALLOC - AND UMF_POOL_SCALABLE_ENABLED) + AND UMF_POOL_SCALABLE_ENABLED + AND (NOT UMF_DISABLE_HWLOC)) add_umf_test( NAME c_api_multi_pool SRCS c_api/multi_pool.c LIBS disjoint_pool jemalloc_pool ${JEMALLOC_LIBRARIES}) endif() -if(UMF_BUILD_LIBUMF_POOL_JEMALLOC) +if(UMF_BUILD_LIBUMF_POOL_JEMALLOC AND (NOT UMF_DISABLE_HWLOC)) add_umf_test( NAME jemalloc_pool SRCS pools/jemalloc_pool.cpp malloc_compliance_tests.cpp LIBS jemalloc_pool) endif() -if(UMF_POOL_SCALABLE_ENABLED) +if(UMF_POOL_SCALABLE_ENABLED AND (NOT UMF_DISABLE_HWLOC)) add_umf_test(NAME scalable_pool SRCS pools/scalable_pool.cpp malloc_compliance_tests.cpp) endif() -if(LINUX) # OS-specific functions are implemented only for Linux now +if(LINUX AND (NOT UMF_DISABLE_HWLOC)) # OS-specific functions are implemented + # only for + # Linux now if(PkgConfig_FOUND) pkg_check_modules(LIBNUMA numa) endif() @@ -262,7 +265,9 @@ add_umf_test( LIBS ${UMF_UTILS_FOR_TEST}) # tests for the proxy library -if(UMF_PROXY_LIB_ENABLED AND UMF_BUILD_SHARED_LIBRARY) +if(UMF_PROXY_LIB_ENABLED + AND UMF_BUILD_SHARED_LIBRARY + AND NOT UMF_DISABLE_HWLOC) add_umf_test( NAME proxy_lib_basic SRCS ${BA_SOURCES_FOR_TEST} test_proxy_lib.cpp @@ -313,22 +318,24 @@ function(add_umf_ipc_test) endfunction() if(LINUX) - build_umf_test( - NAME - ipc_os_prov_consumer - SRCS - ipc_os_prov_consumer.c - common/ipc_common.c - common/ipc_os_prov_common.c) - build_umf_test( - NAME - ipc_os_prov_producer - SRCS - ipc_os_prov_producer.c - common/ipc_common.c - common/ipc_os_prov_common.c) - add_umf_ipc_test(TEST ipc_os_prov_anon_fd) - add_umf_ipc_test(TEST ipc_os_prov_shm) + if(NOT UMF_DISABLE_HWLOC) + build_umf_test( + NAME + ipc_os_prov_consumer + SRCS + ipc_os_prov_consumer.c + common/ipc_common.c + common/ipc_os_prov_common.c) + build_umf_test( + NAME + ipc_os_prov_producer + SRCS + ipc_os_prov_producer.c + common/ipc_common.c + common/ipc_os_prov_common.c) + add_umf_ipc_test(TEST ipc_os_prov_anon_fd) + add_umf_ipc_test(TEST ipc_os_prov_shm) + endif() if(UMF_BUILD_GPU_TESTS AND UMF_BUILD_LEVEL_ZERO_PROVIDER) build_umf_test( NAME @@ -364,7 +371,8 @@ endif() if(LINUX AND UMF_BUILD_SHARED_LIBRARY - AND UMF_POOL_SCALABLE_ENABLED) + AND UMF_POOL_SCALABLE_ENABLED + AND NOT UMF_DISABLE_HWLOC) add_umf_test( NAME init_teardown SRCS test_init_teardown.c @@ -427,11 +435,13 @@ if(LINUX ) endif() - add_test( - NAME umf_standalone_examples - COMMAND - ${UMF_CMAKE_SOURCE_DIR}/test/test_examples.sh - ${UMF_CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR} ${CMAKE_INSTALL_PREFIX} - ${EXAMPLES} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + if(NOT UMF_DISABLE_HWLOC) + add_test( + NAME umf_standalone_examples + COMMAND + ${UMF_CMAKE_SOURCE_DIR}/test/test_examples.sh + ${UMF_CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR} + ${CMAKE_INSTALL_PREFIX} ${EXAMPLES} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) + endif() endif() From daffca6eb256dc95de4249870abb12c8f6b0793c Mon Sep 17 00:00:00 2001 From: Igor Chorazewicz Date: Fri, 26 Jul 2024 18:03:14 +0000 Subject: [PATCH 33/52] Add option to link with hwloc statically on linux Co-authored-by: Krzysztof Filipek --- .github/workflows/basic.yml | 6 ++-- CMakeLists.txt | 64 +++++++++++++++++++++++++++++++++---- test/CMakeLists.txt | 3 +- 3 files changed, 63 insertions(+), 10 deletions(-) diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml index 2842191e4..4105e606f 100644 --- a/.github/workflows/basic.yml +++ b/.github/workflows/basic.yml @@ -153,7 +153,7 @@ jobs: --build-type ${{matrix.build_type}} --disjoint-pool --jemalloc-pool - ${{ matrix.install_tbb == 'ON' && matrix.disable_hwloc != 'ON' && '--proxy' || '' }} + ${{ matrix.install_tbb == 'ON' && matrix.disable_hwloc != 'ON' && matrix.link_hwloc_statically != 'ON' && '--proxy' || '' }} --umf-version ${{env.UMF_VERSION}} ${{ matrix.shared_library == 'ON' && '--shared-library' || '' }} @@ -349,7 +349,7 @@ jobs: run: python3 -m pip install -r third_party/requirements.txt - name: Install hwloc - run: brew install hwloc jemalloc tbb + run: brew install hwloc jemalloc tbb autoconf automake libtool - name: Configure build run: > @@ -376,6 +376,6 @@ jobs: --build-type ${{env.BUILD_TYPE}} --disjoint-pool --jemalloc-pool - --proxy + ${{ matrix.link_hwloc_statically != 'ON' && '--proxy' || '' }} --umf-version ${{env.UMF_VERSION}} --shared-library diff --git a/CMakeLists.txt b/CMakeLists.txt index 6005f9d92..946eb8984 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -48,8 +48,10 @@ option(UMF_BUILD_EXAMPLES "Build UMF examples" ON) option(UMF_BUILD_FUZZTESTS "Build UMF fuzz tests" OFF) option(UMF_BUILD_GPU_EXAMPLES "Build UMF GPU examples" OFF) option(UMF_DEVELOPER_MODE "Enable additional developer checks" OFF) -option(UMF_LINK_HWLOC_STATICALLY - "Link UMF with HWLOC library statically (Windows+Release only)" OFF) +option( + UMF_LINK_HWLOC_STATICALLY + "Link UMF with HWLOC library statically (supported for Linux, MacOS and Release build on Windows)" + OFF) option(UMF_FORMAT_CODE_STYLE "Add clang, cmake, and black -format-check and -format-apply targets" OFF) @@ -110,10 +112,11 @@ if(NOT UMF_LINK_HWLOC_STATICALLY) "${DLL_PATH_LIST};PATH=path_list_append:${LIBHWLOC_LIBRARY_DIRS}/../bin" ) endif() -else() - if(NOT WINDOWS) - message(FATAL_ERROR "hwloc can be statically linked only on Windows") - endif() + # add PATH to DLL on Windows + set(DLL_PATH_LIST + "${DLL_PATH_LIST};PATH=path_list_append:${LIBHWLOC_LIBRARY_DIRS}/../bin" + ) +elseif(WINDOWS) include(FetchContent) set(HWLOC_ENABLE_TESTING OFF) set(HWLOC_SKIP_LSTOPO ON) @@ -134,6 +137,55 @@ else() set(LIBHWLOC_LIBRARY_DIRS ${hwloc_targ_BINARY_DIR}/Release;${hwloc_targ_BINARY_DIR}/Debug) + message(STATUS " LIBHWLOC_LIBRARIES = ${LIBHWLOC_LIBRARIES}") + message(STATUS " LIBHWLOC_INCLUDE_DIRS = ${LIBHWLOC_INCLUDE_DIRS}") + message(STATUS " LIBHWLOC_LIBRARY_DIRS = ${LIBHWLOC_LIBRARY_DIRS}") +else() + include(FetchContent) + FetchContent_Declare( + hwloc_targ + GIT_REPOSITORY "https://github.com/open-mpi/hwloc.git" + GIT_TAG hwloc-2.10.0) + + FetchContent_GetProperties(hwloc_targ) + if(NOT hwloc_targ_POPULATED) + FetchContent_MakeAvailable(hwloc_targ) + endif() + + add_custom_command( + COMMAND ./autogen.sh + WORKING_DIRECTORY ${hwloc_targ_SOURCE_DIR} + OUTPUT ${hwloc_targ_SOURCE_DIR}/configure) + add_custom_command( + COMMAND + ./configure --prefix=${hwloc_targ_BINARY_DIR} --enable-static=yes + --enable-shared=no --disable-libxml2 --disable-levelzero + CFLAGS=-fPIC CXXFLAGS=-fPIC + WORKING_DIRECTORY ${hwloc_targ_SOURCE_DIR} + OUTPUT ${hwloc_targ_SOURCE_DIR}/Makefile + DEPENDS ${hwloc_targ_SOURCE_DIR}/configure) + add_custom_command( + COMMAND make + WORKING_DIRECTORY ${hwloc_targ_SOURCE_DIR} + OUTPUT ${hwloc_targ_SOURCE_DIR}/lib/libhwloc.la + DEPENDS ${hwloc_targ_SOURCE_DIR}/Makefile) + add_custom_command( + COMMAND make install + WORKING_DIRECTORY ${hwloc_targ_SOURCE_DIR} + OUTPUT ${hwloc_targ_BINARY_DIR}/lib/libhwloc.a + DEPENDS ${hwloc_targ_SOURCE_DIR}/lib/libhwloc.la) + + add_custom_target(hwloc_prod + DEPENDS ${hwloc_targ_BINARY_DIR}/lib/libhwloc.a) + add_library(hwloc INTERFACE) + target_link_libraries(hwloc + INTERFACE ${hwloc_targ_BINARY_DIR}/lib/libhwloc.a) + add_dependencies(hwloc hwloc_prod) + + set(LIBHWLOC_LIBRARY_DIRS ${hwloc_targ_BINARY_DIR}/lib) + set(LIBHWLOC_INCLUDE_DIRS ${hwloc_targ_BINARY_DIR}/include) + set(LIBHWLOC_LIBRARIES ${hwloc_targ_BINARY_DIR}/lib/libhwloc.a) + message(STATUS " LIBHWLOC_LIBRARIES = ${LIBHWLOC_LIBRARIES}") message(STATUS " LIBHWLOC_INCLUDE_DIRS = ${LIBHWLOC_INCLUDE_DIRS}") message(STATUS " LIBHWLOC_LIBRARY_DIRS = ${LIBHWLOC_LIBRARY_DIRS}") diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 9a84f71d6..c77a6e326 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -267,7 +267,8 @@ add_umf_test( # tests for the proxy library if(UMF_PROXY_LIB_ENABLED AND UMF_BUILD_SHARED_LIBRARY - AND NOT UMF_DISABLE_HWLOC) + AND NOT UMF_DISABLE_HWLOC + AND NOT UMF_LINK_HWLOC_STATICALLY) add_umf_test( NAME proxy_lib_basic SRCS ${BA_SOURCES_FOR_TEST} test_proxy_lib.cpp From eb821bac0756aafe677429826d8768d15f7c93c3 Mon Sep 17 00:00:00 2001 From: Igor Chorazewicz Date: Fri, 26 Jul 2024 18:06:51 +0000 Subject: [PATCH 34/52] Add more CI runs with UMF_LINK_HWLOC_STATICALLY=1 --- .github/workflows/basic.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml index 4105e606f..736f08eef 100644 --- a/.github/workflows/basic.yml +++ b/.github/workflows/basic.yml @@ -24,6 +24,7 @@ jobs: level_zero_provider: ['ON'] install_tbb: ['ON'] disable_hwloc: ['OFF'] + link_hwloc_statically: ['OFF'] include: - os: 'ubuntu-20.04' build_type: Release @@ -71,12 +72,19 @@ jobs: level_zero_provider: 'ON' install_tbb: 'OFF' - os: 'ubuntu-22.04' - build_type: Release + build_type: Debug compiler: {c: gcc, cxx: g++} shared_library: 'ON' level_zero_provider: 'ON' install_tbb: 'ON' disable_hwloc: 'ON' + - os: 'ubuntu-22.04' + build_type: Release + compiler: {c: gcc, cxx: g++} + shared_library: 'ON' + level_zero_provider: 'ON' + install_tbb: 'ON' + link_hwloc_statically: 'ON' runs-on: ${{matrix.os}} steps: @@ -131,6 +139,7 @@ jobs: -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON -DUMF_TESTS_FAIL_ON_SKIP=ON -DUMF_DISABLE_HWLOC=${{matrix.disable_hwloc}} + -DUMF_LINK_HWLOC_STATICALLY=${{matrix.link_hwloc_statically}} - name: Build UMF run: cmake --build ${{env.BUILD_DIR}} -j $(nproc) @@ -349,7 +358,7 @@ jobs: run: python3 -m pip install -r third_party/requirements.txt - name: Install hwloc - run: brew install hwloc jemalloc tbb autoconf automake libtool + run: brew install hwloc jemalloc tbb - name: Configure build run: > @@ -376,6 +385,6 @@ jobs: --build-type ${{env.BUILD_TYPE}} --disjoint-pool --jemalloc-pool - ${{ matrix.link_hwloc_statically != 'ON' && '--proxy' || '' }} + --proxy --umf-version ${{env.UMF_VERSION}} --shared-library From 682c034dcfee7521631a0f311357fe5b279e1349 Mon Sep 17 00:00:00 2001 From: Igor Chorazewicz Date: Mon, 29 Jul 2024 18:33:27 +0000 Subject: [PATCH 35/52] Add option to change hwloc repo url and tag --- CMakeLists.txt | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 946eb8984..cf245ce10 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -100,6 +100,14 @@ else() message(FATAL_ERROR "Unknown OS type") endif() +if(NOT DEFINED UMF_HWLOC_REPO) + set(UMF_HWLOC_REPO "https://github.com/open-mpi/hwloc.git") +endif() + +if(NOT DEFINED UMF_HWLOC_TAG) + set(UMF_HWLOC_TAG hwloc-2.10.0) +endif() + if(NOT UMF_LINK_HWLOC_STATICALLY) if(NOT UMF_DISABLE_HWLOC) pkg_check_modules(LIBHWLOC hwloc>=2.3.0) @@ -121,10 +129,13 @@ elseif(WINDOWS) set(HWLOC_ENABLE_TESTING OFF) set(HWLOC_SKIP_LSTOPO ON) set(HWLOC_SKIP_TOOLS ON) + + message(STATUS "Will fetch hwloc from ${UMF_HWLOC_REPO}") + FetchContent_Declare( hwloc_targ - GIT_REPOSITORY "https://github.com/open-mpi/hwloc.git" - GIT_TAG hwloc-2.10.0 + GIT_REPOSITORY ${UMF_HWLOC_REPO} + GIT_TAG ${UMF_HWLOC_TAG} SOURCE_SUBDIR contrib/windows-cmake/ FIND_PACKAGE_ARGS) FetchContent_GetProperties(hwloc_targ) @@ -142,10 +153,12 @@ elseif(WINDOWS) message(STATUS " LIBHWLOC_LIBRARY_DIRS = ${LIBHWLOC_LIBRARY_DIRS}") else() include(FetchContent) + message(STATUS "Will fetch hwloc from ${UMF_HWLOC_REPO}") + FetchContent_Declare( hwloc_targ - GIT_REPOSITORY "https://github.com/open-mpi/hwloc.git" - GIT_TAG hwloc-2.10.0) + GIT_REPOSITORY ${UMF_HWLOC_REPO} + GIT_TAG ${UMF_HWLOC_TAG}) FetchContent_GetProperties(hwloc_targ) if(NOT hwloc_targ_POPULATED) From 68e739dacc15faeb23285456ade009cecd7f61f0 Mon Sep 17 00:00:00 2001 From: Igor Chorazewicz Date: Mon, 29 Jul 2024 17:29:39 +0000 Subject: [PATCH 36/52] Avoid using non-existing symbols in .map to fix: "ld.lld: error: version script assignment of 'UMF_1.0' to symbol 'umfLevelZeroMemoryProviderOps' failed: symbol not defined" when using lld linker --- CMakeLists.txt | 27 ++++++++++--------- cmake/helpers.cmake | 9 +++++++ .../third-party-programs.txt | 0 src/CMakeLists.txt | 23 +++++++++++++++- src/libumf.def.in | 3 +-- src/{libumf.map => libumf.map.in} | 8 +----- test/test_installation.py | 3 ++- 7 files changed, 50 insertions(+), 23 deletions(-) rename third-party-programs.txt => licensing/third-party-programs.txt (100%) rename src/{libumf.map => libumf.map.in} (87%) diff --git a/CMakeLists.txt b/CMakeLists.txt index cf245ce10..707a877b3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -390,6 +390,18 @@ else() ) endif() +set(UMF_OPTIONAL_SYMBOLS_LINUX "") +set(UMF_OPTIONAL_SYMBOLS_WINDOWS "") + +# Conditional configuration for Level Zero provider +if(UMF_BUILD_LEVEL_ZERO_PROVIDER) + add_optional_symbol(umfLevelZeroMemoryProviderOps) +endif() + +if(NOT UMF_DISABLE_HWLOC) + add_optional_symbol(umfOsMemoryProviderOps) +endif() + add_subdirectory(src) if(UMF_BUILD_TESTS) @@ -408,17 +420,6 @@ if(UMF_BUILD_EXAMPLES) endif() endif() -# Conditional configuration for Level Zero provider -if(UMF_BUILD_LEVEL_ZERO_PROVIDER) - set(OPTIONAL_SYMBOLS "umfLevelZeroMemoryProviderOps") -else() - set(OPTIONAL_SYMBOLS "") -endif() - -# Configure the DEF file based on whether Level Zero provider is built -configure_file("${CMAKE_CURRENT_SOURCE_DIR}/src/libumf.def.in" - "${CMAKE_CURRENT_BINARY_DIR}/src/libumf.def" @ONLY) - if(UMF_FORMAT_CODE_STYLE) find_program(CLANG_FORMAT NAMES clang-format-15 clang-format-15.0 clang-format) @@ -599,8 +600,10 @@ endif() # Configure make install/uninstall and packages # --------------------------------------------------------------------------- # install(FILES ${CMAKE_SOURCE_DIR}/LICENSE.TXT - ${CMAKE_SOURCE_DIR}/third-party-programs.txt DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/doc/${PROJECT_NAME}/") +install( + FILES ${CMAKE_SOURCE_DIR}/licensing/third-party-programs.txt + DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/doc/${PROJECT_NAME}/licensing/") install(DIRECTORY examples DESTINATION "${CMAKE_INSTALL_DOCDIR}") diff --git a/cmake/helpers.cmake b/cmake/helpers.cmake index 0e226d679..a4f59a763 100644 --- a/cmake/helpers.cmake +++ b/cmake/helpers.cmake @@ -417,3 +417,12 @@ macro(add_sanitizer_flag flag) set(CMAKE_REQUIRED_FLAGS ${SAVED_CMAKE_REQUIRED_FLAGS}) endmacro() + +function(add_optional_symbol symbol) + set(UMF_OPTIONAL_SYMBOLS_WINDOWS + "${UMF_OPTIONAL_SYMBOLS_WINDOWS} \n ${symbol}" + PARENT_SCOPE) + set(UMF_OPTIONAL_SYMBOLS_LINUX + "${UMF_OPTIONAL_SYMBOLS_LINUX} \n ${symbol};" + PARENT_SCOPE) +endfunction() diff --git a/third-party-programs.txt b/licensing/third-party-programs.txt similarity index 100% rename from third-party-programs.txt rename to licensing/third-party-programs.txt diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 81a6a84a7..6a1e1d475 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -112,8 +112,29 @@ if(NOT UMF_DISABLE_HWLOC) provider/provider_os_memory_windows.c) set(UMF_LIBS ${UMF_LIBS} ${LIBHWLOC_LIBRARIES}) + + if(NOT WINDOWS) + add_optional_symbol(umfMemspaceCreateFromNumaArray) + add_optional_symbol(umfMemspaceHighestBandwidthGet) + add_optional_symbol(umfMemspaceHighestCapacityGet) + add_optional_symbol(umfMemspaceHostAllGet) + add_optional_symbol(umfMemspaceLowestLatencyGet) + endif() endif() +if(WINDOWS) + message(STATUS "UMF_OPTIONAL_SYMBOLS: ${UMF_OPTIONAL_SYMBOLS_WINDOWS}") +else() + message(STATUS "UMF_OPTIONAL_SYMBOLS: ${UMF_OPTIONAL_SYMBOLS_LINUX}") +endif() + +# Configure the DEF file based on whether Level Zero provider is built +configure_file("${CMAKE_CURRENT_SOURCE_DIR}/libumf.def.in" + "${CMAKE_CURRENT_BINARY_DIR}/libumf.def" @ONLY) + +configure_file("${CMAKE_CURRENT_SOURCE_DIR}/libumf.map.in" + "${CMAKE_CURRENT_BINARY_DIR}/libumf.map" @ONLY) + set(UMF_PRIVATE_LIBRARY_DIRS ${UMF_PRIVATE_LIBRARY_DIRS} ${LIBHWLOC_LIBRARY_DIRS}) @@ -140,7 +161,7 @@ if(UMF_BUILD_SHARED_LIBRARY) TYPE SHARED SRCS ${UMF_SOURCES} LIBS ${UMF_LIBS} ${HWLOC_LIB} - LINUX_MAP_FILE ${CMAKE_CURRENT_SOURCE_DIR}/libumf.map + LINUX_MAP_FILE ${CMAKE_CURRENT_BINARY_DIR}/libumf.map WINDOWS_DEF_FILE ${CMAKE_CURRENT_BINARY_DIR}/libumf.def) set(UMF_COMMON_COMPILE_DEFINITIONS ${UMF_COMMON_COMPILE_DEFINITIONS} "UMF_SHARED_LIBRARY") diff --git a/src/libumf.def.in b/src/libumf.def.in index 8ee99e024..aa78d0953 100644 --- a/src/libumf.def.in +++ b/src/libumf.def.in @@ -42,7 +42,6 @@ EXPORTS umfMempolicySetInterleavePartSize umfMemspaceDestroy umfOpenIPCHandle - umfOsMemoryProviderOps umfPoolAlignedMalloc umfPoolByPtr umfPoolCalloc @@ -59,4 +58,4 @@ EXPORTS umfProxyPoolOps umfPutIPCHandle umfScalablePoolOps - @OPTIONAL_SYMBOLS@ + @UMF_OPTIONAL_SYMBOLS_WINDOWS@ diff --git a/src/libumf.map b/src/libumf.map.in similarity index 87% rename from src/libumf.map rename to src/libumf.map.in index cb09cdb94..20031f16e 100644 --- a/src/libumf.map +++ b/src/libumf.map.in @@ -11,7 +11,6 @@ UMF_1.0 { umfFree; umfGetIPCHandle; umfGetLastFailedMemoryProvider; - umfLevelZeroMemoryProviderOps; umfMemoryTrackerGetAllocInfo; umfMemoryProviderAlloc; umfMemoryProviderAllocationMerge; @@ -35,14 +34,8 @@ UMF_1.0 { umfMempolicyDestroy; umfMempolicySetCustomSplitPartitions; umfMempolicySetInterleavePartSize; - umfMemspaceCreateFromNumaArray; umfMemspaceDestroy; - umfMemspaceHighestBandwidthGet; - umfMemspaceHighestCapacityGet; - umfMemspaceHostAllGet; - umfMemspaceLowestLatencyGet; umfOpenIPCHandle; - umfOsMemoryProviderOps; umfPoolAlignedMalloc; umfPoolByPtr; umfPoolCalloc; @@ -59,6 +52,7 @@ UMF_1.0 { umfProxyPoolOps; umfPutIPCHandle; umfScalablePoolOps; + @UMF_OPTIONAL_SYMBOLS_LINUX@ local: *; }; diff --git a/test/test_installation.py b/test/test_installation.py index 6acb4a0bc..49a382969 100644 --- a/test/test_installation.py +++ b/test/test_installation.py @@ -146,7 +146,8 @@ def _create_match_list(self) -> List[str]: examples.insert(0, "share/doc/umf/examples") share.extend(examples) share.append("share/doc/umf/LICENSE.TXT") - share.append("share/doc/umf/third-party-programs.txt") + share.append("share/doc/umf/licensing") + share.append("share/doc/umf/licensing/third-party-programs.txt") all_files = bin + include + lib + share if platform.system() == "Windows": From d3fad20a309bfc883a4c0e77175d5bdb38de2e61 Mon Sep 17 00:00:00 2001 From: Igor Chorazewicz Date: Mon, 29 Jul 2024 23:21:36 +0000 Subject: [PATCH 37/52] Allow completely disabling hwloc on windows --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 707a877b3..2e4f7ab1b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -124,7 +124,7 @@ if(NOT UMF_LINK_HWLOC_STATICALLY) set(DLL_PATH_LIST "${DLL_PATH_LIST};PATH=path_list_append:${LIBHWLOC_LIBRARY_DIRS}/../bin" ) -elseif(WINDOWS) +elseif(WINDOWS AND NOT UMF_DISABLE_HWLOC) include(FetchContent) set(HWLOC_ENABLE_TESTING OFF) set(HWLOC_SKIP_LSTOPO ON) @@ -151,7 +151,7 @@ elseif(WINDOWS) message(STATUS " LIBHWLOC_LIBRARIES = ${LIBHWLOC_LIBRARIES}") message(STATUS " LIBHWLOC_INCLUDE_DIRS = ${LIBHWLOC_INCLUDE_DIRS}") message(STATUS " LIBHWLOC_LIBRARY_DIRS = ${LIBHWLOC_LIBRARY_DIRS}") -else() +elseif(NOT UMF_DISABLE_HWLOC) include(FetchContent) message(STATUS "Will fetch hwloc from ${UMF_HWLOC_REPO}") From 19a5bf5389fdea9fb4d54350f231db97afe17005 Mon Sep 17 00:00:00 2001 From: Lukasz Dorau Date: Mon, 29 Jul 2024 16:43:24 +0200 Subject: [PATCH 38/52] Fix: size_fd has to be read under a lock Signed-off-by: Lukasz Dorau --- src/provider/provider_os_memory.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/provider/provider_os_memory.c b/src/provider/provider_os_memory.c index 38cb3ff4a..00251e53b 100644 --- a/src/provider/provider_os_memory.c +++ b/src/provider/provider_os_memory.c @@ -637,7 +637,8 @@ static inline void assert_is_page_aligned(uintptr_t ptr, size_t page_size) { static int os_mmap_aligned(void *hint_addr, size_t length, size_t alignment, size_t page_size, int prot, int flag, int fd, size_t max_fd_size, os_mutex_t *lock_fd, - void **out_addr, size_t *fd_size) { + void **out_addr, size_t *fd_size, + size_t *fd_offset) { assert(out_addr); size_t extended_length = length; @@ -650,7 +651,7 @@ static int os_mmap_aligned(void *hint_addr, size_t length, size_t alignment, extended_length += alignment; } - size_t fd_offset = 0; + *fd_offset = 0; if (fd > 0) { if (util_mutex_lock(lock_fd)) { @@ -664,12 +665,12 @@ static int os_mmap_aligned(void *hint_addr, size_t length, size_t alignment, return -1; } - fd_offset = *fd_size; + *fd_offset = *fd_size; *fd_size += extended_length; util_mutex_unlock(lock_fd); } - void *ptr = os_mmap(hint_addr, extended_length, prot, flag, fd, fd_offset); + void *ptr = os_mmap(hint_addr, extended_length, prot, flag, fd, *fd_offset); if (ptr == NULL) { LOG_PDEBUG("memory mapping failed"); return -1; @@ -912,14 +913,14 @@ static umf_result_t os_alloc(void *provider, size_t size, size_t alignment, return UMF_RESULT_ERROR_INVALID_ARGUMENT; } - size_t fd_offset = os_provider->size_fd; // needed for critnib_insert() + size_t fd_offset; // needed for critnib_insert() void *addr = NULL; errno = 0; - ret = os_mmap_aligned(NULL, size, alignment, page_size, - os_provider->protection, os_provider->visibility, - os_provider->fd, os_provider->max_size_fd, - &os_provider->lock_fd, &addr, &os_provider->size_fd); + ret = os_mmap_aligned( + NULL, size, alignment, page_size, os_provider->protection, + os_provider->visibility, os_provider->fd, os_provider->max_size_fd, + &os_provider->lock_fd, &addr, &os_provider->size_fd, &fd_offset); if (ret) { os_store_last_native_error(UMF_OS_RESULT_ERROR_ALLOC_FAILED, 0); LOG_ERR("memory allocation failed"); From 330dd302791d09aff4b878e99aaa7aa67087ae8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20=C5=BDu=C5=BEek?= Date: Wed, 31 Jul 2024 16:24:07 +0100 Subject: [PATCH 39/52] [CMake] Don't use CMAKE_SOURCE_DIR during install Referring to an installed file using `CMAKE_SOURCE_DIR` only works when UMF is built and installed standalone, but not when it's included by other projects. Only `LICENSE.TXT` and `licensing/third-party-programs.txt` still had these issue, while other files already used `PROJECT_SOURCE_DIR`. --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2e4f7ab1b..1627606dd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -599,10 +599,10 @@ endif() # --------------------------------------------------------------------------- # # Configure make install/uninstall and packages # --------------------------------------------------------------------------- # -install(FILES ${CMAKE_SOURCE_DIR}/LICENSE.TXT +install(FILES ${PROJECT_SOURCE_DIR}/LICENSE.TXT DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/doc/${PROJECT_NAME}/") install( - FILES ${CMAKE_SOURCE_DIR}/licensing/third-party-programs.txt + FILES ${PROJECT_SOURCE_DIR}/licensing/third-party-programs.txt DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/doc/${PROJECT_NAME}/licensing/") install(DIRECTORY examples DESTINATION "${CMAKE_INSTALL_DOCDIR}") From 9cb5f9ab0549a44da271cb27c513c00f63e15245 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Fri, 2 Aug 2024 12:48:02 +0200 Subject: [PATCH 40/52] [CMake] Add missing UMF_ prefix in 'public' umf options --- .github/workflows/nightly.yml | 2 +- .github/workflows/sanitizers.yml | 8 ++++---- .github/workflows/valgrind.yml | 2 +- CMakeLists.txt | 20 ++++++++++---------- CONTRIBUTING.md | 4 ++-- README.md | 12 ++++++------ cmake/helpers.cmake | 4 ++-- src/utils/CMakeLists.txt | 12 ++++++------ test/CMakeLists.txt | 8 ++++---- 9 files changed, 36 insertions(+), 36 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 28a07c3a9..f2bf8f08f 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -79,7 +79,7 @@ jobs: -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON -DUMF_BUILD_LEVEL_ZERO_PROVIDER=OFF - -DUSE_VALGRIND=1 + -DUMF_USE_VALGRIND=1 -DUMF_TESTS_FAIL_ON_SKIP=ON - name: Build diff --git a/.github/workflows/sanitizers.yml b/.github/workflows/sanitizers.yml index 4b4b37aaf..2ca712543 100644 --- a/.github/workflows/sanitizers.yml +++ b/.github/workflows/sanitizers.yml @@ -59,9 +59,9 @@ jobs: -DUMF_DEVELOPER_MODE=ON -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON - -DUSE_ASAN=${{matrix.sanitizers.asan}} - -DUSE_UBSAN=${{matrix.sanitizers.ubsan}} - -DUSE_TSAN=${{matrix.sanitizers.tsan}} + -DUMF_USE_ASAN=${{matrix.sanitizers.asan}} + -DUMF_USE_UBSAN=${{matrix.sanitizers.ubsan}} + -DUMF_USE_TSAN=${{matrix.sanitizers.tsan}} -DUMF_BUILD_EXAMPLES=ON -DUMF_TESTS_FAIL_ON_SKIP=ON @@ -127,7 +127,7 @@ jobs: -DUMF_FORMAT_CODE_STYLE=OFF -DUMF_DEVELOPER_MODE=ON -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON - -DUSE_ASAN=${{matrix.sanitizers.asan}} + -DUMF_USE_ASAN=${{matrix.sanitizers.asan}} -DUMF_BUILD_EXAMPLES=ON -DUMF_BUILD_LEVEL_ZERO_PROVIDER=OFF -DUMF_TESTS_FAIL_ON_SKIP=ON diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 0ca9bf779..53569385e 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -35,7 +35,7 @@ jobs: -DUMF_BUILD_LIBUMF_POOL_DISJOINT=ON -DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON -DUMF_BUILD_LEVEL_ZERO_PROVIDER=OFF - -DUSE_VALGRIND=1 + -DUMF_USE_VALGRIND=1 -DUMF_TESTS_FAIL_ON_SKIP=ON - name: Build diff --git a/CMakeLists.txt b/CMakeLists.txt index 1627606dd..3edd86f4c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -57,12 +57,12 @@ option(UMF_FORMAT_CODE_STYLE OFF) # Only a part of skips is treated as a failure now. TODO: extend to all tests option(UMF_TESTS_FAIL_ON_SKIP "Treat skips in tests as fail" OFF) -option(USE_ASAN "Enable AddressSanitizer checks" OFF) -option(USE_UBSAN "Enable UndefinedBehaviorSanitizer checks" OFF) -option(USE_TSAN "Enable ThreadSanitizer checks" OFF) -option(USE_MSAN "Enable MemorySanitizer checks" OFF) -option(USE_VALGRIND "Enable Valgrind instrumentation" OFF) -option(USE_GCOV "Enable gcov support" OFF) +option(UMF_USE_ASAN "Enable AddressSanitizer checks" OFF) +option(UMF_USE_UBSAN "Enable UndefinedBehaviorSanitizer checks" OFF) +option(UMF_USE_TSAN "Enable ThreadSanitizer checks" OFF) +option(UMF_USE_MSAN "Enable MemorySanitizer checks" OFF) +option(UMF_USE_VALGRIND "Enable Valgrind instrumentation" OFF) +option(UMF_USE_GCOV "Enable gcov support" OFF) option( UMF_DISABLE_HWLOC "Disable features that requires hwloc (OS provider, memory targets, topolgy discovery)" @@ -266,16 +266,16 @@ if(MSVC) endif() # Sanitizer flags -if(USE_ASAN) +if(UMF_USE_ASAN) add_sanitizer_flag(address) endif() -if(USE_UBSAN) +if(UMF_USE_UBSAN) add_sanitizer_flag(undefined) endif() -if(USE_TSAN) +if(UMF_USE_TSAN) add_sanitizer_flag(thread) endif() -if(USE_MSAN) +if(UMF_USE_MSAN) message(WARNING "MemorySanitizer requires instrumented libraries to " "prevent reporting false-positives") add_sanitizer_flag(memory) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 771fa1211..e350cd8d0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -207,12 +207,12 @@ origin: https://dependency_origin.com ## Code coverage After adding a new functionality add tests and check coverage before and after the change. -To do this, enable coverage instrumentation by turning on the USE_GCOV flag in CMake. +To do this, enable coverage instrumentation by turning on the UMF_USE_GCOV flag in CMake. Coverage instrumentation feature is supported only by GCC and Clang. An example flow might look like the following: ```bash -$ cmake -B build -DUSE_GCOV=1 -DCMAKE_BUILD_TYPE=Debug +$ cmake -B build -DUMF_USE_GCOV=1 -DCMAKE_BUILD_TYPE=Debug $ cmake --build build -j $ cd build $ ctest diff --git a/README.md b/README.md index 808965c5f..528fe9c8b 100644 --- a/README.md +++ b/README.md @@ -113,12 +113,12 @@ List of options provided by CMake: | UMF_DEVELOPER_MODE | Enable additional developer checks | ON/OFF | OFF | | UMF_FORMAT_CODE_STYLE | Add clang, cmake, and black -format-check and -format-apply targets to make | ON/OFF | OFF | | UMF_TESTS_FAIL_ON_SKIP | Treat skips in tests as fail | ON/OFF | OFF | -| USE_ASAN | Enable AddressSanitizer checks | ON/OFF | OFF | -| USE_UBSAN | Enable UndefinedBehaviorSanitizer checks | ON/OFF | OFF | -| USE_TSAN | Enable ThreadSanitizer checks | ON/OFF | OFF | -| USE_MSAN | Enable MemorySanitizer checks | ON/OFF | OFF | -| USE_VALGRIND | Enable Valgrind instrumentation | ON/OFF | OFF | -| USE_GCOV | Enable gcov support (Linux only) | ON/OFF | OFF | +| UMF_USE_ASAN | Enable AddressSanitizer checks | ON/OFF | OFF | +| UMF_USE_UBSAN | Enable UndefinedBehaviorSanitizer checks | ON/OFF | OFF | +| UMF_USE_TSAN | Enable ThreadSanitizer checks | ON/OFF | OFF | +| UMF_USE_MSAN | Enable MemorySanitizer checks | ON/OFF | OFF | +| UMF_USE_VALGRIND | Enable Valgrind instrumentation | ON/OFF | OFF | +| UMF_USE_GCOV | Enable gcov support (Linux only) | ON/OFF | OFF | | UMF_LINK_HWLOC_STATICALLY | Link UMF with HWLOC library statically (Windows+Release only) | ON/OFF | OFF | ## Architecture: memory pools and providers diff --git a/cmake/helpers.cmake b/cmake/helpers.cmake index a4f59a763..1d3e175fa 100644 --- a/cmake/helpers.cmake +++ b/cmake/helpers.cmake @@ -218,7 +218,7 @@ function(add_umf_target_compile_options name) target_compile_options(${name} PRIVATE -fno-omit-frame-pointer -fstack-protector-strong) endif() - if(USE_GCOV) + if(UMF_USE_GCOV) if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug") message(FATAL_ERROR "To use gcov, the build type must be Debug") endif() @@ -254,7 +254,7 @@ function(add_umf_target_link_options name) if(NOT MSVC) if(NOT APPLE) target_link_options(${name} PRIVATE "LINKER:-z,relro,-z,now") - if(USE_GCOV) + if(UMF_USE_GCOV) if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug") message( FATAL_ERROR "To use gcov, the build type must be Debug") diff --git a/src/utils/CMakeLists.txt b/src/utils/CMakeLists.txt index 78cd0e129..c7a285ce2 100644 --- a/src/utils/CMakeLists.txt +++ b/src/utils/CMakeLists.txt @@ -13,11 +13,11 @@ set(UMF_UTILS_SOURCES_POSIX utils_posix_common.c utils_posix_concurrency.c set(UMF_UTILS_SOURCES_WINDOWS utils_windows_common.c utils_windows_concurrency.c utils_windows_math.c) -if(USE_VALGRIND) - if(USE_ASAN - OR USE_TSAN - OR USE_UBSAN - OR USE_MSAN) +if(UMF_USE_VALGRIND) + if(UMF_USE_ASAN + OR UMF_USE_TSAN + OR UMF_USE_UBSAN + OR UMF_USE_MSAN) message(FATAL_ERROR "Cannot use valgrind and sanitizers together") endif() @@ -51,7 +51,7 @@ target_include_directories( $ $) -if(USE_VALGRIND) +if(UMF_USE_VALGRIND) set(UMF_UTILS_INTERFACE_DEFS "UMF_VG_ENABLED=1") endif() diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index c77a6e326..2eb16e3b7 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -390,10 +390,10 @@ endif() if(LINUX AND UMF_BUILD_SHARED_LIBRARY AND NOT - (USE_ASAN - OR USE_UBSAN - OR USE_TSAN - OR USE_MSAN)) + (UMF_USE_ASAN + OR UMF_USE_UBSAN + OR UMF_USE_TSAN + OR UMF_USE_MSAN)) set(EXAMPLES "") if(UMF_POOL_SCALABLE_ENABLED) set(EXAMPLES ${EXAMPLES} basic) From 32fd31546c61e93f45f4ac994cfda149736ade3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Fri, 2 Aug 2024 12:51:26 +0200 Subject: [PATCH 41/52] Properly prefix test_common lib with 'umf_' --- test/CMakeLists.txt | 2 +- test/common/CMakeLists.txt | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 2eb16e3b7..9af694489 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -45,7 +45,7 @@ function(build_umf_test) endif() set(TEST_LIBS - test_common + umf_test_common ${ARG_LIBS} GTest::gtest_main ${LIBS_OPTIONAL} diff --git a/test/common/CMakeLists.txt b/test/common/CMakeLists.txt index 14f36ce8e..4f88fd7d8 100644 --- a/test/common/CMakeLists.txt +++ b/test/common/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (C) 2023 Intel Corporation +# Copyright (C) 2023-2024 Intel Corporation # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -10,8 +10,9 @@ set(COMMON_SOURCES provider_trace.c) add_umf_library( - NAME test_common + NAME umf_test_common TYPE STATIC SRCS ${COMMON_SOURCES}) -target_include_directories(test_common PRIVATE ${UMF_CMAKE_SOURCE_DIR}/include) +target_include_directories(umf_test_common + PRIVATE ${UMF_CMAKE_SOURCE_DIR}/include) From 37732d30826483fadc42de7e19951122c01d0539 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Fri, 2 Aug 2024 13:10:33 +0200 Subject: [PATCH 42/52] Fix or add missing include guards Add UMF_ prefix, where missing. In bench, tests, and examples use UMF_X_ prefix. --- benchmark/multithread.hpp | 5 +++++ examples/common/utils_level_zero.h | 5 +++++ src/cpp_helpers.hpp | 6 +++--- src/critnib/critnib.h | 8 ++++---- src/utils/utils_math.h | 7 ++++++- test/common/base.hpp | 2 +- test/common/ipc_common.h | 6 +++--- test/common/ipc_os_prov_common.h | 6 +++--- test/common/multithread_helpers.hpp | 5 +++++ test/common/numa_helpers.h | 6 +++--- test/common/pool.hpp | 2 +- test/common/pool_null.h | 8 ++++---- test/common/pool_trace.h | 8 ++++---- test/common/provider_null.h | 8 ++++---- test/common/provider_trace.h | 8 ++++---- test/fuzz/utils.hpp | 5 +++++ test/memspaces/memspace_fixtures.hpp | 7 ++++--- test/memspaces/memspace_helpers.hpp | 6 +++--- test/providers/ipc_level_zero_prov_common.h | 6 +++--- test/providers/level_zero_helpers.h | 6 +++--- 20 files changed, 73 insertions(+), 47 deletions(-) diff --git a/benchmark/multithread.hpp b/benchmark/multithread.hpp index 3eac38a2d..e642d2987 100644 --- a/benchmark/multithread.hpp +++ b/benchmark/multithread.hpp @@ -7,6 +7,9 @@ * */ +#ifndef UMF_BENCH_MULTITHREAD_HPP +#define UMF_BENCH_MULTITHREAD_HPP + #include #include #include @@ -87,3 +90,5 @@ template double std_dev(const std::vector &values) { } } // namespace umf_bench + +#endif /* UMF_BENCH_MULTITHREAD_HPP */ diff --git a/examples/common/utils_level_zero.h b/examples/common/utils_level_zero.h index 8786e7dea..46f892278 100644 --- a/examples/common/utils_level_zero.h +++ b/examples/common/utils_level_zero.h @@ -7,6 +7,9 @@ * */ +#ifndef UMF_EXAMPLE_UTILS_LEVEL_ZERO_H +#define UMF_EXAMPLE_UTILS_LEVEL_ZERO_H + #include #include @@ -412,3 +415,5 @@ int destroy_context(ze_context_handle_t context) { return 0; } + +#endif // UMF_EXAMPLE_UTILS_LEVEL_ZERO_H diff --git a/src/cpp_helpers.hpp b/src/cpp_helpers.hpp index 2145a7df4..86204a20e 100644 --- a/src/cpp_helpers.hpp +++ b/src/cpp_helpers.hpp @@ -7,8 +7,8 @@ * */ -#ifndef UMF_HELPERS_H -#define UMF_HELPERS_H 1 +#ifndef UMF_HELPERS_HPP +#define UMF_HELPERS_HPP 1 #include #include @@ -164,4 +164,4 @@ template umf_result_t &getPoolLastStatusRef() { } // namespace umf -#endif /* UMF_HELPERS_H */ +#endif /* UMF_HELPERS_HPP */ diff --git a/src/critnib/critnib.h b/src/critnib/critnib.h index 868622ea5..e03780374 100644 --- a/src/critnib/critnib.h +++ b/src/critnib/critnib.h @@ -1,14 +1,14 @@ /* * - * Copyright (C) 2023 Intel Corporation + * Copyright (C) 2023-2024 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception * */ -#ifndef CRITNIB_H -#define CRITNIB_H 1 +#ifndef UMF_CRITNIB_H +#define UMF_CRITNIB_H 1 #include @@ -44,4 +44,4 @@ void critnib_iter(critnib *c, uintptr_t min, uintptr_t max, } #endif -#endif +#endif // UMF_CRITNIB_H diff --git a/src/utils/utils_math.h b/src/utils/utils_math.h index 636ffa35d..c78be1136 100644 --- a/src/utils/utils_math.h +++ b/src/utils/utils_math.h @@ -1,12 +1,15 @@ /* * - * Copyright (C) 2023 Intel Corporation + * Copyright (C) 2023-2024 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception * */ +#ifndef UMF_MATH_H +#define UMF_MATH_H 1 + #include #include @@ -22,3 +25,5 @@ static inline size_t log2Utils(size_t num) { return getLeftmostSetBitPos(num); } #ifdef __cplusplus } #endif + +#endif /* UMF_MATH_H */ diff --git a/test/common/base.hpp b/test/common/base.hpp index e58bc35b1..8f2d5f6be 100644 --- a/test/common/base.hpp +++ b/test/common/base.hpp @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023 Intel Corporation + * Copyright (C) 2023-2024 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/test/common/ipc_common.h b/test/common/ipc_common.h index f51a90b46..a73b01435 100644 --- a/test/common/ipc_common.h +++ b/test/common/ipc_common.h @@ -5,8 +5,8 @@ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception */ -#ifndef IPC_COMMON_H -#define IPC_COMMON_H +#ifndef UMF_TEST_IPC_COMMON_H +#define UMF_TEST_IPC_COMMON_H #include @@ -24,4 +24,4 @@ int run_consumer(int port, umf_memory_provider_ops_t *provider_ops, void *provider_params, memcopy_callback_t memcopy_callback, void *memcopy_ctx); -#endif // IPC_COMMON_H +#endif // UMF_TEST_IPC_COMMON_H diff --git a/test/common/ipc_os_prov_common.h b/test/common/ipc_os_prov_common.h index adcb27e0a..386c9658d 100644 --- a/test/common/ipc_os_prov_common.h +++ b/test/common/ipc_os_prov_common.h @@ -5,11 +5,11 @@ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception */ -#ifndef IPC_OS_PROV_COMMON_H -#define IPC_OS_PROV_COMMON_H +#ifndef UMF_TEST_IPC_OS_PROV_COMMON_H +#define UMF_TEST_IPC_OS_PROV_COMMON_H #include void memcopy(void *dst, const void *src, size_t size, void *context); -#endif // IPC_OS_PROV_COMMON_H +#endif // UMF_TEST_IPC_OS_PROV_COMMON_H diff --git a/test/common/multithread_helpers.hpp b/test/common/multithread_helpers.hpp index 28b91d8ed..500705b92 100644 --- a/test/common/multithread_helpers.hpp +++ b/test/common/multithread_helpers.hpp @@ -7,6 +7,9 @@ * */ +#ifndef UMF_TEST_MULTITHREAD_HELPERS_HPP +#define UMF_TEST_MULTITHREAD_HELPERS_HPP + #include #include #include @@ -84,3 +87,5 @@ struct syncthreads_barrier { }; } // namespace umf_test + +#endif /* UMF_TEST_MULTITHREAD_HELPERS_HPP */ diff --git a/test/common/numa_helpers.h b/test/common/numa_helpers.h index 960d2e850..aa9888fea 100644 --- a/test/common/numa_helpers.h +++ b/test/common/numa_helpers.h @@ -2,8 +2,8 @@ // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#ifndef UMF_NUMA_HELPERS_H -#define UMF_NUMA_HELPERS_H 1 +#ifndef UMF_TEST_NUMA_HELPERS_H +#define UMF_TEST_NUMA_HELPERS_H 1 #include #include @@ -31,4 +31,4 @@ int getNumaNodeByPtr(void *ptr) { } #endif -#endif /* UMF_NUMA_HELPERS_H */ +#endif /* UMF_TEST_NUMA_HELPERS_H */ diff --git a/test/common/pool.hpp b/test/common/pool.hpp index 556e9d23c..9a5739085 100644 --- a/test/common/pool.hpp +++ b/test/common/pool.hpp @@ -1,6 +1,6 @@ /* * - * Copyright (C) 2023 Intel Corporation + * Copyright (C) 2023-2024 Intel Corporation * * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception diff --git a/test/common/pool_null.h b/test/common/pool_null.h index 5130dfcd8..2a5a14d75 100644 --- a/test/common/pool_null.h +++ b/test/common/pool_null.h @@ -1,9 +1,9 @@ -// Copyright (C) 2023 Intel Corporation +// Copyright (C) 2023-2024 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#ifndef UMF_NULL_POOL_H -#define UMF_NULL_POOL_H +#ifndef UMF_TEST_NULL_POOL_H +#define UMF_TEST_NULL_POOL_H #include @@ -17,4 +17,4 @@ extern umf_memory_pool_ops_t UMF_NULL_POOL_OPS; } #endif -#endif // UMF_NULL_POOL_H +#endif // UMF_TEST_NULL_POOL_H diff --git a/test/common/pool_trace.h b/test/common/pool_trace.h index 79f4d5eba..c49d61107 100644 --- a/test/common/pool_trace.h +++ b/test/common/pool_trace.h @@ -1,9 +1,9 @@ -// Copyright (C) 2023 Intel Corporation +// Copyright (C) 2023-2024 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#ifndef UMF_TRACE_POOL_H -#define UMF_TRACE_POOL_H +#ifndef UMF_TEST_TRACE_POOL_H +#define UMF_TEST_TRACE_POOL_H #include @@ -24,4 +24,4 @@ extern umf_memory_pool_ops_t UMF_TRACE_POOL_OPS; } #endif -#endif // UMF_TRACE_POOL_H +#endif // UMF_TEST_TRACE_POOL_H diff --git a/test/common/provider_null.h b/test/common/provider_null.h index d7dab9cb7..fa7f2f6db 100644 --- a/test/common/provider_null.h +++ b/test/common/provider_null.h @@ -1,9 +1,9 @@ -// Copyright (C) 2023 Intel Corporation +// Copyright (C) 2023-2024 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#ifndef UMF_NULL_PROVIDER_H -#define UMF_NULL_PROVIDER_H +#ifndef UMF_TEST_NULL_PROVIDER_H +#define UMF_TEST_NULL_PROVIDER_H #include @@ -17,4 +17,4 @@ extern umf_memory_provider_ops_t UMF_NULL_PROVIDER_OPS; } #endif -#endif // UMF_NULL_PROVIDER_H +#endif // UMF_TEST_NULL_PROVIDER_H diff --git a/test/common/provider_trace.h b/test/common/provider_trace.h index bb1bbd33c..f0f2a367b 100644 --- a/test/common/provider_trace.h +++ b/test/common/provider_trace.h @@ -1,9 +1,9 @@ -// Copyright (C) 2023 Intel Corporation +// Copyright (C) 2023-2024 Intel Corporation // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#ifndef UMF_TRACE_PROVIDER_H -#define UMF_TRACE_PROVIDER_H +#ifndef UMF_TEST_TRACE_PROVIDER_H +#define UMF_TEST_TRACE_PROVIDER_H #include @@ -27,4 +27,4 @@ extern umf_memory_provider_ops_t UMF_TRACE_PROVIDER_OPS; } #endif -#endif // UMF_TRACE_PROVIDER_H +#endif // UMF_TEST_TRACE_PROVIDER_H diff --git a/test/fuzz/utils.hpp b/test/fuzz/utils.hpp index ec1dda700..645353fb2 100644 --- a/test/fuzz/utils.hpp +++ b/test/fuzz/utils.hpp @@ -2,6 +2,9 @@ // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +#ifndef UMF_TEST_FUZZ_UTILS_HPP +#define UMF_TEST_FUZZ_UTILS_HPP + #include "umf/pools/pool_scalable.h" #include "umf/providers/provider_os_memory.h" #include @@ -75,3 +78,5 @@ struct TestState { } }; } // namespace fuzz + +#endif /* UMF_TEST_FUZZ_UTILS_HPP */ diff --git a/test/memspaces/memspace_fixtures.hpp b/test/memspaces/memspace_fixtures.hpp index d408a1352..fac50b031 100644 --- a/test/memspaces/memspace_fixtures.hpp +++ b/test/memspaces/memspace_fixtures.hpp @@ -2,8 +2,9 @@ // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#ifndef UMF_MEMSPACE_FIXTURES_HPP -#define UMF_MEMSPACE_FIXTURES_HPP +#ifndef UMF_TEST_MEMSPACE_FIXTURES_HPP +#define UMF_TEST_MEMSPACE_FIXTURES_HPP + #include #include #include @@ -218,4 +219,4 @@ TEST_P(memspaceProviderTest, allocLocalMt) { } } -#endif /* UMF_MEMSPACE_FIXTURES_HPP */ +#endif /* UMF_TEST_MEMSPACE_FIXTURES_HPP */ diff --git a/test/memspaces/memspace_helpers.hpp b/test/memspaces/memspace_helpers.hpp index a789f0c3e..1adee2607 100644 --- a/test/memspaces/memspace_helpers.hpp +++ b/test/memspaces/memspace_helpers.hpp @@ -2,8 +2,8 @@ // Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#ifndef UMF_MEMSPACE_HELPERS_HPP -#define UMF_MEMSPACE_HELPERS_HPP +#ifndef UMF_TEST_MEMSPACE_HELPERS_HPP +#define UMF_TEST_MEMSPACE_HELPERS_HPP #include "base.hpp" #include "memspace_internal.h" @@ -56,4 +56,4 @@ void getAllocationPolicy(void *ptr, unsigned long maxNodeId, int &mode, allocNodeId = static_cast(nodeId); } -#endif /* UMF_MEMSPACE_HELPERS_HPP */ +#endif /* UMF_TEST_MEMSPACE_HELPERS_HPP */ diff --git a/test/providers/ipc_level_zero_prov_common.h b/test/providers/ipc_level_zero_prov_common.h index fdfdeba6d..dff51d08b 100644 --- a/test/providers/ipc_level_zero_prov_common.h +++ b/test/providers/ipc_level_zero_prov_common.h @@ -5,11 +5,11 @@ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception */ -#ifndef IPC_LEVEL_ZERO_PROV_COMMON_H -#define IPC_LEVEL_ZERO_PROV_COMMON_H +#ifndef UMF_TEST_IPC_LEVEL_ZERO_PROV_COMMON_H +#define UMF_TEST_IPC_LEVEL_ZERO_PROV_COMMON_H #include void memcopy(void *dst, const void *src, size_t size, void *context); -#endif // IPC_LEVEL_ZERO_PROV_COMMON_H +#endif // UMF_TEST_IPC_LEVEL_ZERO_PROV_COMMON_H diff --git a/test/providers/level_zero_helpers.h b/test/providers/level_zero_helpers.h index 2bb8261d5..6cd452c1c 100644 --- a/test/providers/level_zero_helpers.h +++ b/test/providers/level_zero_helpers.h @@ -5,8 +5,8 @@ * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception */ -#ifndef TEST_COMMON_LEVEL_ZERO_HELPERS_HPP -#define TEST_COMMON_LEVEL_ZERO_HELPERS_HPP +#ifndef UMF_TEST_LEVEL_ZERO_HELPERS_H +#define UMF_TEST_LEVEL_ZERO_HELPERS_H #include @@ -45,4 +45,4 @@ create_level_zero_prov_params(umf_usm_memory_type_t memory_type); } #endif -#endif // TEST_COMMON_LEVEL_ZERO_HELPERS_HPP +#endif // UMF_TEST_LEVEL_ZERO_HELPERS_H From 73c2230d23e6e18913d6e308035d83739046a700 Mon Sep 17 00:00:00 2001 From: Igor Chorazewicz Date: Fri, 2 Aug 2024 20:59:34 +0000 Subject: [PATCH 43/52] Disable opencl, cuda and nvml when configuring hwloc Without explicitly disabling them and if a system where hwloc is being build has opencl/cuda/nvml installed those libraries will become hwloc dependencies which would force umf to link with them. The problem manifested on a system where opencl was installed and umf compilation failed due to missing clGetDevices symbol. --- CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3edd86f4c..5e7badf81 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -173,7 +173,8 @@ elseif(NOT UMF_DISABLE_HWLOC) COMMAND ./configure --prefix=${hwloc_targ_BINARY_DIR} --enable-static=yes --enable-shared=no --disable-libxml2 --disable-levelzero - CFLAGS=-fPIC CXXFLAGS=-fPIC + --disable-opencl --disable-cuda --disable-nvml CFLAGS=-fPIC + CXXFLAGS=-fPIC WORKING_DIRECTORY ${hwloc_targ_SOURCE_DIR} OUTPUT ${hwloc_targ_SOURCE_DIR}/Makefile DEPENDS ${hwloc_targ_SOURCE_DIR}/configure) From a7dad77a13e4431e03d9bb17d1f67dbd889cfdcb Mon Sep 17 00:00:00 2001 From: Rafal Rudnicki Date: Mon, 5 Aug 2024 11:44:30 +0200 Subject: [PATCH 44/52] add googletest to 3rd party programs --- licensing/third-party-programs.txt | 34 ++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/licensing/third-party-programs.txt b/licensing/third-party-programs.txt index 4adbd8cf6..54520c141 100644 --- a/licensing/third-party-programs.txt +++ b/licensing/third-party-programs.txt @@ -386,6 +386,40 @@ _______________________________________________________________________________ and prospectively choose to deem waived or otherwise exclude such Section(s) of the License, but only in their entirety and only with respect to the Combined Software. + +_______________________________________________________________________________ + +7. googletest: + + Copyright 2008, Google Inc. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + _______________________________________________________________________________ *Other names and brands may be claimed as the property of others. From 82d7764b90f7ac4e5d1a10c401d4a572ae05be33 Mon Sep 17 00:00:00 2001 From: "Kenneth Benzie (Benie)" Date: Thu, 15 Aug 2024 07:20:20 -0700 Subject: [PATCH 45/52] Use PROJECT_VERSION not CMAKE_PROJECT_VERSION When building as a subproject the `CMAKE_PROJECT_VERSION` is the root project version not the current project version. As such, in a subproject which makes use of UMF, such as UR, the `.so` or `.dll` library version would be set to the UR project version, not the UMF version. This patch fixes this erroneous versioning by using `PROJECT_VERSION` (and derivatives) in place of `CMAKE_PROJECT_VERSION`. --- CMakeLists.txt | 2 +- src/CMakeLists.txt | 4 ++-- src/libumf.rc.in | 2 +- src/proxy_lib/CMakeLists.txt | 3 +-- src/proxy_lib/proxy_lib.rc.in | 2 +- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5e7badf81..c6eb911c7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,7 +23,7 @@ project( VERSION ${UMF_CMAKE_VERSION} LANGUAGES C) -if(CMAKE_PROJECT_VERSION_PATCH GREATER 0) +if(PROJECT_VERSION_PATCH GREATER 0) # set extra variable for Windows dll metadata set(UMF_VERSION_BUGFIX 1) endif() diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6a1e1d475..cb7b489c3 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -168,8 +168,8 @@ if(UMF_BUILD_SHARED_LIBRARY) set_target_properties( umf PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_UMF_OUTPUT_DIRECTORY} - VERSION ${CMAKE_PROJECT_VERSION} - SOVERSION ${CMAKE_PROJECT_VERSION_MAJOR}) + VERSION ${PROJECT_VERSION} + SOVERSION ${PROJECT_VERSION_MAJOR}) else() add_umf_library( NAME umf diff --git a/src/libumf.rc.in b/src/libumf.rc.in index 3915e0a10..7aba79e7e 100644 --- a/src/libumf.rc.in +++ b/src/libumf.rc.in @@ -8,7 +8,7 @@ #include "umf/base.h" -#define UMF_VERNUMBERS @CMAKE_PROJECT_VERSION_MAJOR@,@CMAKE_PROJECT_VERSION_MINOR@,@CMAKE_PROJECT_VERSION_PATCH@,@UMF_VERSION_REVISION@ +#define UMF_VERNUMBERS @PROJECT_VERSION_MAJOR@,@PROJECT_VERSION_MINOR@,@PROJECT_VERSION_PATCH@,@UMF_VERSION_REVISION@ #define _UMF_VERSION "@UMF_VERSION@" #ifdef _DEBUG diff --git a/src/proxy_lib/CMakeLists.txt b/src/proxy_lib/CMakeLists.txt index 379a454d0..d6b07902d 100644 --- a/src/proxy_lib/CMakeLists.txt +++ b/src/proxy_lib/CMakeLists.txt @@ -32,8 +32,7 @@ add_umf_library( LIBS umf_utils ${PROXY_LIBS} LINUX_MAP_FILE ${CMAKE_CURRENT_SOURCE_DIR}/proxy_lib.map WINDOWS_DEF_FILE ${CMAKE_CURRENT_SOURCE_DIR}/proxy_lib.def) -set_target_properties(umf_proxy PROPERTIES SOVERSION - ${CMAKE_PROJECT_VERSION_MAJOR}) +set_target_properties(umf_proxy PROPERTIES SOVERSION ${PROJECT_VERSION_MAJOR}) add_library(${PROJECT_NAME}::proxy ALIAS umf_proxy) diff --git a/src/proxy_lib/proxy_lib.rc.in b/src/proxy_lib/proxy_lib.rc.in index 29c8b0482..dce151ec3 100644 --- a/src/proxy_lib/proxy_lib.rc.in +++ b/src/proxy_lib/proxy_lib.rc.in @@ -8,7 +8,7 @@ #include "umf/base.h" -#define UMF_VERNUMBERS @CMAKE_PROJECT_VERSION_MAJOR@,@CMAKE_PROJECT_VERSION_MINOR@,@CMAKE_PROJECT_VERSION_PATCH@,@UMF_VERSION_REVISION@ +#define UMF_VERNUMBERS @PROJECT_VERSION_MAJOR@,@PROJECT_VERSION_MINOR@,@PROJECT_VERSION_PATCH@,@UMF_VERSION_REVISION@ #define _UMF_VERSION "@UMF_VERSION@" #ifdef _DEBUG From 8f1dc0eea4261f3e9ad037b8eaac0258857345c5 Mon Sep 17 00:00:00 2001 From: Patryk Kaminski Date: Wed, 14 Aug 2024 08:14:22 +0200 Subject: [PATCH 46/52] Disable pciaccess library in hwloc --- CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c6eb911c7..f2e50312a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -172,9 +172,9 @@ elseif(NOT UMF_DISABLE_HWLOC) add_custom_command( COMMAND ./configure --prefix=${hwloc_targ_BINARY_DIR} --enable-static=yes - --enable-shared=no --disable-libxml2 --disable-levelzero - --disable-opencl --disable-cuda --disable-nvml CFLAGS=-fPIC - CXXFLAGS=-fPIC + --enable-shared=no --disable-libxml2 --disable-pciaccess + --disable-levelzero --disable-opencl --disable-cuda --disable-nvml + CFLAGS=-fPIC CXXFLAGS=-fPIC WORKING_DIRECTORY ${hwloc_targ_SOURCE_DIR} OUTPUT ${hwloc_targ_SOURCE_DIR}/Makefile DEPENDS ${hwloc_targ_SOURCE_DIR}/configure) From 5ccf12b581f037cbe2218b68ab29317359ce7e3c Mon Sep 17 00:00:00 2001 From: Igor Chorazewicz Date: Fri, 16 Aug 2024 01:50:40 +0200 Subject: [PATCH 47/52] Fix stype for ze_device_mem_alloc_desc_t in l0 provider --- src/provider/provider_level_zero.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/provider/provider_level_zero.c b/src/provider/provider_level_zero.c index b7a5fde50..3f7340556 100644 --- a/src/provider/provider_level_zero.c +++ b/src/provider/provider_level_zero.c @@ -167,7 +167,7 @@ static umf_result_t ze_memory_provider_alloc(void *provider, size_t size, .pNext = NULL, .flags = 0}; ze_device_mem_alloc_desc_t dev_desc = { - .stype = ZE_STRUCTURE_TYPE_HOST_MEM_ALLOC_DESC, + .stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC, .pNext = NULL, .flags = 0, .ordinal = 0 // TODO From e93b99a7ab8779d421a5e81faf575b1c835f32ff Mon Sep 17 00:00:00 2001 From: Rafal Rudnicki Date: Wed, 21 Aug 2024 16:41:27 +0200 Subject: [PATCH 48/52] fix coverity issues in hwloc --- CMakeLists.txt | 19 +++++++++++++++++-- cmake/fix_coverity_issues.patch | 14 ++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 cmake/fix_coverity_issues.patch diff --git a/CMakeLists.txt b/CMakeLists.txt index f2e50312a..6fc10c5e3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -129,6 +129,12 @@ elseif(WINDOWS AND NOT UMF_DISABLE_HWLOC) set(HWLOC_ENABLE_TESTING OFF) set(HWLOC_SKIP_LSTOPO ON) set(HWLOC_SKIP_TOOLS ON) + set(HWLOC_PATCH + git + apply + ${PROJECT_SOURCE_DIR}/cmake/fix_coverity_issues.patch + || + true) message(STATUS "Will fetch hwloc from ${UMF_HWLOC_REPO}") @@ -136,7 +142,8 @@ elseif(WINDOWS AND NOT UMF_DISABLE_HWLOC) hwloc_targ GIT_REPOSITORY ${UMF_HWLOC_REPO} GIT_TAG ${UMF_HWLOC_TAG} - SOURCE_SUBDIR contrib/windows-cmake/ FIND_PACKAGE_ARGS) + PATCH_COMMAND ${HWLOC_PATCH} SOURCE_SUBDIR contrib/windows-cmake/ + FIND_PACKAGE_ARGS) FetchContent_GetProperties(hwloc_targ) if(NOT hwloc_targ_POPULATED) @@ -153,12 +160,20 @@ elseif(WINDOWS AND NOT UMF_DISABLE_HWLOC) message(STATUS " LIBHWLOC_LIBRARY_DIRS = ${LIBHWLOC_LIBRARY_DIRS}") elseif(NOT UMF_DISABLE_HWLOC) include(FetchContent) + set(HWLOC_PATCH + git + apply + ${PROJECT_SOURCE_DIR}/cmake/fix_coverity_issues.patch + || + true) + message(STATUS "Will fetch hwloc from ${UMF_HWLOC_REPO}") FetchContent_Declare( hwloc_targ GIT_REPOSITORY ${UMF_HWLOC_REPO} - GIT_TAG ${UMF_HWLOC_TAG}) + GIT_TAG ${UMF_HWLOC_TAG} + PATCH_COMMAND ${HWLOC_PATCH}) FetchContent_GetProperties(hwloc_targ) if(NOT hwloc_targ_POPULATED) diff --git a/cmake/fix_coverity_issues.patch b/cmake/fix_coverity_issues.patch new file mode 100644 index 000000000..1edcbd336 --- /dev/null +++ b/cmake/fix_coverity_issues.patch @@ -0,0 +1,14 @@ +diff --git a/hwloc/topology-x86.c b/hwloc/topology-x86.c +index 7aabd168f..b01e44557 100644 +--- a/hwloc/topology-x86.c ++++ b/hwloc/topology-x86.c +@@ -1375,6 +1375,9 @@ look_procs(struct hwloc_backend *backend, struct procinfo *infos, unsigned long + hwloc_bitmap_t set = NULL; + unsigned i; + ++ if(!get_cpubind||!set_cpubind) ++ return -1; ++ + if (!data->src_cpuiddump_path) { + orig_cpuset = hwloc_bitmap_alloc(); + if (get_cpubind(topology, orig_cpuset, HWLOC_CPUBIND_STRICT)) { From 65e465947fe0cceb006f9781ec8964a308fb06e2 Mon Sep 17 00:00:00 2001 From: Rafal Rudnicki Date: Mon, 26 Aug 2024 14:54:30 +0200 Subject: [PATCH 49/52] use *_DLL_DIRS env --- CMakeLists.txt | 6 +++--- cmake/FindJEMALLOC.cmake | 2 +- cmake/FindTBB.cmake | 2 +- examples/cmake/FindLIBHWLOC.cmake | 2 +- examples/cmake/FindTBB.cmake | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6fc10c5e3..8a0234bc6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -117,7 +117,7 @@ if(NOT UMF_LINK_HWLOC_STATICALLY) # add PATH to DLL on Windows set(DLL_PATH_LIST - "${DLL_PATH_LIST};PATH=path_list_append:${LIBHWLOC_LIBRARY_DIRS}/../bin" + "${DLL_PATH_LIST};PATH=path_list_append:${LIBHWLOC_DLL_DIRS}" ) endif() # add PATH to DLL on Windows @@ -337,7 +337,7 @@ endif() if(TBB_FOUND OR TBB_LIBRARY_DIRS) # add PATH to DLL on Windows set(DLL_PATH_LIST - "${DLL_PATH_LIST};PATH=path_list_append:${TBB_LIBRARY_DIRS}/../bin") + "${DLL_PATH_LIST};PATH=path_list_append:${TBB_DLL_DIRS}") set(UMF_POOL_SCALABLE_ENABLED TRUE) else() message( @@ -354,7 +354,7 @@ if(UMF_BUILD_LIBUMF_POOL_JEMALLOC) endif() # add PATH to DLL on Windows set(DLL_PATH_LIST - "${DLL_PATH_LIST};PATH=path_list_append:${JEMALLOC_LIBRARY_DIRS}/../bin" + "${DLL_PATH_LIST};PATH=path_list_append:${JEMALLOC_DLL_DIRS}" ) endif() diff --git a/cmake/FindJEMALLOC.cmake b/cmake/FindJEMALLOC.cmake index f01301d25..89d488ecc 100644 --- a/cmake/FindJEMALLOC.cmake +++ b/cmake/FindJEMALLOC.cmake @@ -28,7 +28,7 @@ else() endif() if(WINDOWS) - find_file(JEMALLOC_DLL NAMES "bin/jemalloc.dll") + find_file(JEMALLOC_DLL NAMES "bin/jemalloc.dll" "jemalloc.dll") get_filename_component(JEMALLOC_DLL_DIR ${JEMALLOC_DLL} DIRECTORY) set(JEMALLOC_DLL_DIRS ${JEMALLOC_DLL_DIR}) endif() diff --git a/cmake/FindTBB.cmake b/cmake/FindTBB.cmake index 8aa6289ef..6536e8c4a 100644 --- a/cmake/FindTBB.cmake +++ b/cmake/FindTBB.cmake @@ -26,7 +26,7 @@ else() endif() if(WINDOWS) - find_file(TBB_DLL NAMES "bin/tbbmalloc.dll") + find_file(TBB_DLL NAMES "bin/tbbmalloc.dll" "tbbmalloc.dll") get_filename_component(TBB_DLL_DIR ${TBB_DLL} DIRECTORY) set(TBB_DLL_DIRS ${TBB_DLL_DIR}) endif() diff --git a/examples/cmake/FindLIBHWLOC.cmake b/examples/cmake/FindLIBHWLOC.cmake index f4e33417a..698946829 100644 --- a/examples/cmake/FindLIBHWLOC.cmake +++ b/examples/cmake/FindLIBHWLOC.cmake @@ -38,7 +38,7 @@ try_run( RUN_OUTPUT_VARIABLE LIBHWLOC_API_VERSION) if(WINDOWS) - find_file(LIBHWLOC_DLL NAMES "bin/hwloc-15.dll" "bin/libhwloc-15.dll") + find_file(LIBHWLOC_DLL NAMES "bin/hwloc-15.dll" "bin/libhwloc-15.dll" "hwloc-15.dll" "libhwloc-15.dll") get_filename_component(LIBHWLOC_DLL_DIR ${LIBHWLOC_DLL} DIRECTORY) set(LIBHWLOC_DLL_DIRS ${LIBHWLOC_DLL_DIR}) endif() diff --git a/examples/cmake/FindTBB.cmake b/examples/cmake/FindTBB.cmake index 8aa6289ef..6536e8c4a 100644 --- a/examples/cmake/FindTBB.cmake +++ b/examples/cmake/FindTBB.cmake @@ -26,7 +26,7 @@ else() endif() if(WINDOWS) - find_file(TBB_DLL NAMES "bin/tbbmalloc.dll") + find_file(TBB_DLL NAMES "bin/tbbmalloc.dll" "tbbmalloc.dll") get_filename_component(TBB_DLL_DIR ${TBB_DLL} DIRECTORY) set(TBB_DLL_DIRS ${TBB_DLL_DIR}) endif() From 490ed4d2fc3f271d7c881581a1bc0d48c5c51387 Mon Sep 17 00:00:00 2001 From: Rafal Rudnicki Date: Mon, 26 Aug 2024 14:54:51 +0200 Subject: [PATCH 50/52] add custom hwloc lib name --- .github/workflows/basic.yml | 50 +++++++++++++++++++++++++++++++ CMakeLists.txt | 16 +++++----- cmake/FindLIBHWLOC.cmake | 5 ++-- examples/cmake/FindLIBHWLOC.cmake | 3 +- src/CMakeLists.txt | 4 +-- 5 files changed, 65 insertions(+), 13 deletions(-) diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml index 736f08eef..7c3d2ebc9 100644 --- a/.github/workflows/basic.yml +++ b/.github/workflows/basic.yml @@ -332,6 +332,56 @@ jobs: -DUMF_TESTS_FAIL_ON_SKIP=ON -DUMF_LINK_HWLOC_STATICALLY=ON + - name: Build UMF + run: cmake --build ${{env.BUILD_DIR}} --config ${{matrix.build_type}} -j $Env:NUMBER_OF_PROCESSORS -v + + - name: Run tests + working-directory: ${{env.BUILD_DIR}} + run: ctest -C ${{matrix.build_type}} --output-on-failure --test-dir test + + windows-dynamic_mingw_hwloc: + env: + HWLOC_PACKAGE_NAME: hwloc-win64-build-2.10.0 + TBB_PACKAGE_NAME: oneapi-tbb-2021.12.0 + TBB_LIB_DIR: lib\intel64\vc14 + TBB_BIN_DIR: redist\intel64\vc14 + + name: "Windows dynamic UMF + mingw libhwloc" + strategy: + matrix: + build_type: [Release] + + runs-on: 'windows-2022' + + steps: + - name: Checkout + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 + + - name: Get hwloc from official repo (mingw version) + run: | + Invoke-WebRequest -Uri https://download.open-mpi.org/release/hwloc/v2.10/${{env.HWLOC_PACKAGE_NAME}}.zip -OutFile ${{github.workspace}}\${{env.HWLOC_PACKAGE_NAME}}.zip -TimeoutSec 360 + Expand-Archive ${{github.workspace}}\${{env.HWLOC_PACKAGE_NAME}}.zip -DestinationPath ${{github.workspace}} + + - name: Get TBB from github + run: | + Invoke-WebRequest -Uri https://github.com/oneapi-src/oneTBB/releases/download/v2021.12.0/${{env.TBB_PACKAGE_NAME}}-win.zip -OutFile "${{github.workspace}}\${{env.TBB_PACKAGE_NAME}}-win.zip" -TimeoutSec 360 + Expand-Archive "${{github.workspace}}\${{env.TBB_PACKAGE_NAME}}-win.zip" -DestinationPath ${{github.workspace}} + + - name: Configure build + run: > + cmake + -B ${{env.BUILD_DIR}} + -DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}" + -DCMAKE_PREFIX_PATH="${{github.workspace}}\${{env.HWLOC_PACKAGE_NAME}};${{github.workspace}}\${{env.TBB_PACKAGE_NAME}};${{github.workspace}}\${{env.TBB_PACKAGE_NAME}}\${{env.TBB_LIB_DIR}};${{github.workspace}}\${{env.TBB_PACKAGE_NAME}}\${{env.TBB_BIN_DIR}}" + -DUMF_BUILD_SHARED_LIBRARY=ON + -DUMF_BUILD_EXAMPLES=ON + -DUMF_FORMAT_CODE_STYLE=OFF + -DUMF_DEVELOPER_MODE=ON + -DUMF_TESTS_FAIL_ON_SKIP=ON + -DUMF_HWLOC_NAME=libhwloc + - name: Build UMF run: cmake --build ${{env.BUILD_DIR}} --config ${{matrix.build_type}} -j $Env:NUMBER_OF_PROCESSORS diff --git a/CMakeLists.txt b/CMakeLists.txt index 8a0234bc6..903999b88 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,6 +55,10 @@ option( option(UMF_FORMAT_CODE_STYLE "Add clang, cmake, and black -format-check and -format-apply targets" OFF) +set(UMF_HWLOC_NAME + "hwloc" + CACHE STRING "Custom name for hwloc library w/o extension") + # Only a part of skips is treated as a failure now. TODO: extend to all tests option(UMF_TESTS_FAIL_ON_SKIP "Treat skips in tests as fail" OFF) option(UMF_USE_ASAN "Enable AddressSanitizer checks" OFF) @@ -117,13 +121,11 @@ if(NOT UMF_LINK_HWLOC_STATICALLY) # add PATH to DLL on Windows set(DLL_PATH_LIST - "${DLL_PATH_LIST};PATH=path_list_append:${LIBHWLOC_DLL_DIRS}" - ) + "${DLL_PATH_LIST};PATH=path_list_append:${LIBHWLOC_DLL_DIRS}") endif() # add PATH to DLL on Windows set(DLL_PATH_LIST - "${DLL_PATH_LIST};PATH=path_list_append:${LIBHWLOC_LIBRARY_DIRS}/../bin" - ) + "${DLL_PATH_LIST};PATH=path_list_append:${LIBHWLOC_DLL_DIRS}") elseif(WINDOWS AND NOT UMF_DISABLE_HWLOC) include(FetchContent) set(HWLOC_ENABLE_TESTING OFF) @@ -336,8 +338,7 @@ if(NOT TBB_FOUND) endif() if(TBB_FOUND OR TBB_LIBRARY_DIRS) # add PATH to DLL on Windows - set(DLL_PATH_LIST - "${DLL_PATH_LIST};PATH=path_list_append:${TBB_DLL_DIRS}") + set(DLL_PATH_LIST "${DLL_PATH_LIST};PATH=path_list_append:${TBB_DLL_DIRS}") set(UMF_POOL_SCALABLE_ENABLED TRUE) else() message( @@ -354,8 +355,7 @@ if(UMF_BUILD_LIBUMF_POOL_JEMALLOC) endif() # add PATH to DLL on Windows set(DLL_PATH_LIST - "${DLL_PATH_LIST};PATH=path_list_append:${JEMALLOC_DLL_DIRS}" - ) + "${DLL_PATH_LIST};PATH=path_list_append:${JEMALLOC_DLL_DIRS}") endif() # set UMF_PROXY_LIB_ENABLED diff --git a/cmake/FindLIBHWLOC.cmake b/cmake/FindLIBHWLOC.cmake index f4e33417a..a7c9201a0 100644 --- a/cmake/FindLIBHWLOC.cmake +++ b/cmake/FindLIBHWLOC.cmake @@ -4,7 +4,7 @@ message(STATUS "Checking for module 'libhwloc' using find_library()") -find_library(LIBHWLOC_LIBRARY NAMES libhwloc hwloc) +find_library(LIBHWLOC_LIBRARY NAMES ${UMF_HWLOC_NAME}) set(LIBHWLOC_LIBRARIES ${LIBHWLOC_LIBRARY}) get_filename_component(LIBHWLOC_LIB_DIR ${LIBHWLOC_LIBRARIES} DIRECTORY) @@ -38,7 +38,8 @@ try_run( RUN_OUTPUT_VARIABLE LIBHWLOC_API_VERSION) if(WINDOWS) - find_file(LIBHWLOC_DLL NAMES "bin/hwloc-15.dll" "bin/libhwloc-15.dll") + find_file(LIBHWLOC_DLL NAMES "bin/${UMF_HWLOC_NAME}-15.dll" + "${UMF_HWLOC_NAME}-15.dll") get_filename_component(LIBHWLOC_DLL_DIR ${LIBHWLOC_DLL} DIRECTORY) set(LIBHWLOC_DLL_DIRS ${LIBHWLOC_DLL_DIR}) endif() diff --git a/examples/cmake/FindLIBHWLOC.cmake b/examples/cmake/FindLIBHWLOC.cmake index 698946829..aa7620bc2 100644 --- a/examples/cmake/FindLIBHWLOC.cmake +++ b/examples/cmake/FindLIBHWLOC.cmake @@ -38,7 +38,8 @@ try_run( RUN_OUTPUT_VARIABLE LIBHWLOC_API_VERSION) if(WINDOWS) - find_file(LIBHWLOC_DLL NAMES "bin/hwloc-15.dll" "bin/libhwloc-15.dll" "hwloc-15.dll" "libhwloc-15.dll") + find_file(LIBHWLOC_DLL NAMES "bin/hwloc-15.dll" "bin/libhwloc-15.dll" + "hwloc-15.dll" "libhwloc-15.dll") get_filename_component(LIBHWLOC_DLL_DIR ${LIBHWLOC_DLL} DIRECTORY) set(LIBHWLOC_DLL_DIRS ${LIBHWLOC_DLL_DIR}) endif() diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index cb7b489c3..0ebd1160f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -154,7 +154,7 @@ endif() if(UMF_BUILD_SHARED_LIBRARY) if(NOT UMF_DISABLE_HWLOC) - set(HWLOC_LIB hwloc) + set(HWLOC_LIB ${UMF_HWLOC_NAME}) endif() add_umf_library( NAME umf @@ -184,7 +184,7 @@ if(UMF_DISABLE_HWLOC) endif() if(UMF_LINK_HWLOC_STATICALLY) - add_dependencies(umf hwloc) + add_dependencies(umf ${UMF_HWLOC_NAME}) endif() target_link_directories(umf PRIVATE ${UMF_PRIVATE_LIBRARY_DIRS}) From e306403aad43b5bb79820af0bb23393371a32c5e Mon Sep 17 00:00:00 2001 From: Nicolas Miller Date: Fri, 6 Sep 2024 09:48:37 +0100 Subject: [PATCH 51/52] Fix hwloc patch application on windows Windows `cmd.exe` doesn't have a `true` command, `(exit 0)` will do the same thing but it will work on both linux and windows. This issue only shows up on Windows and not on a first build, since when building the first time the patch applies cleanly so the `||` branch is not taken, but then when rebuilding the patch doens't apply and the command errors out in the `||` branch. --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 903999b88..cb4045776 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -136,7 +136,7 @@ elseif(WINDOWS AND NOT UMF_DISABLE_HWLOC) apply ${PROJECT_SOURCE_DIR}/cmake/fix_coverity_issues.patch || - true) + (exit 0)) message(STATUS "Will fetch hwloc from ${UMF_HWLOC_REPO}") @@ -167,7 +167,7 @@ elseif(NOT UMF_DISABLE_HWLOC) apply ${PROJECT_SOURCE_DIR}/cmake/fix_coverity_issues.patch || - true) + (exit 0)) message(STATUS "Will fetch hwloc from ${UMF_HWLOC_REPO}") From 57e819999a262b79bcceec47cca24f5015611ea1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Stolarczuk?= Date: Thu, 12 Sep 2024 11:23:52 +0200 Subject: [PATCH 52/52] 0.9.0 release --- ChangeLog | 17 ++++++++++++----- scripts/docs_config/conf.py | 2 +- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/ChangeLog b/ChangeLog index 78c1cc7ef..867e59f0f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,10 +1,17 @@ -Thu Jul 18 2024 Łukasz Stolarczuk +Thu Sep 12 2024 Łukasz Stolarczuk - * Version 0.9.0-rc1 + * Version 0.9.0 - This is a pre-release version. When 0.9.0 will be ready (with no rc status), - it will aim to be the first complete release of the UMF project. - With this release we don't yet guarantee a fully stable API. + This release aims to be the first complete release of the UMF project. + We don't yet guarantee a fully stable API, though. + + This release contains all the features listed in the version 0.1.0 + plus, additionally: + - IPC API + - fixes in the building system + - proper versioning + - minor patches in the source code + - improved CI and docs Thu Jul 04 2024 Łukasz Stolarczuk diff --git a/scripts/docs_config/conf.py b/scripts/docs_config/conf.py index e0cf028a8..b93d7d977 100644 --- a/scripts/docs_config/conf.py +++ b/scripts/docs_config/conf.py @@ -22,7 +22,7 @@ author = "Intel" # The full version, including alpha/beta/rc tags -release = "0.9.0-rc1" +release = "0.9.0" # -- General configuration ---------------------------------------------------