diff --git a/.github/workflows/pr_push.yml b/.github/workflows/pr_push.yml index 8ad9c83c1..c8cce6c2b 100644 --- a/.github/workflows/pr_push.yml +++ b/.github/workflows/pr_push.yml @@ -17,31 +17,17 @@ permissions: packages: read jobs: - CodeChecks: - uses: ./.github/workflows/reusable_checks.yml - FastBuild: - name: Fast builds - needs: [CodeChecks] - uses: ./.github/workflows/reusable_fast.yml - Build: - name: Basic builds - needs: [FastBuild] - uses: ./.github/workflows/reusable_basic.yml DevDax: - needs: [FastBuild] uses: ./.github/workflows/reusable_dax.yml MultiNuma: - needs: [FastBuild] uses: ./.github/workflows/reusable_multi_numa.yml L0: - needs: [Build] uses: ./.github/workflows/reusable_gpu.yml with: provider: "LEVEL_ZERO" runner: "L0" shared_lib: "['ON']" L0-BMG: - needs: [Build] uses: ./.github/workflows/reusable_gpu.yml with: provider: "LEVEL_ZERO" @@ -49,56 +35,16 @@ jobs: shared_lib: "['ON']" os: "['Ubuntu']" CUDA: - needs: [Build] uses: ./.github/workflows/reusable_gpu.yml with: provider: "CUDA" runner: "CUDA" shared_lib: "['ON']" Sanitizers: - needs: [FastBuild] uses: ./.github/workflows/reusable_sanitizers.yml QEMU: - needs: [FastBuild] uses: ./.github/workflows/reusable_qemu.yml with: short_run: true ProxyLib: - needs: [Build] uses: ./.github/workflows/reusable_proxy_lib.yml - Valgrind: - needs: [Build] - uses: ./.github/workflows/reusable_valgrind.yml - Coverage: - # total coverage (on upstream only) - if: github.repository == 'oneapi-src/unified-memory-framework' - needs: [Build, DevDax, L0, CUDA, MultiNuma, QEMU, ProxyLib] - uses: ./.github/workflows/reusable_coverage.yml - secrets: inherit - with: - trigger: "${{github.event_name}}" - Coverage_partial: - # partial coverage (on forks) - if: github.repository != 'oneapi-src/unified-memory-framework' - needs: [Build, QEMU, ProxyLib] - uses: ./.github/workflows/reusable_coverage.yml - CodeQL: - needs: [Build] - permissions: - contents: read - security-events: write - uses: ./.github/workflows/reusable_codeql.yml - Trivy: - needs: [Build] - permissions: - contents: read - security-events: write - uses: ./.github/workflows/reusable_trivy.yml - Compatibility: - needs: [Build] - uses: ./.github/workflows/reusable_compatibility.yml - strategy: - matrix: - tag: ["v0.11.0-rc1"] - with: - tag: ${{matrix.tag}} diff --git a/src/critnib/critnib.c b/src/critnib/critnib.c index 47a6568c5..d0c5275dc 100644 --- a/src/critnib/critnib.c +++ b/src/critnib/critnib.c @@ -331,18 +331,26 @@ static void free_leaf(struct critnib *__restrict c, return; } + // k should be added to the c->deleted_leaf list here + // or in critnib_release() when the reference count drops to 0. + utils_atomic_store_release_u8(&k->pending_deleted_leaf, 1); + if (c->cb_free_leaf) { uint64_t ref_count; utils_atomic_load_acquire_u64(&k->ref_count, &ref_count); if (ref_count > 0) { - // k will be added to c->deleted_leaf in critnib_release() + // k will be added to the c->deleted_leaf list in critnib_release() // when the reference count drops to 0. - utils_atomic_store_release_u8(&k->pending_deleted_leaf, 1); return; } } - add_to_deleted_leaf_list(c, k); + uint8_t expected = 1; + uint8_t desired = 0; + if (utils_compare_exchange_u8(&k->pending_deleted_leaf, &expected, + &desired)) { + add_to_deleted_leaf_list(c, k); + } } /* @@ -392,8 +400,8 @@ int critnib_insert(struct critnib *c, word key, void *value, int update) { utils_atomic_store_release_u8(&k->pending_deleted_leaf, 0); if (c->cb_free_leaf) { - // mark the leaf as valid (ref_count == 1) - utils_atomic_store_release_u64(&k->ref_count, 1ULL); + // mark the leaf as valid (ref_count == 2) + utils_atomic_store_release_u64(&k->ref_count, 2ULL); } else { // the reference counter is not used in this case utils_atomic_store_release_u64(&k->ref_count, 0ULL); @@ -609,20 +617,25 @@ int critnib_release(struct critnib *c, void *ref) { } /* decrement the reference count */ - if (utils_atomic_decrement_u64(&k->ref_count) == 0) { + if (utils_atomic_decrement_u64(&k->ref_count) == 1) { void *to_be_freed = NULL; utils_atomic_load_acquire_ptr(&k->to_be_freed, &to_be_freed); if (to_be_freed) { utils_atomic_store_release_ptr(&k->to_be_freed, NULL); c->cb_free_leaf(c->leaf_allocator, to_be_freed); } - uint8_t pending_deleted_leaf; - utils_atomic_load_acquire_u8(&k->pending_deleted_leaf, - &pending_deleted_leaf); - if (pending_deleted_leaf) { - utils_atomic_store_release_u8(&k->pending_deleted_leaf, 0); + + // mark the leaf as not used (ref_count == 0) + utils_atomic_store_release_u64(&k->ref_count, 0ULL); + + uint8_t expected = 1; + uint8_t desired = 0; + if (utils_compare_exchange_u8(&k->pending_deleted_leaf, &expected, + &desired)) { add_to_deleted_leaf_list(c, k); } + + return 0; } #ifndef NDEBUG diff --git a/src/utils/utils_concurrency.h b/src/utils/utils_concurrency.h index 14d7fe54e..a00b8bc40 100644 --- a/src/utils/utils_concurrency.h +++ b/src/utils/utils_concurrency.h @@ -171,6 +171,19 @@ static inline bool utils_compare_exchange_u64(uint64_t *ptr, uint64_t *expected, return false; } +static inline bool utils_compare_exchange_u8(uint8_t *ptr, uint8_t *expected, + uint8_t *desired) { + char out = _InterlockedCompareExchange8( + (char volatile *)ptr, *(char *)desired, *(char *)expected); + if (out == *(char *)expected) { + return true; + } + + // else + *expected = out; + return false; +} + #else // !defined(_WIN32) static inline void utils_atomic_load_acquire_u64(uint64_t *ptr, uint64_t *out) { @@ -241,6 +254,13 @@ static inline bool utils_compare_exchange_u64(uint64_t *ptr, uint64_t *expected, memory_order_relaxed); } +static inline bool utils_compare_exchange_u8(uint8_t *ptr, uint8_t *expected, + uint8_t *desired) { + return __atomic_compare_exchange(ptr, expected, desired, 0 /* strong */, + memory_order_acq_rel, + memory_order_relaxed); +} + #endif // !defined(_WIN32) static inline void utils_atomic_load_acquire_size_t(size_t *ptr, size_t *out) {