diff --git a/.cmake-format b/.cmake-format index 57ad821ef..ed478467a 100644 --- a/.cmake-format +++ b/.cmake-format @@ -9,31 +9,33 @@ with section("parse"): "pargs": 0, "flags": [], 'kwargs': { - 'NAME': '*', - 'SRCS': '*', - 'LIBS': '*' , - 'LIBDIRS': '*'}}, + 'NAME': '*', + 'SRCS': '*', + 'LIBS': '*' , + 'LIBDIRS': '*', + 'TESTARGS': '*'}}, 'add_umf_executable': { "pargs": 0, "flags": [], 'kwargs': { - 'NAME': '*', - 'SRCS': '*', + 'NAME': '*', + 'SRCS': '*', 'LIBS': '*'}}, 'add_umf_test': { "pargs": 0, "flags": [], 'kwargs': { - 'NAME': '*', - 'SRCS': '*', - 'LIBS': '*'}}, + 'NAME': '*', + 'SRCS': '*', + 'LIBS': '*', + 'ENVS': '*'}}, 'add_umf_library': { "pargs": 0, "flags": [], 'kwargs': { - 'NAME': '*', - 'TYPE': '*', - 'SRCS': '*', + 'NAME': '*', + 'TYPE': '*', + 'SRCS': '*', 'LIBS': '*', 'LINUX_MAP_FILE': '*', 'WINDOWS_DEF_FILE': '*'}}, @@ -43,7 +45,20 @@ with section("parse"): 'kwargs': { 'LABELS': '*', 'PASS_REGULAR_EXPRESSION': '*'}}, - } + 'build_umf_test': { + "pargs": 0, + "flags": [], + 'kwargs': { + 'NAME': '*', + 'SRCS': '*', + 'LIBS': '*' }}, + 'add_umf_ipc_test': { + "pargs": 0, + "flags": [], + 'kwargs': { + 'TEST': '*', + 'SRC_DIR': '*'}}, + } # Override configurations per-command where available override_spec = {} diff --git a/.github/dependabot.yml b/.github/dependabot.yml index f00070d21..c24160fbb 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -14,3 +14,12 @@ updates: applies-to: version-updates patterns: - "*" + - package-ecosystem: "github-actions" + directory: "/.github/workflows/" + schedule: + interval: "daily" + groups: + actions-dependencies: + applies-to: version-updates + patterns: + - "*" diff --git a/.github/docker/ubuntu-20.04.Dockerfile b/.github/docker/ubuntu-20.04.Dockerfile index 069deeac9..8869303b7 100644 --- a/.github/docker/ubuntu-20.04.Dockerfile +++ b/.github/docker/ubuntu-20.04.Dockerfile @@ -1,10 +1,10 @@ -# Copyright (C) 2024 Intel Corporation +# Copyright (C) 2024-2025 Intel Corporation # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # # Dockerfile - a 'recipe' for Docker to build an image of ubuntu-based -# environment for building the Unified Memory Framework project. +# environment for building the Unified Memory Framework project. # # Pull base image ("20.04") @@ -22,43 +22,50 @@ ARG BASE_DEPS="\ cmake \ git" -# UMF's dependencies -ARG UMF_DEPS="\ - libjemalloc-dev \ - libhwloc-dev \ - libtbb-dev" +# Hwloc installation dependencies +ARG HWLOC_DEPS="\ + libtool" # Dependencies for tests (optional) ARG TEST_DEPS="\ - libnuma-dev" + libnuma-dev \ + libtbb-dev \ + valgrind" # Miscellaneous for our builds/CI (optional) ARG MISC_DEPS="\ + automake \ clang \ g++-7 \ + lcov \ python3-pip \ sudo \ whois" +# libhwloc-dev is required - installed via script because hwloc version is too old on this OS +COPY .github/scripts/install_hwloc.sh /opt/umf/install_hwloc.sh + # Update and install required packages RUN apt-get update \ && apt-get install -y --no-install-recommends \ ${BASE_DEPS} \ - ${UMF_DEPS} \ ${TEST_DEPS} \ ${MISC_DEPS} \ + ${HWLOC_DEPS} \ + && /opt/umf/install_hwloc.sh \ && rm -rf /var/lib/apt/lists/* \ && apt-get clean all # Prepare a dir (accessible by anyone) -RUN mkdir --mode 777 /opt/umf/ +RUN mkdir -p --mode 777 /opt/umf/ # Additional dependencies (installed via pip) +# It's actively used and tested only on selected distros. Be aware +# they may not work, because pip packages list differ from OS to OS. COPY third_party/requirements.txt /opt/umf/requirements.txt -RUN pip3 install --no-cache-dir -r /opt/umf/requirements.txt # Add a new (non-root) 'test_user' ENV USER test_user ENV USERPASS pass -RUN useradd -m "${USER}" -g sudo -p "$(mkpasswd ${USERPASS})" +RUN useradd -m -u 1001 "${USER}" -g sudo -p "$(mkpasswd ${USERPASS})" USER test_user diff --git a/.github/docker/ubuntu-22.04.Dockerfile b/.github/docker/ubuntu-22.04.Dockerfile index 08d546083..915d92c26 100644 --- a/.github/docker/ubuntu-22.04.Dockerfile +++ b/.github/docker/ubuntu-22.04.Dockerfile @@ -1,4 +1,4 @@ -# Copyright (C) 2024 Intel Corporation +# Copyright (C) 2024-2025 Intel Corporation # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -17,24 +17,30 @@ ENV NOTTY 1 ENV DEBIAN_FRONTEND noninteractive # Base development packages +# It seems that libtool is not directly needed +# but it is still required when Building UMF ARG BASE_DEPS="\ build-essential \ cmake \ - git" + git \ + libtool \ + wget" # UMF's dependencies ARG UMF_DEPS="\ - libjemalloc-dev \ - libhwloc-dev \ - libtbb-dev" + libhwloc-dev" # Dependencies for tests (optional) ARG TEST_DEPS="\ - libnuma-dev" + libnuma-dev \ + libtbb-dev \ + valgrind" # Miscellaneous for our builds/CI (optional) ARG MISC_DEPS="\ + automake \ clang \ + lcov \ python3-pip \ sudo \ whois" @@ -43,14 +49,14 @@ ARG MISC_DEPS="\ RUN apt-get update \ && apt-get install -y --no-install-recommends \ ${BASE_DEPS} \ - ${UMF_DEPS} \ ${TEST_DEPS} \ ${MISC_DEPS} \ + ${UMF_DEPS} \ && rm -rf /var/lib/apt/lists/* \ && apt-get clean all # Prepare a dir (accessible by anyone) -RUN mkdir --mode 777 /opt/umf/ +RUN mkdir -p --mode 777 /opt/umf/ # Additional dependencies (installed via pip) COPY third_party/requirements.txt /opt/umf/requirements.txt @@ -59,5 +65,5 @@ RUN pip3 install --no-cache-dir -r /opt/umf/requirements.txt # Add a new (non-root) 'test_user' ENV USER test_user ENV USERPASS pass -RUN useradd -m "${USER}" -g sudo -p "$(mkpasswd ${USERPASS})" +RUN useradd -m -u 1001 "${USER}" -g sudo -p "$(mkpasswd ${USERPASS})" USER test_user diff --git a/.github/docker/ubuntu-24.04.Dockerfile b/.github/docker/ubuntu-24.04.Dockerfile new file mode 100644 index 000000000..e56338ab9 --- /dev/null +++ b/.github/docker/ubuntu-24.04.Dockerfile @@ -0,0 +1,66 @@ +# Copyright (C) 2025 Intel Corporation +# Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# +# Dockerfile - a 'recipe' for Docker to build an image of ubuntu-based +# environment for building the Unified Memory Framework project. +# + +# Pull base image ("24.04") +FROM registry.hub.docker.com/library/ubuntu@sha256:72297848456d5d37d1262630108ab308d3e9ec7ed1c3286a32fe09856619a782 + +# Set environment variables +ENV OS ubuntu +ENV OS_VER 24.04 +ENV NOTTY 1 +ENV DEBIAN_FRONTEND noninteractive + +# Base development packages +ARG BASE_DEPS="\ + build-essential \ + cmake \ + git \ + wget" + +# UMF's dependencies +ARG UMF_DEPS="\ + libhwloc-dev" + +# Dependencies for tests (optional) +ARG TEST_DEPS="\ + libnuma-dev \ + libtbb-dev \ + valgrind" + +# Miscellaneous for our builds/CI (optional) +ARG MISC_DEPS="\ + automake \ + clang \ + lcov \ + python3-pip \ + sudo \ + whois" + +# Update and install required packages +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ${BASE_DEPS} \ + ${TEST_DEPS} \ + ${MISC_DEPS} \ + ${UMF_DEPS} \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get clean all + +# Prepare a dir (accessible by anyone) +RUN mkdir -p --mode 777 /opt/umf/ + +# Additional dependencies (installed via pip) +COPY third_party/requirements.txt /opt/umf/requirements.txt +RUN pip3 install --no-cache-dir --break-system-packages -r /opt/umf/requirements.txt + +# Add a new (non-root) 'test_user' +ENV USER test_user +ENV USERPASS pass +RUN useradd -m "${USER}" -g sudo -p "$(mkpasswd ${USERPASS})" +USER test_user diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 89e814856..35a7f05b6 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -36,3 +36,4 @@ Before checking all the boxes please mark the PR as draft. - [ ] All newly added source files have a license - [ ] All newly added source files are referenced in CMake files - [ ] Logger (with debug/info/... messages) is used +- [ ] All API changes are reflected in docs and def/map files, and are tested diff --git a/.github/scripts/get_system_info.sh b/.github/scripts/get_system_info.sh index 595d5e31a..573c7195d 100755 --- a/.github/scripts/get_system_info.sh +++ b/.github/scripts/get_system_info.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (C) 2024 Intel Corporation +# Copyright (C) 2024-2025 Intel Corporation # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -7,7 +7,7 @@ function check_L0_version { if command -v dpkg &> /dev/null; then - dpkg -l | grep level-zero && return + dpkg -l | grep -iE "level-zero|libze|Compute Runtime|Level Zero" && return fi if command -v rpm &> /dev/null; then @@ -15,7 +15,7 @@ function check_L0_version { fi if command -v zypper &> /dev/null; then - zypper se level-zero && return + zypper -n se level-zero || true fi echo "level-zero not installed" @@ -26,32 +26,32 @@ function system_info { cat /etc/os-release | grep -oP "PRETTY_NAME=\K.*" cat /proc/version - # echo "**********SYCL-LS**********" - # source /opt/intel/oneapi/setvars.sh - # sycl-ls + echo "**********SYCL-LS**********" + source /opt/intel/oneapi/setvars.sh + sycl-ls echo "**********numactl topology**********" numactl -H echo "**********VGA info**********" - lspci | grep -i VGA + lspci | grep -iE "vga|display|gpu" - # echo "**********CUDA Version**********" - # if command -v nvidia-smi &> /dev/null; then - # nvidia-smi - # else - # echo "CUDA not installed" - # fi + echo "**********CUDA Version**********" + if command -v nvidia-smi &> /dev/null; then + nvidia-smi + else + echo "CUDA not installed" + fi echo "**********L0 Version**********" check_L0_version - # echo "**********ROCm Version**********" - # if command -v rocminfo &> /dev/null; then - # rocminfo - # else - # echo "ROCm not installed" - # fi + echo "**********ROCm Version**********" + if command -v rocminfo &> /dev/null; then + rocminfo + else + echo "ROCm not installed" + fi echo "******OpenCL*******" # The driver version of OpenCL Graphics is the compute-runtime version @@ -67,11 +67,15 @@ function system_info { cat /proc/meminfo echo "**********env variables**********" - echo "PATH=${PATH}" - echo "CPATH=${CPATH}" - echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" - echo "LIBRARY_PATH=${LIBRARY_PATH}" - echo "PKG_CONFIG_PATH=${PKG_CONFIG_PATH}" + echo "PATH=$PATH" + echo + echo "CPATH=$CPATH" + echo + echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" + echo + echo "LIBRARY_PATH=$LIBRARY_PATH" + echo + echo "PKG_CONFIG_PATH=$PKG_CONFIG_PATH" echo echo "******build tools versions*******" diff --git a/.github/scripts/install_oneAPI.sh b/.github/scripts/install_oneAPI.sh new file mode 100755 index 000000000..f2331e2eb --- /dev/null +++ b/.github/scripts/install_oneAPI.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# Copyright (C) 2025 Intel Corporation +# Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +# install_oneAPI.sh - Script for installing Intel oneAPI from the official repository + +apt-get update +apt-get install -y gpg-agent gnupg wget +wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor -o /usr/share/keyrings/oneapi-archive-keyring.gpg +echo 'deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main' > /etc/apt/sources.list.d/oneAPI.list +apt-get update +apt-get install -y intel-oneapi-ippcp-devel intel-oneapi-ipp-devel intel-oneapi-common-oneapi-vars intel-oneapi-compiler-dpcpp-cpp diff --git a/.github/scripts/run-codespell.py b/.github/scripts/run-codespell.py new file mode 100644 index 000000000..b87bf37bd --- /dev/null +++ b/.github/scripts/run-codespell.py @@ -0,0 +1,40 @@ +""" + Copyright (C) 2024 Intel Corporation + + Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +""" + +import subprocess # nosec B404 +import logging +import sys + +logging.basicConfig( + level=logging.INFO, format="[%(levelname)s]: [%(asctime)s] %(message)s" +) + + +def codespell_scan(): + try: + codespell_result = subprocess.run( # nosec + [ + "codespell", + "-H", + "--quiet-level=3", + "--skip=./.git,./.venv,./.github/workflows/.spellcheck-conf.toml", + ], + text=True, + stdout=subprocess.PIPE, + ) + if codespell_result.returncode != 0: + for line in codespell_result.stdout.splitlines(): + logging.error(line.strip()) + sys.exit(1) + else: + logging.info("No spelling errors found") + except subprocess.CalledProcessError as ex: + logging.error(ex) + sys.exit(1) + + +codespell_scan() diff --git a/.github/workflows/.spellcheck-conf.toml b/.github/workflows/.spellcheck-conf.toml index 24b9023fe..288af6a19 100644 --- a/.github/workflows/.spellcheck-conf.toml +++ b/.github/workflows/.spellcheck-conf.toml @@ -1,6 +1,6 @@ [default] # Don't correct the following words: -extend-ignore-words-re = ["ASSER", "Tne", "ba", "BA"] +extend-ignore-words-re = ["ASSER", "Tne", "ba", "BA", "PN"] [files] # completely exclude those files from consideration: diff --git a/.github/workflows/bandit.yml b/.github/workflows/bandit.yml deleted file mode 100644 index 80b383665..000000000 --- a/.github/workflows/bandit.yml +++ /dev/null @@ -1,35 +0,0 @@ -# Bandit static analysis (for Python code) -name: Bandit -on: - push: - branches-ignore: - - 'dependabot/**' - pull_request: - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -permissions: - contents: read - -jobs: - bandit: - name: Bandit - strategy: - matrix: - os: [ubuntu-latest, windows-latest] - runs-on: ${{matrix.os}} - - steps: - - name: Checkout repository - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - fetch-depth: 0 - - - name: Install Bandit - run: python3 -m pip install bandit - - # Run Bandit recursively, but omit _deps directory (with 3rd party code) - - name: Run Bandit - run: python3 -m bandit -r . -x '/_deps/' diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index de48173bf..6f540ae39 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -1,79 +1,45 @@ -# Executes benchmarks implemented in this repository -name: Benchmarks - -on: workflow_call +name: Compute Benchmarks + +on: + workflow_dispatch: + inputs: + pr_no: + description: PR number (if 0, it'll run on the main) + type: number + bench_script_params: + # If you want to save the results of the manual run in 'benchmark-results' branch, + # you have to pass '--save XXX', where XXX is the label of your results. + description: Parameters passed to script executing benchmark + type: string + required: false + default: '' + bench_script_compare: + description: Compare label, passed to script executing benchmark as '--compare