10000 [WIP] Docker in the nightly testing by rbanka1 · Pull Request #1033 · oneapi-src/unified-memory-framework · GitHub
[go: up one dir, main page]

Skip to content
10000

[WIP] Docker in the nightly testing #1033

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 29 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
143f4a2
Docker in the nightly testing
rbanka1 Jan 13, 2025
46b60fb
Try to fix bad substitution error
rbanka1 Jan 14, 2025
726fd23
Merge branch 'main' into addDocker
rbanka1 Jan 14, 2025
81abd42
Try to fix bad substitution error V2
rbanka1 Jan 14, 2025
0d602d9
Try to fix bad substitution error V2.2
rbanka1 Jan 14, 2025
bfb7e5e
Merge branch 'oneapi-src:main' into addDocker
rbanka1 Jan 14, 2025
ede9ec5
install hwloc
rbanka1 Jan 15, 2025
b565cd3
,
rbanka1 Jan 15, 2025
69bde86
Merge branch 'main' into addDocker
rbanka1 Jan 15, 2025
888c185
working docker
rbanka1 Jan 15, 2025
b610766
add valgrind
rbanka1 Jan 15, 2025
8d27aea
..
rbanka1 Jan 16, 2025
3207e4c
Add FindVALGRIND.cmake
rbanka1 Jan 16, 2025
6f9ed9b
build-docker-BasicBuilds
rbanka1 Jan 23, 2025
c7053bf
build-docker-BasicBuilds
rbanka1 Jan 23, 2025
ae630ec
build-docker-BasicBuilds V2
rbanka1 Jan 23, 2025
a15b792
build-docker-BasicBuilds V2.2
rbanka1 Jan 23, 2025
10ab5cf
dockerHub test
rbanka1 Jan 28, 2025
cd042a6
dockerHub test
rbanka1 Jan 28, 2025
84f5db8
dockerHub test
rbanka1 Jan 28, 2025
5d68917
dockerHub test
rbanka1 Jan 28, 2025
233a67d
dockerHub test
rbanka1 Jan 28, 2025
1dc7cca
dockerHub test L&Win
rbanka1 Jan 28, 2025
8fd1ab1
Merge branch 'main' into addDocker
rbanka1 Jan 28, 2025
6fc641a
dockerHub test L&Win
rbanka1 Jan 28, 2025
e298464
dockerHub test L&Win
rbanka1 Jan 28, 2025
eef0189
dockerHub test L&Win
rbanka1 Jan 28, 2025
6246020
dockerHub test L&Win
rbanka1 Jan 28, 2025
75afc7b
dockerHub test L&Win
rbanka1 Jan 28, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
10000
Diff view
17 changes: 15 additions & 2 deletions .github/docker/ubuntu-20.04.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

#
# Dockerfile - a 'recipe' for Docker to build an image of ubuntu-based
# environment for building the Unified Memory Framework project.
# environment for building the Unified Memory Framework project.
#

# Pull base image ("20.04")
Expand Down Expand Up @@ -50,8 +50,21 @@ RUN apt-get update \
&& rm -rf /var/lib/apt/lists/* \
&& apt-get clean all

# Install hwloc
COPY .github/scripts/install_hwloc.sh /opt/umf/install_hwloc.sh
RUN apt-get update \
&& apt-get install -y dos2unix libtool \
&& dos2unix /opt/umf/install_hwloc.sh \
&& bash -x /opt/umf/install_hwloc.sh \
&& ldconfig \
&& rm -f /opt/umf/install_hwloc.sh

# Install valgrind
RUN apt-get update && \
apt-get install -y valgrind cmake hwloc libhwloc-dev libnuma-dev libtbb-dev

# Prepare a dir (accessible by anyone)
RUN mkdir --mode 777 /opt/umf/
RUN mkdir -p --mode 777 /opt/umf/

# Additional dependencies (installed via pip)
COPY third_party/requirements.txt /opt/umf/requirements.txt
Expand Down
14 changes: 9 additions & 5 deletions .github/workflows/nightly.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@
name: Nightly

# This job is run at 00:00 UTC every day or on demand.
on:
workflow_dispatch:
schedule:
- cron: '0 0 * * *'
on: workflow_call
# workflow_dispatch:
# schedule:
# - cron: '0 0 * * *'

permissions:
contents: read
Expand Down Expand Up @@ -50,6 +50,10 @@ jobs:
working-directory: ${{github.workspace}}/build
run: ctest -C ${{matrix.build_type}} --output-on-failure --verbose -L "fuzz-long"

FastBuild:
name: Fast builds
uses: ./.github/workflows/reusable_fast.yml

valgrind:
name: Valgrind
strategy:
Expand Down Expand Up @@ -193,4 +197,4 @@ jobs:
short_run: false
# Beside the 2 LTS Ubuntu, we also test this on the latest Ubuntu - to be updated
# every 6 months, so we verify the latest version of packages (compilers, etc.).
os: "['ubuntu-22.04', 'ubuntu-24.04', 'ubuntu-24.10']"
os: "['ubuntu-22.04', 'ubuntu-24.04', 'ubuntu-24.10']"
2 changes: 2 additions & 0 deletions .github/workflows/pr_push.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ jobs:
uses: ./.github/workflows/reusable_checks.yml
DocsBuild:
uses: ./.github/workflows/reusable_docs_build.yml
Nightly:
uses: ./.github/workflows/nightly.yml
FastBuild:
name: Fast builds
needs: [CodeChecks, DocsBuild]
Expand Down
129 changes: 93 additions & 36 deletions .github/workflows/reusable_fast.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
# Fast builds
name: FastBuild

on: workflow_call
Expand All @@ -11,72 +10,48 @@ env:
INSTL_DIR : "${{github.workspace}}/../install-dir"

jobs:
FastBuild:
name: Fast builds
FastBuild_Linux:
name: Fast builds (Linux)
runs-on: ubuntu-latest
container:
image: rbanka974/umf-ubuntu-20.04:latest
options: --user root --privileged
volumes:
- /home/runner/work/unified-memory-framework/unified-memory-framework:/home/runner/work/unified-memory-framework/unified-memory-framework

env:
VCPKG_PATH: "${{github.workspace}}/build/vcpkg/packages/hwloc_x64-windows;${{github.workspace}}/build/vcpkg/packages/tbb_x64-windows;${{github.workspace}}/build/vcpkg/packages/jemalloc_x64-windows"
strategy:
matrix:
include:
- os: windows-latest
disjoint: 'OFF'
build_tests: 'ON'
simple_cmake: 'OFF'
# pure C build (Windows)
- os: windows-latest
disjoint: 'OFF'
# Tests' building is off for a pure C build
build_tests: 'OFF'
simple_cmake: 'OFF'
- os: ubuntu-latest
disjoint: 'ON'
build_tests: 'ON'
# Windows doesn't recognize 'CMAKE_BUILD_TYPE', it uses '--config' param in build command
extra_build_options: '-DCMAKE_BUILD_TYPE=Release -DUMF_BUILD_BENCHMARKS=ON -DUMF_BUILD_BENCHMARKS_MT=ON'
simple_cmake: 'OFF'
# pure C build (Linux)
- os: ubuntu-latest
disjoint: 'OFF'
# Windows doesn't recognize 'CMAKE_BUILD_TYPE', it uses '--config' param in build command
# Tests' building is off for a pure C build
build_tests: 'OFF'
extra_build_options: '-DCMAKE_BUILD_TYPE=Release -DUMF_BUILD_BENCHMARKS=ON'
simple_cmake: 'OFF'
# simplest CMake on ubuntu-latest
- os: ubuntu-latest
disjoint: 'OFF'
build_tests: 'ON'
extra_build_options: '-DCMAKE_BUILD_TYPE=Release'
simple_cmake: 'ON'
# simplest CMake ubuntu-20.04
- os: ubuntu-20.04
disjoint: 'OFF'
build_tests: 'ON'
extra_build_options: '-DCMAKE_BUILD_TYPE=Release'
simple_cmake: 'ON'
runs-on: ${{ (matrix.os == 'ubuntu-latest' && github.repository_owner == 'oneapi-src') && 'intel-ubuntu-22.04' || matrix.os }}

steps:
- name: Checkout repository
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
fetch-depth: 0

- name: Initialize vcpkg
if: matrix.os == 'windows-latest'
uses: lukka/run-vcpkg@5e0cab206a5ea620130caf672fce3e4a6b5666a1 # v11.5
with:
vcpkgGitCommitId: 3dd44b931481d7a8e9ba412621fa810232b66289
vcpkgDirectory: ${{env.BUILD_DIR}}/vcpkg
vcpkgJsonGlob: '**/vcpkg.json'

- name: Install dependencies (windows-latest)
if: matrix.os == 'windows-latest'
run: vcpkg install
shell: pwsh # Specifies PowerShell as the shell for running the script.

- name: Install dependencies (ubuntu-latest)
if: matrix.os == 'ubuntu-latest'
run: |
sudo apt-get update
sudo apt-get install -y cmake libhwloc-dev libnuma-dev libtbb-dev
Expand All @@ -88,6 +63,90 @@ jobs:
sudo apt-get install -y cmake libnuma-dev libtbb-dev
.github/scripts/install_hwloc.sh # install hwloc-2.3.0 instead of hwloc-2.1.0 present in the OS package

- name: Set ptrace value for IPC test (on Linux only)
run: sudo bash -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope"

- name: Configure CMake
if: matrix.simple_cmake == 'OFF'
run: >
cmake
-B ${{env.BUILD_DIR}}
-DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}"
-DCMAKE_PREFIX_PATH="${{env.VCPKG_PATH}}"
-DUMF_FORMAT_CODE_STYLE=OFF
-DUMF_DEVELOPER_MODE=ON
-DUMF_BUILD_LIBUMF_POOL_DISJOINT=${{matrix.disjoint}}
-DUMF_BUILD_LIBUMF_POOL_JEMALLOC=ON
-DUMF_BUILD_TESTS=${{matrix.build_tests}}
-DUMF_BUILD_EXAMPLES=ON
-DUMF_BUILD_LEVEL_ZERO_PROVIDER=ON
-DUMF_BUILD_CUDA_PROVIDER=ON
-DUMF_TESTS_FAIL_ON_SKIP=ON
-DUMF_BUILD_SHARED_LIBRARY=ON
${{matrix.extra_build_options}}

- name: Configure CMake (simple)
if: matrix.simple_cmake == 'ON'
run: >
cmake
-B ${{env.BUILD_DIR}}
-DCMAKE_INSTALL_PREFIX="${{env.INSTL_DIR}}"
-DUMF_BUILD_SHARED_LIBRARY=ON
-DUMF_TESTS_FAIL_ON_SKIP=ON
${{matrix.extra_build_options}}

- name: Build
run: cmake --build ${{env.BUILD_DIR}} --config Release -j

- name: Create build directory
run: mkdir -p ${{env.BUILD_DIR}}

- name: Run examples
working-directory: ${{env.BUILD_DIR}}
run: ctest --output-on-failure --test-dir examples -C Release

- name: Run tests
if: matrix.build_tests == 'ON'
working-directory: ${{env.BUILD_DIR}}
run: ctest --output-on-failure --test-dir test -C Release

FastBuild_Windows:
name: Fast builds (Windows)
runs-on: windows-latest

env:
VCPKG_PATH: "${{github.workspace}}/build/vcpkg/packages/hwloc_x64-windows;${{github.workspace}}/build/vcpkg/packages/tbb_x64-windows;${{github.workspace}}/build/vcpkg/packages/jemalloc_x64-windows"
strategy:
matrix:
include:
- os: windows-latest
disjoint: 'OFF'
build_tests: 'ON'
simple_cmake: 'OFF'
- os: windows-latest
disjoint: 'OFF'
build_tests: 'OFF'
simple_cmake: 'OFF'

steps:
- name: Checkout repository
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
fetch-depth: 0

- name: Initialize vcpkg
if: matrix.os == 'windows-latest'
uses: lukka/run-vcpkg@5e0cab206a5ea620130caf672fce3e4a6b5666a1 # v11.5
with:
vcpkgGitCommitId: 3dd44b931481d7a8e9ba412621fa810232b66289
vcpkgDirectory: ${{env.BUILD_DIR}}/vcpkg
vcpkgJsonGlob: '**/vcpkg.json'

- name: Install dependencies (windows-latest)
if: matrix.os == 'windows-latest'
run: vcpkg install
shell: pwsh

- name: Configure CMake
if: matrix.simple_cmake == 'OFF'
run: >
Expand Down Expand Up @@ -129,8 +188,6 @@ jobs:
working-directory: ${{env.BUILD_DIR}}
run: ctest --output-on-failure --test-dir test -C Release

# TODO: We could add some script to verify metadata of dll's (selected fields, perhaps)
# ref. https://superuser.com/questions/381276/what-are-some-nice-command-line-ways-to-inspect-dll-exe-details
- name: Print metadata of our dll's
if: matrix.os == 'windows-latest'
run: |
Expand Down
22 changes: 22 additions & 0 deletions cmake/FindVALGRIND.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# Copyright (C) 2025 Intel Corporation
# Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

if(NOT VALGRIND_ROOT AND DEFINED ENV{VALGRIND_ROOT})
set(VALGRIND_ROOT "$ENV{VALGRIND_ROOT}" CACHE PATH "Valgrind base directory location (optional, used for nonstandard installation paths)")
mark_as_advanced(VALGRIND_ROOT)
endif()

# Search path for nonstandard locations
if(VALGRIND_ROOT)
set(Valgrind_INCLUDE_PATH PATHS "${VALGRIND_ROOT}/include" "${VALGRIND_ROOT}/valgrind/current/usr/include" NO_DEFAULT_PATH)
set(Valgrind_BINARY_PATH PATHS "${VALGRIND_ROOT}/bin" NO_DEFAULT_PATH)
endif()

find_path(Valgrind_INCLUDE_DIR valgrind/memcheck.h HINTS ${Valgrind_INCLUDE_PATH})
find_program(Valgrind_EXECUTABLE NAMES valgrind PATH ${Valgrind_BINARY_PATH})

include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(Valgrind DEFAULT_MSG Valgrind_INCLUDE_DIR Valgrind_EXECUTABLE)

mark_as_advanced(Valgrind_INCLUDE_DIR Valgrind_EXECUTABLE)
16 changes: 8 additions & 8 deletions third_party/requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,15 @@ black==24.3.0
# Tests
packaging==24.2
# Generating HTML documentation
pygments==2.19.1
sphinxcontrib_applehelp==2.0.0
sphinxcontrib_devhelp==2.0.0
sphinxcontrib_htmlhelp==2.1.0
sphinxcontrib_serializinghtml==2.0.0
sphinxcontrib_qthelp==2.0.0
pygments==2.18.0
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we should rather not downgrade packages, rather move them up

sphinxcontrib_applehelp==1.0.0
sphinxcontrib_devhelp==1.0.0
sphinxcontrib_htmlhelp==2.0.1
sphinxcontrib_serializinghtml==1.1.5
sphinxcontrib_qthelp==1.0.3
breathe==4.35.0
sphinx==8.1.3
sphinx_book_theme==1.1.3
sphinx==7.1.2
sphinx_book_theme==1.0.1
# Spelling check in documentation
pyenchant==3.2.2
sphinxcontrib-spelling==8.0.0
Loading
0