8000 Fix GPU tests for size>=2 by leofang · Pull Request #60 · mpi4py/mpi4py · GitHub
[go: up one dir, main page]
Skip to content

Fix GPU tests for size>=2 #60

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jul 19, 2021
Merged

Fix GPU tests for size>=2 #60

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
41607b7
Fix GPU test array classes
leofang Jul 19, 2021
7613d24
skip GPU segfaults
leofang Jul 19, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions test/arrayimpl.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -347,6 +347,10 @@ def flat(self):
def size(self):
return self.array.size

def as_raw(self):
cupy.cuda.get_current_stream().synchronize()
return self.array


if numba is not None:

Expand Down Expand Up @@ -413,6 +417,11 @@ def flat(self):
def size(self):
return self.array.size

def as_raw(self):
# numba by default always runs on the legacy default stream
numba.cuda.default_stream().synchronize()
return self.array


def subTest(case, skip=(), skiptypecode=()):
for array in ArrayBackends:
Expand Down
4 changes: 3 additions & 1 deletion test/test_cco_buf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -453,6 +453,8 @@ def testReduceScatterBlock(self):
size = self.COMM.Get_size()
rank = self.COMM.Get_rank()
for array, typecode in arrayimpl.subTest(self):
# one of the ranks would fail as of OpenMPI 4.1.1
if unittest.is_mpi_gpu('openmpi', array): continue
for op in (MPI.SUM, MPI.MAX, MPI.MIN, MPI.PROD):
if skip_op(typecode, op): continue
for rcnt in range(size):
Expand Down Expand Up @@ -581,6 +583,7 @@ class TestReduceLocal(unittest.TestCase):

def testReduceLocal(self):
for array, typecode in arrayimpl.subTest(self):
# segfault as of OpenMPI 4.1.1
if unittest.is_mpi_gpu('openmpi', array): continue
for op in (MPI.SUM, MPI.PROD, MPI.MAX, MPI.MIN):
if skip_op(typecode, op): continue
Expand All @@ -604,7 +607,6 @@ def testReduceLocal(self):

def testReduceLocalBadCount(self):
for array, typecode in arrayimpl.subTest(self):
if unittest.is_mpi_gpu('openmpi', array): continue
for op in (MPI.SUM, MPI.PROD, MPI.MAX, MPI.MIN):
sbuf = array(range(3), typecode)
rbuf = array(range(3), typecode)
Expand Down
24 changes: 24 additions & 0 deletions test/test_cco_nb_buf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,8 @@ def testReduce(self):
size = self.COMM.Get_size()
rank = self.COMM.Get_rank()
for array, typecode in arrayimpl.subTest(self):
# segfault as of OpenMPI 4.1.1
if unittest.is_mpi_gpu('openmpi', array): continue
for op in (MPI.SUM, MPI.PROD, MPI.MAX, MPI.MIN):
if skip_op(typecode, op): continue
for root in range(size):
Expand Down Expand Up @@ -136,6 +138,8 @@ def testAllreduce(self):
size = self.COMM.Get_size()
rank = self.COMM.Get_rank()
for array, typecode in arrayimpl.subTest(self):
# segfault as of OpenMPI 4.1.1
if unittest.is_mpi_gpu('openmpi', array): continue
for op in (MPI.SUM, MPI.MAX, MPI.MIN, MPI.PROD):
if skip_op(typecode, op): continue
sbuf = array(range(size), typecode)
Expand All @@ -161,6 +165,8 @@ def testReduceScatter(self):
size = self.COMM.Get_size()
rank = self.COMM.Get_rank()
for array, typecode in arrayimpl.subTest(self):
# segfault as of OpenMPI 4.1.1
if unittest.is_mpi_gpu('openmpi', array): continue
for op in (MPI.SUM, MPI.MAX, MPI.MIN, MPI.PROD):
if skip_op(typecode, op): continue
rcnt = list(range(1,size+1))
Expand Down Expand Up @@ -206,6 +212,8 @@ def testReduceScatterBlock(self):
size = self.COMM.Get_size()
rank = self.COMM.Get_rank()
for array, typecode in arrayimpl.subTest(self):
# segfault as of OpenMPI 4.1.1
if unittest.is_mpi_gpu('openmpi', array): continue
for op in (MPI.SUM, MPI.MAX, MPI.MIN, MPI.PROD):
if skip_op(typecode, op): continue
for rcnt in range(1, size+1):
Expand Down Expand Up @@ -239,6 +247,8 @@ def testScan(self):
rank = self.COMM.Get_rank()
# --
for array, typecode in arrayimpl.subTest(self):
# segfault as of OpenMPI 4.1.1
if unittest.is_mpi_gpu('openmpi', array): continue
for op in (MPI.SUM, MPI.PROD, MPI.MAX, MPI.MIN):
if skip_op(typecode, op): continue
sbuf = array(range(size), typecode)
Expand All @@ -264,6 +274,8 @@ def testExscan(self):
size = self.COMM.Get_size()
rank = self.COMM.Get_rank()
for array, typecode in arrayimpl.subTest(self):
# segfault as of OpenMPI 4.1.1
if unittest.is_mpi_gpu('openmpi', array): continue
for op in (MPI.SUM, MPI.PROD, MPI.MAX, MPI.MIN):
if skip_op(typecode, op): continue
sbuf = array(range(size), typecode)
Expand Down Expand Up @@ -400,6 +412,8 @@ def testReduce(self):
size = self.COMM.Get_size()
rank = self.COMM.Get_rank()
for array, typecode in arrayimpl.subTest(self):
# segfault as of OpenMPI 4.1.1
if unittest.is_mpi_gpu('openmpi', array): continue
for op in (MPI.SUM, MPI.PROD, MPI.MAX, MPI.MIN):
if skip_op(typecode, op): continue
for root in range(size):
Expand Down Expand Up @@ -432,6 +446,8 @@ def testAllreduce(self):
size = self.COMM.Get_size()
rank = self.COMM.Get_rank()
for array, typecode in arrayimpl.subTest(self):
# segfault as of OpenMPI 4.1.1
if unittest.is_mpi_gpu('openmpi', array): continue
for op in (MPI.SUM, MPI.MAX, MPI.MIN, MPI.PROD):
if skip_op(typecode, op): continue
buf = array(range(size), typecode)
Expand All @@ -456,6 +472,8 @@ def testReduceScatterBlock(self):
size = self.COMM.Get_size()
rank = self.COMM.Get_rank()
for array, typecode in arrayimpl.subTest(self):
# segfault as of OpenMPI 4.1.1
if unittest.is_mpi_gpu('openmpi', array): continue
for op in (MPI.SUM, MPI.MAX, MPI.MIN, MPI.PROD):
if skip_op(typecode, op): continue
for rcnt in range(size):
Expand Down Expand Up @@ -492,6 +510,8 @@ def testReduceScatter(self):
size = self.COMM.Get_size()
rank = self.COMM.Get_rank()
for array, typecode in arrayimpl.subTest(self):
# segfault as of OpenMPI 4.1.1
if unittest.is_mpi_gpu('openmpi', array): continue
for op in (MPI.SUM, MPI.MAX, MPI.MIN, MPI.PROD):
if skip_op(typecode, op): continue
rcnt = list(range(1, size+1))
Expand Down Expand Up @@ -529,6 +549,8 @@ def testScan(self):
rank = self.COMM.Get_rank()
# --
for array, typecode in arrayimpl.subTest(self):
# segfault as of OpenMPI 4.1.1
if unittest.is_mpi_gpu('openmpi', array): continue
for op in (MPI.SUM, MPI.PROD, MPI.MAX, MPI.MIN):
if skip_op(typecode, op): continue
buf = array(range(size), typecode)
Expand All @@ -553,6 +575,8 @@ def testExscan(self):
size = self.COMM.Get_size()
rank = self.COMM.Get_rank()
for array, typecode in arrayimpl.subTest(self):
# segfault as of OpenMPI 4.1.1
if unittest.is_mpi_gpu('openmpi', array): continue
for op in (MPI.SUM, MPI.PROD, MPI.MAX, MPI.MIN):
if skip_op(typecode, op): continue
buf = array(range(size), typecode)
Expand Down
4 changes: 4 additions & 0 deletions test/test_cco_ngh_buf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,10 @@ def testNeighborAllgather(self):
for comm in create_topo_comms(self.COMM):
rsize, ssize = get_neighbors_count(comm)
for array, typecode in arrayimpl.subTest(self):
if unittest.is_mpi_gpu('openmpi', array):
# segfault as of OpenMPI 4.1.1; TODO(leofang): why?
if array.backend == 'numba':
continue
for v in range(3):
sbuf = array( v, typecode, 3)
rbuf = array(-1, typecode, (rsize, 3))
Expand Down
2 changes: 2 additions & 0 deletions test/test_p2p_buf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ def testSendRecv(self):
size = self.COMM.Get_size()
rank = self.COMM.Get_rank()
for array, typecode in arrayimpl.subTest(self):
if unittest.is_mpi_gpu('openmpi', array): continue
if unittest.is_mpi_gpu('mvapich2', array): continue
for s in range(0, size):
#
Expand Down Expand Up @@ -147,6 +148,7 @@ def testPersistent(self):
dest = (rank + 1) % size
source = (rank - 1) % size
for array, typecode in arrayimpl.subTest(self):
if unittest.is_mpi_gpu('openmpi', array): continue
if unittest.is_mpi_gpu('mvapich2', array): continue
for s in range(size):
for xs in range(3):
Expand Down
11 changes: 11 additions & 0 deletions test/test_spawn.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,16 @@ def badport():
port = ""
return port == ""

def using_GPU():
# Once a CUDA context is created, the process cannot be forked.
# Note: This seems to be a partial fix. Even if we are running cpu-only
# tests, if MPI is built with CUDA support we can still fail. Unfortunately
# there is no runtime check for us to detect if it's the case...
disabled_cupy = (sys.modules.get('cupy', -1) is None)
disabled_numba = (sys.modules.get('numba', -1) is None)
return False if (disabled_cupy and disabled_numba) else True


@unittest.skipMPI('MPI(<2.0)')
@unittest.skipMPI('openmpi(<3.0.0)')
@unittest.skipMPI('openmpi(==4.0.0)')
Expand All @@ -69,6 +79,7 @@ def badport():
@unittest.skipMPI('MPICH2')
@unittest.skipMPI('MPICH1')
@unittest.skipMPI('PlatformMPI')
@unittest.skipIf(using_GPU(), 'using CUDA')
class BaseTestSpawn(object):

COMM = MPI.COMM_NULL
Expand Down
0