From bddd6967e0ea256950f87edca579a8a63e425a0f Mon Sep 17 00:00:00 2001 From: Francesc Verdugo Date: Sun, 20 Aug 2023 10:39:44 +0200 Subject: [PATCH 1/7] Added partition_from_color --- src/PartitionedArrays.jl | 1 + src/p_range.jl | 41 +++++++++++++++++++++++++++++++++++--- test/p_range_tests.jl | 43 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 3 deletions(-) diff --git a/src/PartitionedArrays.jl b/src/PartitionedArrays.jl index 6db70d1e..b2029575 100644 --- a/src/PartitionedArrays.jl +++ b/src/PartitionedArrays.jl @@ -61,6 +61,7 @@ include("mpi_array.jl") export PRange export uniform_partition export variable_partition +export partition_from_color export AbstractLocalIndices export OwnAndGhostIndices export LocalIndices diff --git a/src/p_range.jl b/src/p_range.jl index 5b9616a5..2a332c86 100644 --- a/src/p_range.jl +++ b/src/p_range.jl @@ -696,6 +696,30 @@ function variable_partition( indices end +""" + partition_from_color(ranks,global_to_owner;multicast=false,source=MAIN) + +!!! warning + Document me! Looking for help. Open a PR. + +""" +function partition_from_color(ranks,global_to_color;multicast=false,source=MAIN) + if multicast == true + global_to_owner = getany(emit(global_to_color;source)) + else + global_to_owner = global_to_color + end + map(ranks) do rank + nglobal = length(global_to_owner) + own_to_global = findall(owner->owner==rank,global_to_owner) + ghost_to_global = Int[] + ghost_to_owner = Int32[] + own = OwnIndices(nglobal,rank,own_to_global) + ghost = GhostIndices(nglobal,ghost_to_global,ghost_to_owner) + OwnAndGhostIndices(own,ghost,global_to_owner) + end +end + function local_range(p,np,n,ghost=false,periodic=false) l = n รท np offset = l * (p-1) @@ -1125,17 +1149,19 @@ Local indices are defined by concatenating own and ghost ones. OwnAndGhostIndices <: AbstractLocalIndices """ -struct OwnAndGhostIndices <: AbstractLocalIndices +struct OwnAndGhostIndices{A} <: AbstractLocalIndices own::OwnIndices ghost::GhostIndices + global_to_owner::A assembly_cache::AssemblyCache @doc """ OwnAndGhostIndices(own::OwnIndices,ghost::GhostIndices) Build an instance of [`OwnAndGhostIndices`](@ref) from the underlying properties `own` and `ghost`. """ - function OwnAndGhostIndices(own::OwnIndices,ghost::GhostIndices) - new(own,ghost,AssemblyCache()) + function OwnAndGhostIndices(own::OwnIndices,ghost::GhostIndices,global_to_owner=nothing) + A = typeof(global_to_owner) + new{A}(own,ghost,global_to_owner,AssemblyCache()) end end assembly_cache(a::OwnAndGhostIndices) = a.assembly_cache @@ -1146,6 +1172,15 @@ function replace_ghost(a::OwnAndGhostIndices,ghost::GhostIndices) OwnAndGhostIndices(a.own,ghost) end +function find_owner(indices,global_ids,::Type{<:OwnAndGhostIndices{T}}) where T + if T == Nothing + error("Not enough data to perform this operation without communciation") + end + map(indices,global_ids) do indices,global_ids + indices.global_to_owner[global_ids] + end +end + part_id(a::OwnAndGhostIndices) = a.own.owner function own_to_global(a::OwnAndGhostIndices) diff --git a/test/p_range_tests.jl b/test/p_range_tests.jl index daee9f9b..696946d5 100644 --- a/test/p_range_tests.jl +++ b/test/p_range_tests.jl @@ -245,4 +245,47 @@ function p_range_tests(distribute) display(PRange(ids4)) + n = 21 + np = 4 + rank = distribute(LinearIndices((4,))) + global_to_color = zeros(Int,n) + for p in 1:np + global_to_color[local_range(p,np,n)] .= p + end + ids_color = partition_from_color(rank,global_to_color) + ids_uniform = uniform_partition(rank,n) + map(ids_color,ids_uniform) do ids1,ids2 + @test ids1 == ids2 + end + + global_to_color = map_main(rank) do rank + my_global_to_color = zeros(Int,n) + for p in 1:np + my_global_to_color[local_range(p,np,n)] .= p + end + my_global_to_color + end + ids_color = partition_from_color(rank,global_to_color;multicast=true) + ids_color = partition_from_color(rank,global_to_color;multicast=true,source=MAIN) + map(ids_color,ids_uniform) do ids1,ids2 + @test ids1 == ids2 + end + + gids = map(rank) do part + if part == 1 + [1,4,6] + elseif part == 2 + [3,1,2,8] + elseif part == 3 + [1,9,6] + else + [3,2,8,10] + end + end + owners1 = find_owner(ids_color,gids) + owners2 = find_owner(ids_uniform,gids) + map(owners1,owners2) do ids1,ids2 + @test ids1 == ids2 + end + end From d97858aef44355fc9829a9c11244f27a8288e89e Mon Sep 17 00:00:00 2001 From: Francesc Verdugo Date: Sun, 20 Aug 2023 10:51:34 +0200 Subject: [PATCH 2/7] Improvements in documentation. --- docs/src/reference/partition.md | 1 + src/p_range.jl | 9 ++++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/src/reference/partition.md b/docs/src/reference/partition.md index 99cdaab3..4f7ad290 100644 --- a/docs/src/reference/partition.md +++ b/docs/src/reference/partition.md @@ -5,6 +5,7 @@ ```@docs uniform_partition variable_partition +partition_from_color ``` ## AbstractLocalIndices diff --git a/src/p_range.jl b/src/p_range.jl index 2a332c86..f23e2a83 100644 --- a/src/p_range.jl +++ b/src/p_range.jl @@ -1143,10 +1143,13 @@ Local indices are defined by concatenating own and ghost ones. - `own::OwnIndices`: Container for the own indices. - `ghost::GhostIndices`: Container for the ghost indices. +- `global_to_owner`: [optional: it can be `nothing`] Vector containing the owner of each global id. # Supertype hierarchy - OwnAndGhostIndices <: AbstractLocalIndices + OwnAndGhostIndices{A} <: AbstractLocalIndices + +where `A=typeof(global_to_owner)`. """ struct OwnAndGhostIndices{A} <: AbstractLocalIndices @@ -1155,9 +1158,9 @@ struct OwnAndGhostIndices{A} <: AbstractLocalIndices global_to_owner::A assembly_cache::AssemblyCache @doc """ - OwnAndGhostIndices(own::OwnIndices,ghost::GhostIndices) + OwnAndGhostIndices(own::OwnIndices,ghost::GhostIndices,global_to_owner=nothing) - Build an instance of [`OwnAndGhostIndices`](@ref) from the underlying properties `own` and `ghost`. + Build an instance of [`OwnAndGhostIndices`](@ref) from the underlying properties `own`, `ghost`, and `global_to_owner`. """ function OwnAndGhostIndices(own::OwnIndices,ghost::GhostIndices,global_to_owner=nothing) A = typeof(global_to_owner) From 224027c75b5b9baaca844a4b209c91f899c30aa6 Mon Sep 17 00:00:00 2001 From: Francesc Verdugo Date: Sun, 20 Aug 2023 11:27:22 +0200 Subject: [PATCH 3/7] Fixing test --- test/p_range_tests.jl | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/test/p_range_tests.jl b/test/p_range_tests.jl index 696946d5..b3228a61 100644 --- a/test/p_range_tests.jl +++ b/test/p_range_tests.jl @@ -258,10 +258,14 @@ function p_range_tests(distribute) @test ids1 == ids2 end - global_to_color = map_main(rank) do rank - my_global_to_color = zeros(Int,n) - for p in 1:np - my_global_to_color[local_range(p,np,n)] .= p + global_to_color = map(rank) do rank + if rank == MAIN + my_global_to_color = zeros(Int,n) + for p in 1:np + my_global_to_color[local_range(p,np,n)] .= p + end + else + my_global_to_color = Int[] end my_global_to_color end From 091ea2266ce7ea25cba45ba9d9adf10633eb36f7 Mon Sep 17 00:00:00 2001 From: Francesc Verdugo Date: Sun, 20 Aug 2023 16:14:04 +0200 Subject: [PATCH 4/7] Some docs --- src/p_range.jl | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/p_range.jl b/src/p_range.jl index f23e2a83..d16840d6 100644 --- a/src/p_range.jl +++ b/src/p_range.jl @@ -697,11 +697,24 @@ function variable_partition( end """ - partition_from_color(ranks,global_to_owner;multicast=false,source=MAIN) + partition_from_color(ranks,global_to_color;multicast=false,source=MAIN) -!!! warning - Document me! Looking for help. Open a PR. +Build an arbitrary 1d partition by defining the parts via the argument `global_to_color`. +The output is a vector of vectors containing the indices in each component of +the partition. The `eltype` of the result implements the [`AbstractLocalIndices`](@ref) +interface. + +# Arguments + +- `ranks`: Array containing the distribution of ranks. +- `global_to_color`: If `multicast==false`, `global_to_color[gid]` contains the part id that own the global id `gid`. If `multicast==true`, then `global_to_color[source][gid]` contains the part id that own the global id `gid`. + +# Key-word arguments +- `multicast=false` +- `source=MAIN` +This function is useful when generating a partition using a graph partitioner such as METIS. +The argument `global_to_color` is the usual output of such tools. """ function partition_from_color(ranks,global_to_color;multicast=false,source=MAIN) if multicast == true From e081f21ffe6eb20f52c1f3337603fb58cb59f8b7 Mon Sep 17 00:00:00 2001 From: Francesc Verdugo Date: Sun, 20 Aug 2023 16:15:31 +0200 Subject: [PATCH 5/7] Minor --- src/p_range.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/p_range.jl b/src/p_range.jl index d16840d6..834bd626 100644 --- a/src/p_range.jl +++ b/src/p_range.jl @@ -707,7 +707,7 @@ interface. # Arguments - `ranks`: Array containing the distribution of ranks. -- `global_to_color`: If `multicast==false`, `global_to_color[gid]` contains the part id that own the global id `gid`. If `multicast==true`, then `global_to_color[source][gid]` contains the part id that own the global id `gid`. +- `global_to_color`: If `multicast==false`, `global_to_color[gid]` contains the part id that own the global id `gid`. If `multicast==true`, then `global_to_color[source][gid]` contains the part id that owns the global id `gid`. # Key-word arguments - `multicast=false` From bbc3e5e116bebb13a78cf7c1138c9522c4d7e5ad Mon Sep 17 00:00:00 2001 From: Francesc Verdugo Date: Mon, 21 Aug 2023 11:50:59 +0200 Subject: [PATCH 6/7] Typo --- src/p_range.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/p_range.jl b/src/p_range.jl index 834bd626..795f8351 100644 --- a/src/p_range.jl +++ b/src/p_range.jl @@ -699,7 +699,7 @@ end """ partition_from_color(ranks,global_to_color;multicast=false,source=MAIN) -Build an arbitrary 1d partition by defining the parts via the argument `global_to_color`. +Build an arbitrary 1d partition by defining the parts via the argument `global_to_color` (see below). The output is a vector of vectors containing the indices in each component of the partition. The `eltype` of the result implements the [`AbstractLocalIndices`](@ref) interface. @@ -707,7 +707,7 @@ interface. # Arguments - `ranks`: Array containing the distribution of ranks. -- `global_to_color`: If `multicast==false`, `global_to_color[gid]` contains the part id that own the global id `gid`. If `multicast==true`, then `global_to_color[source][gid]` contains the part id that owns the global id `gid`. +- `global_to_color`: If `multicast==false`, `global_to_color[gid]` contains the part id that owns the global id `gid`. If `multicast==true`, then `global_to_color[source][gid]` contains the part id that owns the global id `gid`. # Key-word arguments - `multicast=false` @@ -725,7 +725,7 @@ function partition_from_color(ranks,global_to_color;multicast=false,source=MAIN) map(ranks) do rank nglobal = length(global_to_owner) own_to_global = findall(owner->owner==rank,global_to_owner) - ghost_to_global = Int[] + ghost_to_global = Int[] ghost_to_owner = Int32[] own = OwnIndices(nglobal,rank,own_to_global) ghost = GhostIndices(nglobal,ghost_to_global,ghost_to_owner) From f751eb3b0fc100b992e604a10389acd77c45ed7d Mon Sep 17 00:00:00 2001 From: Francesc Verdugo Date: Mon, 21 Aug 2023 11:51:20 +0200 Subject: [PATCH 7/7] Update Changelog --- CHANGELOG.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df0e631a..7f59d084 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,12 +5,18 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Added + +- Function `partition_from_color`. + ## [0.3.3] - 2023-08-09 ### Added -- Added an MPI ibarrier-based (supposedly scalable) algorithm to find rcv neighbours in a sparse all-to-all communication graph given the snd neighbors. We left the previous non-scalable algorithm as default (based on gather-scatter) until we have experimental evidence on the relative performance and scalability of the former with respect to the latter and for which core ranges. -- Added a new kwarg `discover_cols=true` to the `psparse!` constructor, which allows the user to skip column index discovery. +- MPI ibarrier-based (supposedly scalable) algorithm to find rcv neighbours in a sparse all-to-all communication graph given the snd neighbors. We left the previous non-scalable algorithm as default (based on gather-scatter) until we have experimental evidence on the relative performance and scalability of the former with respect to the latter and for which core ranges. +- New kwarg `discover_cols=true` to the `psparse!` constructor, which allows the user to skip column index discovery. ### Fixed