From be039cb407437a3695ad50475d6a77c91ff0615c Mon Sep 17 00:00:00 2001 From: JakeNewmanUEA <85290121+JakeNewmanUEA@users.noreply.github.com> Date: Wed, 31 Aug 2022 10:06:02 +0100 Subject: [PATCH 1/5] Update collectionsoa.py Added some stuff --- parcels/collection/collectionsoa.py | 1 + 1 file changed, 1 insertion(+) diff --git a/parcels/collection/collectionsoa.py b/parcels/collection/collectionsoa.py index f739acad8..fc7a871aa 100644 --- a/parcels/collection/collectionsoa.py +++ b/parcels/collection/collectionsoa.py @@ -104,6 +104,7 @@ def __init__(self, pclass, lon, lat, depth, time, lonlatdepth_dtype, pid_orig, p if mpi_rank == 0: coords = np.vstack((lon, lat)).transpose() kmeans = KMeans(n_clusters=mpi_size, random_state=0).fit(coords) + # New comment self._pu_indicators = kmeans.labels_ else: self._pu_indicators = None From 8b21bc94f99ae6b785fe26862d015c9e7815bcb6 Mon Sep 17 00:00:00 2001 From: JakeNewmanUEA <85290121+JakeNewmanUEA@users.noreply.github.com> Date: Wed, 31 Aug 2022 14:53:44 +0100 Subject: [PATCH 2/5] Equal distribution of particles to MPI processors When running simulations in MPI mode involving the repeated release of small quantities of particles (e.g. 100 particles across 10 MPI processors), we would occasionally receive the error 'Cannot initialise with fewer particles than MPI processors'. The cause of this appeared to be the way that K-means was distributing the particles among the MPI processors. Specifically, K-means does not return clusters of a fixed or minimum size, thus it is highly possible that some MPI processors will be allocated fewer than the minimum number of required particles, especially if the number of particles is small and the number of MPI processors approaches the maximum for a given number of particles (e.g. 10 MPI processors is the maximum for 100 particles, as 100 particles / 10 processors = 10 particles per processor). To overcome this, we equally distribute the available particles among the MPI processors using Numpy's linspace method. We create a linearly spaced array that is the same length as the number particles. The array contains decimals ranging from 0 to the number of MPI processors, and that array is then rounded down so that the values represent integer indices to MPI processors. As before, this change requires that the user has requested at least the minimum required number of MPI processors, but it ensures that the minimum number is always sufficient. --- parcels/collection/collectionsoa.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/parcels/collection/collectionsoa.py b/parcels/collection/collectionsoa.py index fc7a871aa..1d16c2a6b 100644 --- a/parcels/collection/collectionsoa.py +++ b/parcels/collection/collectionsoa.py @@ -102,10 +102,10 @@ def __init__(self, pclass, lon, lat, depth, time, lonlatdepth_dtype, pid_orig, p if partitions is not False: if (self._pu_indicators is None) or (len(self._pu_indicators) != len(lon)): if mpi_rank == 0: - coords = np.vstack((lon, lat)).transpose() - kmeans = KMeans(n_clusters=mpi_size, random_state=0).fit(coords) - # New comment - self._pu_indicators = kmeans.labels_ + # distribute particles equally among MPI processors + labels = np.linspace(0, mpi_size, lon.size, endpoint=False) + labels = np.floor(labels) + self._pu_indicators = labels else: self._pu_indicators = None self._pu_indicators = mpi_comm.bcast(self._pu_indicators, root=0) From e3b100d81666ae0c4b086b273c65faeec83ef4fc Mon Sep 17 00:00:00 2001 From: JakeNewmanUEA <85290121+JakeNewmanUEA@users.noreply.github.com> Date: Wed, 31 Aug 2022 15:02:00 +0100 Subject: [PATCH 3/5] Equal distribution of particles to MPI processors Related to pull request #1231. This change has only been tested for collectionsoa.py but not collectionaos.py. When running simulations in MPI mode involving the repeated release of small quantities of particles (e.g. 100 particles across 10 MPI processors), we would occasionally receive the error 'Cannot initialise with fewer particles than MPI processors'. The cause of this appeared to be the way that K-means was distributing the particles among the MPI processors. Specifically, K-means does not return clusters of a fixed or minimum size, thus it is highly possible that some MPI processors will be allocated fewer than the minimum number of required particles, especially if the number of particles is small and the number of MPI processors approaches the maximum for a given number of particles (e.g. 10 MPI processors is the maximum for 100 particles, as 100 particles / 10 processors = 10 particles per processor). To overcome this, we equally distribute the available particles among the MPI processors using Numpy's linspace method. We create a linearly spaced array that is the same length as the number particles. The array contains decimals ranging from 0 to the number of MPI processors, and that array is then rounded down so that the values represent integer indices to MPI processors. As before, this change requires that the user has requested at least the minimum required number of MPI processors, but it ensures that the minimum number is always sufficient. --- parcels/collection/collectionaos.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/parcels/collection/collectionaos.py b/parcels/collection/collectionaos.py index adddb8fe7..7161489c2 100644 --- a/parcels/collection/collectionaos.py +++ b/parcels/collection/collectionaos.py @@ -101,9 +101,10 @@ def __init__(self, pclass, lon, lat, depth, time, lonlatdepth_dtype, pid_orig, p if partitions is not False: if self._pu_indicators is None: if mpi_rank == 0: - coords = np.vstack((lon, lat)).transpose() - kmeans = KMeans(n_clusters=mpi_size, random_state=0).fit(coords) - self._pu_indicators = kmeans.labels_ + # distribute particles equally among MPI processors + labels = np.linspace(0, mpi_size, lon.size, endpoint=False) + labels = np.floor(labels) + self._pu_indicators = labels else: self._pu_indicators = None self._pu_indicators = mpi_comm.bcast(self._pu_indicators, root=0) From b44e59cf0b3a2c3561988dc74cd5c00d44e2af16 Mon Sep 17 00:00:00 2001 From: JakeNewmanUEA <85290121+JakeNewmanUEA@users.noreply.github.com> Date: Thu, 1 Sep 2022 12:26:28 +0100 Subject: [PATCH 4/5] Equal distribution of particles to MPI processors Following pull request #1231: Reinstated K-Means as the method of grouping particle locations. Rather than using the clusters assignments directly, we find the X nearest coordinates to each cluster centroid, where X is the minimum number of particles required to distribute the particles equally among MPI processors. The result is an (approximately) equal number of particles assigned to each MPI processor, with particles being grouped according to their spatial distribution. --- parcels/collection/collectionsoa.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/parcels/collection/collectionsoa.py b/parcels/collection/collectionsoa.py index 1d16c2a6b..ba84cd08b 100644 --- a/parcels/collection/collectionsoa.py +++ b/parcels/collection/collectionsoa.py @@ -3,7 +3,7 @@ from ctypes import Structure, POINTER from bisect import bisect_left from math import floor - +import copy import numpy as np from parcels.collection.collections import ParticleCollection @@ -101,11 +101,20 @@ def __init__(self, pclass, lon, lat, depth, time, lonlatdepth_dtype, pid_orig, p if mpi_size > 1: if partitions is not False: if (self._pu_indicators is None) or (len(self._pu_indicators) != len(lon)): - if mpi_rank == 0: - # distribute particles equally among MPI processors + if mpi_rank == 0: + coords = np.vstack((lon, lat)).transpose() + kmeans = KMeans(n_clusters=mpi_size, random_state=0).fit(coords) labels = np.linspace(0, mpi_size, lon.size, endpoint=False) labels = np.floor(labels) - self._pu_indicators = labels + reorderedCoords = copy.copy(labels) + for i in range(0, len(kmeans.cluster_centers_)): + clusterCentre = kmeans.cluster_centers_[i] + distances = map(lambda point: ((point[0]-clusterCentre[0])**2 + (point[1]-clusterCentre[1])**2), coords) + sortedDistanceIdxs = np.argsort(list(distances)) + numberToChoose = sum(labels == i) + reorderedCoords[sortedDistanceIdxs[0:numberToChoose]] = i + coords[sortedDistanceIdxs[0:numberToChoose],:] = float('inf') + self._pu_indicators = reorderedCoords else: self._pu_indicators = None self._pu_indicators = mpi_comm.bcast(self._pu_indicators, root=0) From 11fdf50f1fa16f5c909ce64716456d4f0cf85707 Mon Sep 17 00:00:00 2001 From: JakeNewmanUEA <85290121+JakeNewmanUEA@users.noreply.github.com> Date: Thu, 1 Sep 2022 12:39:16 +0100 Subject: [PATCH 5/5] Equal distribution of particles to MPI processors Following pull request #1231: Reinstated K-Means as the method of grouping particle locations. Rather than using the clusters assignments directly, we find the X nearest coordinates to each cluster centroid, where X is the minimum number of particles required to distribute the particles equally among MPI processors. The result is an (approximately) equal number of particles assigned to each MPI processor, with particles being grouped according to their spatial distribution. --- parcels/collection/collectionaos.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/parcels/collection/collectionaos.py b/parcels/collection/collectionaos.py index 7161489c2..7917906d0 100644 --- a/parcels/collection/collectionaos.py +++ b/parcels/collection/collectionaos.py @@ -3,6 +3,7 @@ from ctypes import c_void_p +import copy import numpy as np from parcels.collection.collections import ParticleCollection @@ -101,10 +102,19 @@ def __init__(self, pclass, lon, lat, depth, time, lonlatdepth_dtype, pid_orig, p if partitions is not False: if self._pu_indicators is None: if mpi_rank == 0: - # distribute particles equally among MPI processors + coords = np.vstack((lon, lat)).transpose() + kmeans = KMeans(n_clusters=mpi_size, random_state=0).fit(coords) labels = np.linspace(0, mpi_size, lon.size, endpoint=False) labels = np.floor(labels) - self._pu_indicators = labels + reorderedCoords = copy.copy(labels) + for i in range(0, len(kmeans.cluster_centers_)): + clusterCentre = kmeans.cluster_centers_[i] + distances = map(lambda point: ((point[0]-clusterCentre[0])**2 + (point[1]-clusterCentre[1])**2), coords) + sortedDistanceIdxs = np.argsort(list(distances)) + numberToChoose = sum(labels == i) + reorderedCoords[sortedDistanceIdxs[0:numberToChoose]] = i + coords[sortedDistanceIdxs[0:numberToChoose],:] = float('inf') + self._pu_indicators = reorderedCoords else: self._pu_indicators = None self._pu_indicators = mpi_comm.bcast(self._pu_indicators, root=0)