CLN Cleaned cluster/_hdbscan/_linkage.pyx
#24857
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,210 +1,241 @@ | ||
| # Minimum spanning tree single linkage implementation for hdbscan | ||
| # Authors: Leland McInnes <[email protected]> | ||
| # Steve Astels <[email protected]> | ||
| # Meekail Zain <[email protected]> | ||
| # License: 3-clause BSD | ||
|
|
||
| cimport numpy as cnp | ||
| from libc.float cimport DBL_MAX | ||
|
|
||
| import numpy as np | ||
| from ...metrics._dist_metrics cimport DistanceMetric | ||
| from ...cluster._hierarchical_fast cimport UnionFind | ||
| from ...utils._typedefs cimport ITYPE_t, DTYPE_t | ||
| from ...utils._typedefs import ITYPE, DTYPE | ||
|
|
||
| # Numpy structured dtype representing a single ordered edge in Prim's algorithm | ||
| MST_edge_dtype = np.dtype([ | ||
| ("current_node", np.int64), | ||
| ("next_node", np.int64), | ||
| ("distance", np.float64), | ||
| ]) | ||
|
|
||
| # Packed shouldn't make a difference since they're all 8-byte quantities, | ||
| # but it's included just to be safe. | ||
| ctypedef packed struct MST_edge_t: | ||
| cnp.int64_t current_node | ||
| cnp.int64_t next_node | ||
| cnp.float64_t distance | ||
|
|
||
| cpdef cnp.ndarray[MST_edge_t, ndim=1, mode='c'] mst_from_mutual_reachability( | ||
| cnp.ndarray[cnp.float64_t, ndim=2] mutual_reachability | ||
| ): | ||
|
Comment on lines
+30
to
32
There was a problem hiding this comment. We should only use Is the usage of Generally, most There was a problem hiding this comment. The current algorithm uses binary masks for indexing into subsets of scikit-learn/sklearn/cluster/_hdbscan/_linkage.pyx Lines 48 to 52 in 39e7d7e There was a problem hiding this comment. I could attempt to change the algorithm to avoid using this, but it is a pretty eloquent solution. Ultimately I don't know how costly the python interactions here are, and whether they outweigh the usefulness of the algorithm as it is now. There was a problem hiding this comment. There are some overhead of just typing and referencing variable with We can leave this for now. If this shows to be a hotspot, we can have a PR to rewrite this part. What do you think? In this case, is it possible to add a comment to indicate that numpy arrays are used for binary masks and convenient indexing? There was a problem hiding this comment. Sounds good, I will add in such a comment. There was a problem hiding this comment. Was this comment added somewhere? |
||
| """Compute the Minimum Spanning Tree (MST) representation of the mutual- | ||
| reachability graph using Prim's algorithm. | ||
|
|
||
| Parameters | ||
| ---------- | ||
| mutual_reachability : ndarray of shape (n_samples, n_samples) | ||
| Array of mutual-reachabilities between samples. | ||
|
|
||
| Returns | ||
| ------- | ||
| mst : ndarray of shape (n_samples - 1,), dtype=MST_edge_dtype | ||
| The MST representation of the mutual-reahability graph. The MST is | ||
| represented as a collecteion of edges. | ||
| """ | ||
| cdef: | ||
| # Note: we utilize ndarray's over memory-views to make use of numpy | ||
| # binary indexing and sub-selection below. | ||
|
Comment on lines
+48
to
+49
There was a problem hiding this comment. Thank you for this comment, this answers one of @Vincent-Maladiere's questions. 👍 |
||
| cnp.ndarray[cnp.int64_t, ndim=1, mode='c'] current_labels | ||
| cnp.ndarray[cnp.float64_t, ndim=1, mode='c'] min_reachability, left, right | ||
| cnp.ndarray[MST_edge_t, ndim=1, mode='c'] mst | ||
|
|
||
| cnp.ndarray[cnp.uint8_t, mode='c'] label_filter | ||
|
|
||
| cnp.int64_t n_samples = mutual_reachability.shape[0] | ||
| cnp.int64_t current_node, new_node_index, new_node, i | ||
|
|
||
| mst = np.empty(n_samples - 1, dtype=MST_edge_dtype) | ||
| current_labels = np.arange(n_samples, dtype=np.int64) | ||
| current_node = 0 | ||
| min_reachability = np.full(n_samples, fill_value=np.infty, dtype=np.float64) | ||
| for i in range(0, n_samples - 1): | ||
| label_filter = current_labels != current_node | ||
| current_labels = current_labels[label_filter] | ||
| left = min_reachability[label_filter] | ||
| right = mutual_reachability[current_node][current_labels] | ||
| min_reachability = np.minimum(left, right) | ||
|
|
||
| new_node_index = np.argmin(min_reachability) | ||
| new_node = current_labels[new_node_index] | ||
| mst[i].current_node = current_node | ||
| mst[i].next_node = new_node | ||
| mst[i].distance = min_reachability[new_node_index] | ||
| current_node = new_node | ||
|
|
||
| return mst | ||
|
|
||
|
|
||
| cpdef cnp.ndarray[MST_edge_t, ndim=1, mode='c'] mst_from_data_matrix( | ||
| const cnp.float64_t[:, ::1] raw_data, | ||
| const cnp.float64_t[::1] core_distances, | ||
| DistanceMetric dist_metric, | ||
| cnp.float64_t alpha=1.0 | ||
| ): | ||
| """Compute the Minimum Spanning Tree (MST) representation of the mutual- | ||
| reachability graph generated from the provided `raw_data` and | ||
| `core_distances` using Prim's algorithm. | ||
|
|
||
| Parameters | ||
| ---------- | ||
| raw_data : ndarray of shape (n_samples, n_features) | ||
| Input array of data samples. | ||
|
|
||
| core_distances : ndarray of shape (n_samples,) | ||
| An array containing the core-distance calculated for each corresponding | ||
| sample. | ||
|
|
||
| dist_metric : DistanceMetric | ||
| The distance metric to use when calculating pairwise distances for | ||
| determining mutual-reachability. | ||
|
|
||
| Returns | ||
| ------- | ||
| mst : ndarray of shape (n_samples - 1,), dtype=MST_edge_dtype | ||
| The MST representation of the mutual-reahability graph. The MST is | ||
| represented as a collecteion of edges. | ||
| """ | ||
|
|
||
| cdef: | ||
| cnp.int8_t[::1] in_tree | ||
| cnp.float64_t[::1] min_reachability | ||
| cnp.int64_t[::1] current_sources | ||
| cnp.ndarray[MST_edge_t, ndim=1, mode='c'] mst | ||
|
|
||
| cnp.int64_t current_node, source_node, right_node, left_node, new_node, next_node_source | ||
| cnp.int64_t i, j, n_samples, num_features | ||
|
|
||
| cnp.float64_t current_node_core_dist, new_reachability, mutual_reachability_distance | ||
| cnp.float64_t next_node_min_reach, pair_distance, next_node_core_dist | ||
|
|
||
| n_samples = raw_data.shape[0] | ||
| num_features = raw_data.shape[1] | ||
|
|
||
| mst = np.empty(n_samples - 1, dtype=MST_edge_dtype) | ||
|
|
||
| in_tree = np.zeros(n_samples, dtype=np.int8) | ||
| min_reachability = np.full(n_samples, fill_value=np.infty, dtype=np.float64) | ||
| current_sources = np.ones(n_samples, dtype=np.int64) | ||
|
|
||
| current_node = 0 | ||
|
|
||
| for i in range(0, n_samples - 1): | ||
|
|
||
| in_tree[current_node] = 1 | ||
|
|
||
| current_node_core_dist = core_distances[current_node] | ||
|
|
||
| new_reachability = DBL_MAX | ||
| source_node = 0 | ||
| new_node = 0 | ||
|
|
||
| for j in range(n_samples): | ||
| if in_tree[j]: | ||
| continue | ||
|
|
||
| next_node_min_reach = min_reachability[j] | ||
| next_node_source = current_sources[j] | ||
|
|
||
| pair_distance = dist_metric.dist( | ||
| &raw_data[current_node, 0], | ||
| &raw_data[j, 0], | ||
| num_features | ||
| ) | ||
|
|
||
| pair_distance /= alpha | ||
|
|
||
| next_node_core_dist = core_distances[j] | ||
| mutual_reachability_distance = max( | ||
| current_node_core_dist, | ||
| next_node_core_dist, | ||
| pair_distance | ||
| ) | ||
Micky774 marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| if mutual_reachability_distance > next_node_min_reach: | ||
| if next_node_min_reach < new_reachability: | ||
| new_reachability = next_node_min_reach | ||
| source_node = next_node_source | ||
| new_node = j | ||
| continue | ||
|
|
||
| if mutual_reachability_distance < next_node_min_reach: | ||
| min_reachability[j] = mutual_reachability_distance | ||
| current_sources[j] = current_node | ||
| if mutual_reachability_distance < new_reachability: | ||
| new_reachability = mutual_reachability_distance | ||
| source_node = current_node | ||
| new_node = j | ||
| else: | ||
| if next_node_min_reach < new_reachability: | ||
| new_reachability = next_node_min_reach | ||
| source_node = next_node_source | ||
| new_node = j | ||
|
|
||
| mst[i].current_node = source_node | ||
| mst[i].next_node = new_node | ||
| mst[i].distance = new_reachability | ||
| current_node = new_node | ||
|
|
||
| return mst | ||
|
|
||
| cpdef cnp.ndarray[cnp.float64_t, ndim=2, mode='c'] make_single_linkage(const MST_edge_t[::1] mst): | ||
| """Construct a single-linkage tree from an MST. | ||
|
|
||
| Parameters | ||
| ---------- | ||
| mst : ndarray of shape (n_samples - 1,), dtype=MST_edge_dtype | ||
| The MST representation of the mutual-reahability graph. The MST is | ||
| represented as a collecteion of edges. | ||
|
|
||
| Returns | ||
| ------- | ||
| single_linkage : ndarray of shape (n_samples - 1, 4) | ||
| The single-linkage tree tree (dendrogram) built from the MST. Each | ||
| of the array represents the following: | ||
|
|
||
| - left node/cluster | ||
| - right node/cluster | ||
| - distance | ||
| - new cluster size | ||
| """ | ||
| cdef: | ||
| cnp.ndarray[cnp.float64_t, ndim=2, mode='c'] single_linkage | ||
|
|
||
| # Note mst.shape[0] is one fewer than the number of samples | ||
| cnp.int64_t n_samples = mst.shape[0] + 1 | ||
| cnp.int64_t current_node_cluster, next_node_cluster | ||
| cnp.int64_t current_node, next_node, index | ||
| cnp.float64_t distance | ||
| UnionFind U = UnionFind(n_samples) | ||
|
|
||
| single_linkage = np.zeros((n_samples - 1, 4), dtype=np.float64) | ||
|
|
||
| for i in range(n_samples - 1): | ||
|
|
||
| current_node = mst[i].current_node | ||
| next_node = mst[i].next_node | ||
| distance = mst[i].distance | ||
|
|
||
| current_node_cluster = U.fast_find(current_node) | ||
| next_node_cluster = U.fast_find(next_node) | ||
|
|
||
| # TODO: Update this to an array of structs (AoS). | ||
| # Should be done simultaneously in _tree.pyx to ensure compatability. | ||
| single_linkage[i][0] = <cnp.float64_t> current_node_cluster | ||
| single_linkage[i][1] = <cnp.float64_t> next_node_cluster | ||
| single_linkage[i][2] = distance | ||
| single_linkage[i][3] = U.size[current_node_cluster] + U.size[next_node_cluster] | ||
|
|
||
| U.union(current_node_cluster, next_node_cluster) | ||
|
|
||
| return single_linkage | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.