Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Chore/style cleanup utility functions #80

Merged
Merged
Show file tree
Hide file tree
Changes from 23 commits
Commits
Show all changes
48 commits
Select commit Hold shift + click to select a range
4da5d57
feat: simplify interface.py, while adding type hints, decorators, and…
Sep 4, 2024
c62b679
ref: utility functions; add type hints, optimize memory usage, and en…
Sep 4, 2024
eadf6c1
feat: Enhance _configure_if_nx_active decorator: add type hints, opti…
Sep 4, 2024
aa6bc39
chore: standard NX_GTYPE hints, remove dangling import
Sep 4, 2024
fafa1df
fix: circular import errors
Sep 4, 2024
7f42f2f
fix: failing tests
Sep 4, 2024
30d8b17
test: add unit tests for utils/chunk.py
Sep 4, 2024
00714ce
fix: create separate types.py module to avoid import order int __init…
Sep 4, 2024
f94b58b
fix: create separate types.py module to avoid import order int __init…
Sep 4, 2024
621c3ed
chore: pre-commit formatting
Sep 4, 2024
6c6387c
fix: revert to orig
Sep 4, 2024
f51d803
chore: Add .history to .gitignore
Sep 4, 2024
8884992
chore: Make script.sh executable
Sep 4, 2024
eed0485
chore: revert script.sh from main
Sep 4, 2024
3ced657
fix(update_get_info): correctly exclude utils functions from __init__…
Sep 4, 2024
ac3ffc9
feat: remove type annotations from function signatures
Sep 13, 2024
751a554
feat: remove type annotations from function signatures
Sep 13, 2024
8d71691
Update nx_parallel/interface.py
dPys Sep 14, 2024
e5cc374
Update nx_parallel/interface.py
dPys Sep 14, 2024
acd2c6a
feat: respond to code review
Sep 14, 2024
b8c5942
chore: commit unstaged changes
Sep 14, 2024
be877e9
fix(test): update enums to plural
Sep 14, 2024
e26e890
fix: revert enums to singular to prevent breaking changes
Sep 14, 2024
164e827
Update .gitignore
dPys Sep 20, 2024
3580262
Update nx_parallel/utils/chunk.py
dPys Sep 20, 2024
24944b0
Update nx_parallel/utils/chunk.py
dPys Sep 20, 2024
f354c53
Update nx_parallel/utils/tests/test_chunk.py
dPys Sep 20, 2024
b93fd77
Update nx_parallel/utils/chunk.py
dPys Sep 20, 2024
74b9258
chore: run precommit
Sep 20, 2024
fae06b5
chore: run precommit
Sep 20, 2024
b00873e
feat: remove GraphIteratorType
Sep 21, 2024
5f19baf
feat: remove GraphIteratorType
Sep 21, 2024
815e279
feat: remove GraphIteratorType
Sep 21, 2024
db52df2
chire: docstring spacing
Sep 22, 2024
18a0b14
chore: .gitignore
Sep 22, 2024
cb69e26
chore: .gitignore
Sep 22, 2024
f885745
Update nx_parallel/utils/chunk.py
dPys Sep 24, 2024
0e10c02
chore: address styling discrepancies highlighted in code review
Sep 24, 2024
b1a1f3a
chore: remove .DS_Store
Sep 24, 2024
6ec895e
chore: manually add in ValueError wording change
Sep 24, 2024
c84b87b
chore: manually add in ValueError wording change
Sep 24, 2024
ef60ca3
Merge remote-tracking branch 'origin/chore/style-cleanup-utility-func…
Sep 24, 2024
add9823
doc: enforce imperative mood in docstring
Sep 26, 2024
54e8589
chore: rerun precommit
Sep 26, 2024
7a40471
chore: rerun precommit
Sep 26, 2024
90dd4ba
chore: rerun precommit
Sep 26, 2024
37eba94
chore: rerun precommit
Sep 26, 2024
7473edf
moved update-get-info hook to the end
Schefflera-Arboricola Sep 27, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -135,3 +135,6 @@ html/

# get_info update script
temp__init__.py
.history/

.DS_Store
dPys marked this conversation as resolved.
Show resolved Hide resolved
2 changes: 1 addition & 1 deletion _nx_parallel/update_get_info.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ def get_funcs_info():

nx_parallel_dir = os.path.join(os.getcwd(), "nx_parallel")
for root, dirs, files in os.walk(nx_parallel_dir):
if root[-29:] == "nx-parallel/nx_parallel/utils":
if "nx_parallel/utils" in root:
continue
for file in files:
if (
Expand Down
148 changes: 64 additions & 84 deletions nx_parallel/interface.py
Original file line number Diff line number Diff line change
@@ -1,50 +1,64 @@
from nx_parallel.algorithms.bipartite.redundancy import node_redundancy
from nx_parallel.algorithms.centrality.betweenness import (
betweenness_centrality,
edge_betweenness_centrality,
)
from nx_parallel.algorithms.shortest_paths.generic import all_pairs_all_shortest_paths
from nx_parallel.algorithms.shortest_paths.weighted import (
all_pairs_dijkstra,
all_pairs_dijkstra_path_length,
all_pairs_dijkstra_path,
all_pairs_bellman_ford_path_length,
all_pairs_bellman_ford_path,
johnson,
)
from nx_parallel.algorithms.shortest_paths.unweighted import (
all_pairs_shortest_path,
all_pairs_shortest_path_length,
)
from nx_parallel.algorithms.efficiency_measures import local_efficiency
from nx_parallel.algorithms.isolate import number_of_isolates
from nx_parallel.algorithms.tournament import (
is_reachable,
tournament_is_strongly_connected,
)
from nx_parallel.algorithms.vitality import closeness_vitality
from nx_parallel.algorithms.approximation.connectivity import (
approximate_all_pairs_node_connectivity,
)
from nx_parallel.algorithms.connectivity import connectivity
from nx_parallel.algorithms.cluster import square_clustering
from typing import Union
from operator import attrgetter
dschult marked this conversation as resolved.
Show resolved Hide resolved

import networkx as nx

from nx_parallel import algorithms

__all__ = ["BackendInterface", "ParallelGraph"]

NX_GTYPES = Union[nx.Graph, nx.DiGraph, nx.MultiGraph, nx.MultiDiGraph]

ALGORITHMS = [
# Bipartite
"node_redundancy",
# Isolates
"number_of_isolates",
# Vitality
"closeness_vitality",
# Tournament
"is_reachable",
"tournament_is_strongly_connected",
# Centrality
"betweenness_centrality",
"edge_betweenness_centrality",
# Efficiency
"local_efficiency",
# Shortest Paths : generic
"all_pairs_all_shortest_paths",
# Shortest Paths : weighted graphs
"all_pairs_dijkstra",
"all_pairs_dijkstra_path_length",
"all_pairs_dijkstra_path",
"all_pairs_bellman_ford_path_length",
"all_pairs_bellman_ford_path",
"johnson",
# Clustering
"square_clustering",
# Shortest Paths : unweighted graphs
"all_pairs_shortest_path",
"all_pairs_shortest_path_length",
# Approximation
"approximate_all_pairs_node_connectivity",
# Connectivity
"connectivity.all_pairs_node_connectivity",
]


class ParallelGraph:
"""A wrapper class for networkx.Graph, networkx.DiGraph, networkx.MultiGraph,
and networkx.MultiDiGraph."""
and networkx.MultiDiGraph.
"""

__networkx_backend__ = "parallel"

def __init__(self, graph_object=None):
def __init__(
self,
graph_object,
):
if graph_object is None:
dPys marked this conversation as resolved.
Show resolved Hide resolved
self.graph_object = nx.Graph()
elif isinstance(
graph_object, (nx.Graph, nx.DiGraph, nx.MultiGraph, nx.MultiDiGraph)
):
elif isinstance(graph_object, NX_GTYPES):
self.graph_object = graph_object
else:
self.graph_object = nx.Graph(graph_object)
Expand All @@ -56,70 +70,36 @@ def is_directed(self):
return self.graph_object.is_directed()

def __str__(self):
return "Parallel" + str(self.graph_object)
return f"Parallel{self.graph_object}"


class BackendInterface:
"""BackendInterface class for parallel algorithms."""
def assign_algorithms(cls):
"""Class decorator to assign algorithms to the class attributes."""
for attr in ALGORITHMS:
# get the function name by parsing the module hierarchy
func_name = attr.rsplit(".", 1)[-1]
setattr(cls, func_name, attrgetter(attr)(algorithms))
return cls

# Bipartite
node_redundancy = node_redundancy

# Isolates
number_of_isolates = number_of_isolates

# Vitality
closeness_vitality = closeness_vitality

# Tournament
is_reachable = is_reachable
tournament_is_strongly_connected = tournament_is_strongly_connected

# Centrality
betweenness_centrality = betweenness_centrality
edge_betweenness_centrality = edge_betweenness_centrality

# Efficiency
local_efficiency = local_efficiency

# Shortest Paths : generic
all_pairs_all_shortest_paths = all_pairs_all_shortest_paths

# Shortest Paths : weighted graphs
all_pairs_dijkstra = all_pairs_dijkstra
all_pairs_dijkstra_path_length = all_pairs_dijkstra_path_length
all_pairs_dijkstra_path = all_pairs_dijkstra_path
all_pairs_bellman_ford_path_length = all_pairs_bellman_ford_path_length
all_pairs_bellman_ford_path = all_pairs_bellman_ford_path
johnson = johnson

# Clustering
square_clustering = square_clustering

# Shortest Paths : unweighted graphs
all_pairs_shortest_path = all_pairs_shortest_path
all_pairs_shortest_path_length = all_pairs_shortest_path_length

# Approximation
approximate_all_pairs_node_connectivity = approximate_all_pairs_node_connectivity

# Connectivity
all_pairs_node_connectivity = connectivity.all_pairs_node_connectivity

# =============================
@assign_algorithms
class BackendInterface:
"""BackendInterface class for parallel algorithms."""

@staticmethod
def convert_from_nx(graph, *args, **kwargs):
"""Convert a networkx.Graph, networkx.DiGraph, networkx.MultiGraph,
or networkx.MultiDiGraph to a ParallelGraph."""
or networkx.MultiDiGraph to a ParallelGraph.
"""
if isinstance(graph, ParallelGraph):
return graph
return ParallelGraph(graph)

@staticmethod
def convert_to_nx(result, *, name=None):
"""Convert a ParallelGraph to a networkx.Graph, networkx.DiGraph,
networkx.MultiGraph, or networkx.MultiDiGraph."""
networkx.MultiGraph, or networkx.MultiDiGraph.
"""
if isinstance(result, ParallelGraph):
return result.graph_object
return result
5 changes: 3 additions & 2 deletions nx_parallel/tests/test_get_chunks.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,12 @@

import inspect
import importlib
import networkx as nx
import nx_parallel as nxp
import random
import types
import math
import networkx as nx

import nx_parallel as nxp
dPys marked this conversation as resolved.
Show resolved Hide resolved


def get_all_functions(package_name="nx_parallel"):
Expand Down
111 changes: 72 additions & 39 deletions nx_parallel/utils/chunk.py
Original file line number Diff line number Diff line change
@@ -1,66 +1,99 @@
import itertools
import os

dPys marked this conversation as resolved.
Show resolved Hide resolved
import networkx as nx

from nx_parallel.utils.types import GraphIteratorType


__all__ = ["chunks", "get_n_jobs", "create_iterables"]


def chunks(iterable, n):
"""Divides an iterable into chunks of size n"""
def chunks(iterable, n_chunks):
"""Yield exactly `n_chunks` chunks from `iterable`, balancing the chunk sizes."""
iterable = list(iterable)
k, m = divmod(len(iterable), n_chunks)
it = iter(iterable)
while True:
x = tuple(itertools.islice(it, n))
if not x:
return
yield x
for _ in range(n_chunks):
chunk_size = k + (1 if m > 0 else 0)
m -= 1
yield tuple(itertools.islice(it, chunk_size))
dPys marked this conversation as resolved.
Show resolved Hide resolved


def get_n_jobs(n_jobs=None):
"""Returns the positive value of `n_jobs`."""
"""Return the positive value of `n_jobs`, adjusting for the environment.

If running under pytest, returns 2 jobs. If using a parallel backend,
returns the number of jobs configured for the backend. Otherwise, returns
the number of CPUs adjusted for negative `n_jobs` values.
"""
dPys marked this conversation as resolved.
Show resolved Hide resolved
if "PYTEST_CURRENT_TEST" in os.environ:
return 2
else:

if n_jobs == 0:
raise ValueError("n_jobs == 0 in Parallel has no meaning")
dPys marked this conversation as resolved.
Show resolved Hide resolved

if not n_jobs:
dPys marked this conversation as resolved.
Show resolved Hide resolved
if nx.config.backends.parallel.active:
n_jobs = nx.config.backends.parallel.n_jobs
else:
from joblib.parallel import get_active_backend

n_jobs = get_active_backend()[1]
n_cpus = os.cpu_count()
if n_jobs is None:
return 1
if n_jobs < 0:
return n_cpus + n_jobs + 1
if n_jobs == 0:
raise ValueError("n_jobs == 0 in Parallel has no meaning")
return int(n_jobs)


def create_iterables(G, iterator, n_cores, list_of_iterator=None):
"""Creates an iterable of function inputs for parallel computation

if not n_jobs:
return 1 # Default to 1 if no valid n_jobs is found or passed
dPys marked this conversation as resolved.
Show resolved Hide resolved
if n_jobs < 0:
return os.cpu_count() + n_jobs + 1

return int(n_jobs)


def create_iterables(
G,
iterator,
n_cores,
list_of_iterator=None,
):
dPys marked this conversation as resolved.
Show resolved Hide resolved
"""Create an iterable of function inputs for parallel computation
dschult marked this conversation as resolved.
Show resolved Hide resolved
based on the provided iterator type.

Parameters
-----------
----------
G : NetworkX graph
iterator : str
Type of iterator. Valid values are 'node', 'edge', 'isolate'
The NetworkX graph.
iterator : GraphIteratorType
Type of iterator. Valid values are 'NODE', 'EDGE', 'ISOLATE'.
dPys marked this conversation as resolved.
Show resolved Hide resolved
n_cores : int
The number of cores to use.
list_of_iterator : list, optional
A precomputed list of items to iterate over. If None, it will
be generated based on the iterator type.

Returns:
--------
iterable : Iterable
Returns
-------
iterable
dPys marked this conversation as resolved.
Show resolved Hide resolved
An iterable of function inputs.

Raises
------
ValueError
If the iterator type is not valid.
dPys marked this conversation as resolved.
Show resolved Hide resolved
"""
if isinstance(iterator, str):
iterator = GraphIteratorType(iterator)

if not list_of_iterator:
if iterator == GraphIteratorType.NODE:
list_of_iterator = list(G.nodes)
elif iterator == GraphIteratorType.EDGE:
list_of_iterator = list(G.edges)
elif iterator == GraphIteratorType.ISOLATE:
list_of_iterator = list(nx.isolates(G))
else:
raise ValueError(f"Invalid iterator type: {iterator}")

if not list_of_iterator:
return iter([])

if iterator in ["node", "edge", "isolate"]:
if list_of_iterator is None:
if iterator == "node":
list_of_iterator = list(G.nodes)
elif iterator == "edge":
list_of_iterator = list(G.edges)
elif iterator == "isolate":
list_of_iterator = list(nx.isolates(G))
num_in_chunk = max(len(list_of_iterator) // n_cores, 1)
return chunks(list_of_iterator, num_in_chunk)
else:
raise ValueError("Invalid iterator type.")
return chunks(list_of_iterator, n_cores)
Loading
Loading