diff --git a/docs/API.md b/docs/API.md index fefc492..9e63df2 100644 --- a/docs/API.md +++ b/docs/API.md @@ -86,7 +86,7 @@ df.show(10) ```python from ng_ai import NebulaReader from ng_ai.config import NebulaGraphConfig -# read data with spark engine, query mode +# read data with nebula engine, query mode config_dict = { "graphd_hosts": "127.0.0.1:9669", "user": "root", @@ -197,6 +197,46 @@ MATCH (v:louvain) RETURN id(v), v.louvain.cluster_id LIMIT 10; ``` +#### NebulaGraph Engine(NetworkX) Writer + +Create schema in NebulaGraph: + +```ngql +CREATE TAG IF NOT EXISTS pagerank ( + pagerank double NOT NULL +); +``` + +Assuming we have a `graph_result` computed with `graph.algo.pagerank()`: + +```python +graph_result = g.algo.pagerank() +``` + +Then we could write the pagerank result back to NebulaGraph with the following code: + +```python +from ng_ai import NebulaWriter + +writer = NebulaWriter( + data=graph_result, + sink="nebulagraph_vertex", + config=config, + engine="nebula", +) + +# properties to write +properties = ["pagerank"] + +writer.set_options( + tag="pagerank", + properties=properties, + batch_size=256, + write_mode="insert", +) +# write back to NebulaGraph +writer.write() +``` ## NebulaGNN diff --git a/examples/networkx_engine.ipynb b/examples/networkx_engine.ipynb index 0165926..a87b847 100644 --- a/examples/networkx_engine.ipynb +++ b/examples/networkx_engine.ipynb @@ -212,17 +212,14 @@ "metadata": {}, "outputs": [], "source": [ - "# lpa_result = df.algo.label_propagation()\n", - "# louvain_result = df.algo.louvain()\n", - "# k_core_result = df.algo.k_core()\n", - "# degree_statics_result = df.algo.degree_statics()\n", - "# betweenness_centrality_result = df.algo.betweenness_centrality()\n", - "# coefficient_centrality_result = df.algo.coefficient_centrality()\n", - "# bfs_result = df.algo.bfs()\n", - "# hanp_result = df.algo.hanp()\n", - "# jaccard_result = df.algo.jaccard()\n", - "# strong_connected_components_result = df.algo.strong_connected_components()\n", - "# triangle_count_result = df.algo.triangle_count()" + "# get all algorithms\n", + "g.algo.get_all_algo()\n", + "\n", + "# get help of each algo\n", + "help(g.algo.node2vec)\n", + "\n", + "# call the algo\n", + "g.algo.node2vec()" ] } ], diff --git a/examples/ng_ai_from_ngql_udf.ipynb b/examples/ng_ai_from_ngql_udf.ipynb index b7c5821..8e28c35 100644 --- a/examples/ng_ai_from_ngql_udf.ipynb +++ b/examples/ng_ai_from_ngql_udf.ipynb @@ -119,6 +119,7 @@ ] }, { + "attachments": {}, "cell_type": "markdown", "id": "c4630751", "metadata": {}, @@ -131,7 +132,7 @@ "CREATE TAG IF NOT EXISTS k_core(kcore string);\n", "CREATE TAG IF NOT EXISTS degree_statics(degree int,inDegree int,outDegree int);\n", "CREATE TAG IF NOT EXISTS betweenness_centrality(betweenness double);\n", - "CREATE TAG IF NOT EXISTS coefficient_centrality(clustercoefficient double);\n", + "CREATE TAG IF NOT EXISTS clustering_coefficient(clustercoefficient double);\n", "CREATE TAG IF NOT EXISTS bfs(bfs string);\n", "CREATE TAG IF NOT EXISTS hanp(hanp string);\n", "CREATE TAG IF NOT EXISTS jaccard(jaccard string);\n", diff --git a/examples/spark_engine.ipynb b/examples/spark_engine.ipynb index 73d5cc1..06ee206 100644 --- a/examples/spark_engine.ipynb +++ b/examples/spark_engine.ipynb @@ -594,7 +594,7 @@ "# k_core_result = df.algo.k_core()\n", "# degree_statics_result = df.algo.degree_statics()\n", "# betweenness_centrality_result = df.algo.betweenness_centrality()\n", - "# coefficient_centrality_result = df.algo.coefficient_centrality()\n", + "# clustering_coefficient_result = df.algo.clustering_coefficient()\n", "# bfs_result = df.algo.bfs()\n", "# hanp_result = df.algo.hanp()\n", "# jaccard_result = df.algo.jaccard()\n", diff --git a/ng_ai/engines.py b/ng_ai/engines.py index e4cc2cb..5b86e5e 100644 --- a/ng_ai/engines.py +++ b/ng_ai/engines.py @@ -118,6 +118,7 @@ def __init__(self, config=None): self.config = config # let's make all nx related import here + import community as community_louvain import networkx as nx import ng_nx from ng_nx import NebulaReader as NxReader @@ -125,12 +126,15 @@ def __init__(self, config=None): from ng_nx import NebulaWriter as NxWriter from ng_nx.utils import NebulaGraphConfig as NxConfig from ng_nx.utils import result_to_df + from node2vec import Node2Vec self.nx = nx self.ng_nx = ng_nx self.nx_reader = NxReader self.nx_writer = NxWriter self.nx_scan_reader = NxScanReader + self.nx_community_louvain = community_louvain + self.nx_node2vec = Node2Vec self._nx_config = NxConfig self.nx_config = None diff --git a/ng_ai/nebula_algo.py b/ng_ai/nebula_algo.py index 5d7fb4e..5bd4e8b 100644 --- a/ng_ai/nebula_algo.py +++ b/ng_ai/nebula_algo.py @@ -181,9 +181,7 @@ def degree_statics(self): return result @algo - def betweenness_centrality( - self, max_iter: int = 10, degree: int = 2, weighted: bool = False - ): + def betweenness_centrality(self, max_iter: int = 10, weighted: bool = False): engine, spark, jspark, encode_vid = self.get_spark_engine_context( "BetweennessConfig", "BetweennessCentralityAlgo" ) @@ -197,11 +195,11 @@ def betweenness_centrality( return result @algo - def coefficient_centrality(self, type: str = "local"): + def clustering_coefficient(self, type: str = "local"): # type could be either "local" or "global" assert type.lower() in ["local", "global"], ( "type should be either local or global" - f"in coefficient_centrality algo. Got type: {type}" + f"in clustering_coefficient algo. Got type: {type}" ) engine, spark, jspark, encode_vid = self.get_spark_engine_context( "CoefficientConfig", "ClusteringCoefficientAlgo" @@ -247,6 +245,9 @@ def hanp( weighted: bool = False, preferences=None, ): + """ + Hop Attenuation & Node Preference + """ engine, spark, jspark, encode_vid = self.get_spark_engine_context( "HanpConfig", "HanpAlgo" ) @@ -345,7 +346,7 @@ def triangle_count(self): return result # @algo - # def closeness(self, weighted: bool = False): + # def closeness_centrality(self, weighted: bool = False): # # TBD: ClosenessAlgo is not yet encodeID compatible # engine, spark, jspark, encode_vid = self.get_spark_engine_context( # "ClosenessConfig", "ClosenessAlgo" @@ -410,3 +411,280 @@ def pagerank(self, reset_prob=0.15, max_iter=10, **kwargs): return self.engine.nx.pagerank( g, alpha=1 - reset_prob, max_iter=max_iter, tol=tol, weight=weight ) + + @algo + def connected_components(self): + self.check_engine() + g = self.ngraph.get_nx_graph() + ug = g.to_undirected() + return self.engine.nx.connected_components(ug) + + @algo + def louvain(self, weight: str = None, resolution: float = 1.0): + """ + doc: https://github.com/taynaud/python-louvain + """ + weight = weight if weight else "" + self.check_engine() + g = self.ngraph.get_nx_graph() + ug = g.to_undirected() + return self.engine.nx_community_louvain.best_partition( + ug, weight=weight, resolution=resolution + ) + + @algo + def label_propagation(self, **kwargs): + """ + doc: https://networkx.org/documentation/networkx-2.6.2/reference/algorithms/community.html + """ + + self.check_engine() + g = self.ngraph.get_nx_graph() + ug = g.to_undirected() + return self.engine.nx.algorithms.community.label_propagation_communities( + ug, **kwargs + ) + + @algo + def k_core(self, k: int = 2): + """ + doc: https://networkx.org/documentation/networkx-2.6.2/reference/algorithms/generated/networkx.algorithms.core.k_core.html + return: networkx.classes.digraph.DiGraph + """ + # TBD, k_core requires single graph + self.check_engine() + g = self.ngraph.get_nx_graph() + # MultiDiGraph to DiGraph + single_g = self.engine.nx.DiGraph() + for u, v in g.edges(): + single_g.add_edge(u, v) + return self.engine.nx.k_core(single_g, k=k) + + @algo + def k_truss(self, k: int = 2): + """ + doc: https://networkx.org/documentation/networkx-2.6.2/reference/algorithms/generated/networkx.algorithms.core.k_truss.html + return: networkx.classes.graph.Graph + """ + self.check_engine() + g = self.ngraph.get_nx_graph() + # MultiDiGraph to Graph + single_ug = self.engine.nx.Graph() + for u, v in g.edges(): + single_ug.add_edge(u, v) + return self.engine.nx.k_truss(single_ug, k=k) + + @algo + def k_clique_communities(self, k: int = 2, **kwargs): + """ + doc: https://networkx.org/documentation/networkx-2.6.2/reference/algorithms/community.html + return: Yields sets of nodes, one for each k-clique community. + """ + self.check_engine() + g = self.ngraph.get_nx_graph() + ug = g.to_undirected() + return self.engine.nx.algorithms.community.k_clique_communities( + ug, k=k, **kwargs + ) + + @algo + def degree_statics(self): + """ + doc: https://networkx.org/documentation/networkx-2.6.2/reference/classes/generated/networkx.MultiDiGraph.degree.html + return: List[Tuple[str, int, int, int]], (node_id, degree, in_degree, out_degree) + """ + self.check_engine() + g = self.ngraph.get_nx_graph() + degrees = dict(g.degree()) + in_degrees = dict(g.in_degree()) + out_degrees = dict(g.out_degree()) + + result = [] + for node_id in g.nodes(): + degree = degrees[node_id] + in_degree = in_degrees[node_id] + out_degree = out_degrees[node_id] + result.append((node_id, degree, in_degree, out_degree)) + return result + + @algo + def betweenness_centrality( + self, k: int = None, normalized: bool = True, weight: str = None, **kwargs + ): + """ + doc: https://networkx.org/documentation/networkx-2.6.2/reference/algorithms/generated/networkx.algorithms.centrality.betweenness_centrality.html + return: Dictionary of nodes with betweenness centrality as the value. + """ + self.check_engine() + g = self.ngraph.get_nx_graph() + # MultiDiGraph to DiGraph + single_g = self.engine.nx.DiGraph() + for u, v, data in g.edges(data=True): + if weight is not None and weight in data: + single_g.add_edge(u, v, weight=data[weight]) + else: + single_g.add_edge(u, v) + return self.engine.nx.betweenness_centrality( + single_g, k=k, normalized=normalized, weight=weight, **kwargs + ) + + @algo + def clustering_coefficient(self, weight: str = None, **kwargs): + """ + doc: https://networkx.org/documentation/networkx-2.6.2/reference/algorithms/generated/networkx.algorithms.cluster.clustering.html + return: Dictionary of nodes with clustering coefficient as the value. + """ + self.check_engine() + g = self.ngraph.get_nx_graph() + # MultiDiGraph to DiGraph + single_g = self.engine.nx.DiGraph() + for u, v, data in g.edges(data=True): + if weight is not None and weight in data: + single_g.add_edge(u, v, weight=data[weight]) + else: + single_g.add_edge(u, v) + return self.engine.nx.clustering(single_g, weight=weight, **kwargs) + + @algo + def bfs(self, root=None, max_depth: int = 10, reverse: bool = False, **kwargs): + """ + doc: https://networkx.org/documentation/networkx-2.6.2/reference/algorithms/generated/networkx.algorithms.traversal.breadth_first_search.bfs_edges.html + root: The node at which to start the search, defaults to the first node in the graph. + reverse: If True, perform a reverse breadth-first-search. + return: Yields edges in a breadth-first-search starting at source. + """ + self.check_engine() + g = self.ngraph.get_nx_graph() + if root is None: + root = next(iter(g.nodes())) + return self.engine.nx.bfs_edges( + g, source=root, depth_limit=max_depth, reverse=reverse, **kwargs + ) + + @algo + def dfs(self, root=None, max_depth: int = 10, **kwargs): + """ + doc: https://networkx.org/documentation/networkx-2.6.2/reference/algorithms/generated/networkx.algorithms.traversal.depth_first_search.dfs_edges.html + root: The node at which to start the search, defaults to the first node in the graph. + return: Yields edges in a depth-first-search starting at source. + """ + self.check_engine() + g = self.ngraph.get_nx_graph() + if root is None: + root = next(iter(g.nodes())) + return self.engine.nx.dfs_edges( + g, source=root, depth_limit=max_depth, **kwargs + ) + + @algo + def node2vec( + self, + dimensions: int = 128, + walk_length: int = 80, + num_walks: int = 10, + workers: int = 1, + fit_args: dict = {}, + **kwargs, + ): + """ + doc: https://github.com/eliorc/node2vec + dimensions: int, optional (default = 128), Dimensionality of the word vectors. + walk_length: int, optional (default = 80), Length of walk per source. Default value is 80. + num_walks: int, optional (default = 10), Number of walks per source. Default value is 10. + workers: int, optional (default = 1), Number of parallel workers. Default is 1. + fit_args: dict, optional (default = {}), Arguments for gensim.models.Word2Vec.fit() + return: gensim.models.keyedvectors.Word2VecKeyedVectorsmodel + """ + self.check_engine() + g = self.ngraph.get_nx_graph() + node2vec = self.engine.nx_node2vec( + graph=g, + dimensions=dimensions, + walk_length=walk_length, + num_walks=num_walks, + workers=workers, + **kwargs, + ) + model = node2vec.fit(**fit_args) + return model + + @algo + def jaccard(self, ebunch: list = None, **kwargs): + """ + doc: https://networkx.org/documentation/networkx-2.6.2/reference/algorithms/generated/networkx.algorithms.link_prediction.jaccard_coefficient.html + ebunch: iterable of node pairs, optional (default = None), If provided, only return the Jaccard coefficient for the specified pairs. + example: [('A', 'B'), ('A', 'C')] + return: Yields tuples of (u, v, p) where u and v are nodes and p is the Jaccard coefficient of the neighbors of u and v. + """ + self.check_engine() + g = self.ngraph.get_nx_graph() + # MultiDiGraph to Graph + single_ug = self.engine.nx.Graph() + for u, v in g.edges(): + single_ug.add_edge(u, v) + return self.engine.nx.jaccard_coefficient( + single_ug, ebunch=ebunch, **kwargs + ) + + @algo + def connected_components(self, **kwargs): + """ + doc: https://networkx.org/documentation/networkx-2.6.2/reference/algorithms/generated/networkx.algorithms.components.connected_components.html + return: A generator of sets of nodes, one for each connected component in the graph. + """ + self.check_engine() + g = self.ngraph.get_nx_graph() + # MultiDiGraph to MultiGraph + ug = g.to_undirected() + return self.engine.nx.connected_components(ug, **kwargs) + + @algo + def weakly_connected_components(self, **kwargs): + """ + doc: https://networkx.org/documentation/networkx-2.6.2/reference/algorithms/generated/networkx.algorithms.components.weakly_connected_components.html + return: A generator of sets of nodes, one for each weakly connected component in the graph. + """ + self.check_engine() + g = self.ngraph.get_nx_graph() + return self.engine.nx.weakly_connected_components(g, **kwargs) + + @algo + def strongly_connected_components(self, **kwargs): + """ + doc: https://networkx.org/documentation/networkx-2.6.2/reference/algorithms/generated/networkx.algorithms.components.strongly_connected_components.html + return: A generator of sets of nodes, one for each strongly connected component in the graph. + """ + self.check_engine() + g = self.ngraph.get_nx_graph() + return self.engine.nx.strongly_connected_components(g, **kwargs) + + @algo + def triangle_count(self, **kwargs): + """ + doc: https://networkx.org/documentation/networkx-2.6.2/reference/algorithms/generated/networkx.algorithms.cluster.triangles.html + return: A dictionary keyed by node to the number of triangles that include that node as a vertex. + """ + self.check_engine() + g = self.ngraph.get_nx_graph() + # MultiDiGraph to Graph + single_ug = self.engine.nx.Graph() + for u, v in g.edges(): + single_ug.add_edge(u, v) + return self.engine.nx.triangles(single_ug, **kwargs) + + @algo + def closeness_centrality( + self, u=None, distance=None, wf_improved=True, **kwargs + ): + """ + doc: https://networkx.org/documentation/networkx-2.6.2/reference/algorithms/generated/networkx.algorithms.centrality.closeness_centrality.html + u: node, optional (default = None), If specified, return only the value for the node u. + distance: edge attribute key, optional (default = None), Use the specified edge attribute as the edge distance in shortest path calculations. + wf_improved: bool, optional (default = True), If True, use the improved algorithm of Freeman and Bader which computes the closeness centrality using the number of reachable nodes instead of the number of nodes in the graph. + return: A dictionary keyed by node to the closeness centrality of that node. + """ + self.check_engine() + g = self.ngraph.get_nx_graph() + return self.engine.nx.closeness_centrality( + g, u=u, distance=distance, wf_improved=wf_improved, **kwargs + ) diff --git a/pdm.lock b/pdm.lock index 86366b4..eead203 100644 --- a/pdm.lock +++ b/pdm.lock @@ -19,7 +19,7 @@ summary = "Specifications for callback functions passed in to an API" [[package]] name = "black" -version = "23.1.0" +version = "23.3.0" requires_python = ">=3.7" summary = "The uncompromising code formatter." dependencies = [ @@ -35,12 +35,12 @@ dependencies = [ [[package]] name = "black" -version = "23.1.0" +version = "23.3.0" extras = ["jupyter"] requires_python = ">=3.7" summary = "The uncompromising code formatter." dependencies = [ - "black==23.1.0", + "black==23.3.0", "ipython>=7.8.0", "tokenize-rt>=3.2.0", ] @@ -110,6 +110,17 @@ version = "0.18.3" requires_python = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" summary = "Clean single-source support for Python 3 and 2" +[[package]] +name = "gensim" +version = "4.1.0" +requires_python = ">=3.6" +summary = "Python framework for fast Vector Space Modelling" +dependencies = [ + "numpy>=1.17.0", + "scipy>=0.18.1", + "smart-open>=1.8.1", +] + [[package]] name = "httplib2" version = "0.22.0" @@ -185,6 +196,12 @@ dependencies = [ "MarkupSafe>=2.0", ] +[[package]] +name = "joblib" +version = "1.2.0" +requires_python = ">=3.7" +summary = "Lightweight pipelining with Python functions" + [[package]] name = "markupsafe" version = "2.1.2" @@ -243,6 +260,18 @@ dependencies = [ "scipy>=1.7.3", ] +[[package]] +name = "node2vec" +version = "0.4.3" +summary = "Implementation of the node2vec algorithm." +dependencies = [ + "gensim", + "joblib>=0.13.2", + "networkx", + "numpy", + "tqdm", +] + [[package]] name = "numpy" version = "1.21.6" @@ -423,6 +452,12 @@ version = "1.16.0" requires_python = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" summary = "Python 2 and 3 compatibility utilities" +[[package]] +name = "smart-open" +version = "6.3.0" +requires_python = ">=3.6,<4.0" +summary = "Utils for streaming large files (S3, HDFS, GCS, Azure Blob Storage, gzip, bz2...)" + [[package]] name = "tokenize-rt" version = "5.0.0" @@ -435,6 +470,15 @@ version = "2.0.1" requires_python = ">=3.7" summary = "A lil' TOML parser" +[[package]] +name = "tqdm" +version = "4.65.0" +requires_python = ">=3.7" +summary = "Fast, Extensible Progress Meter" +dependencies = [ + "colorama; platform_system == \"Windows\"", +] + [[package]] name = "traitlets" version = "5.9.0" @@ -475,7 +519,7 @@ summary = "Backport of pathlib-compatible object wrapper for zip files" [metadata] lock_version = "4.1" -content_hash = "sha256:295dabb866e124f2b45815873bec9a5be9a1fe6893514bc26faa256753ab5016" +content_hash = "sha256:2a51745270d8b492cc07d7bd2260508fa52b52ef89e2cca3b79a26236d42620e" [metadata.files] "appnope 0.1.3" = [ @@ -490,32 +534,32 @@ content_hash = "sha256:295dabb866e124f2b45815873bec9a5be9a1fe6893514bc26faa25675 {url = "https://files.pythonhosted.org/packages/4c/1c/ff6546b6c12603d8dd1070aa3c3d273ad4c07f5771689a7b69a550e8c951/backcall-0.2.0-py2.py3-none-any.whl", hash = "sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255"}, {url = "https://files.pythonhosted.org/packages/a2/40/764a663805d84deee23043e1426a9175567db89c8b3287b5c2ad9f71aa93/backcall-0.2.0.tar.gz", hash = "sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e"}, ] -"black 23.1.0" = [ - {url = "https://files.pythonhosted.org/packages/01/8a/065d0a59c1ebe13186b12a2fa3965a41fc1588828709995e2630004d216e/black-23.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8178318cb74f98bc571eef19068f6ab5613b3e59d4f47771582f04e175570ed8"}, - {url = "https://files.pythonhosted.org/packages/15/11/533355217b1cc4a6df3263048060c1527f733d4720e158de2085293112bb/black-23.1.0.tar.gz", hash = "sha256:b0bd97bea8903f5a2ba7219257a44e3f1f9d00073d6cc1add68f0beec69692ac"}, - {url = "https://files.pythonhosted.org/packages/18/99/bb1be0ff3a7e912679ad234a3c4884fa7689dfcc4eae85bddb6c04feaa62/black-23.1.0-py3-none-any.whl", hash = "sha256:7a0f701d314cfa0896b9001df70a530eb2472babb76086344e688829efd97d32"}, - {url = "https://files.pythonhosted.org/packages/20/de/eff8e3ccc22b5c2be1265a9e61f1006d03e194519a3ca2e83dd8483dbbb5/black-23.1.0-cp38-cp38-macosx_10_16_x86_64.whl", hash = "sha256:49f7b39e30f326a34b5c9a4213213a6b221d7ae9d58ec70df1c4a307cf2a1580"}, - {url = "https://files.pythonhosted.org/packages/2d/9a/a81bf384a08d8a5e13d97223a60a74ac3c16c0aecdbd85edbc662d158bde/black-23.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:9afd3f493666a0cd8f8df9a0200c6359ac53940cbde049dcb1a7eb6ee2dd7074"}, - {url = "https://files.pythonhosted.org/packages/32/a7/1d207427b87780c505a41c9430d26362e729954503b8ffba27c4f53a6810/black-23.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2bf649fda611c8550ca9d7592b69f0637218c2369b7744694c5e4902873b2f3a"}, - {url = "https://files.pythonhosted.org/packages/3d/dc/12dc29bb38b8db68c79b8339de1590fe1ae796858bfa6cf7494eb672be21/black-23.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:8b70eb40a78dfac24842458476135f9b99ab952dd3f2dab738c1881a9b38b753"}, - {url = "https://files.pythonhosted.org/packages/3e/c0/abc7031d670d211e4e2a063910d587dfcb62ce469631e779b23b66653442/black-23.1.0-cp310-cp310-macosx_10_16_universal2.whl", hash = "sha256:57c18c5165c1dbe291d5306e53fb3988122890e57bd9b3dcb75f967f13411a26"}, - {url = "https://files.pythonhosted.org/packages/43/bc/5232fd6b0fd6d6177140cfb7d8f0f0e06638e2a750122767e265beb91161/black-23.1.0-cp39-cp39-macosx_10_16_arm64.whl", hash = "sha256:a29650759a6a0944e7cca036674655c2f0f63806ddecc45ed40b7b8aa314b651"}, - {url = "https://files.pythonhosted.org/packages/6b/d1/4394e4b0a24ad0f556aca3ab11e27f2e199f03b43f147c31a4befbf62b48/black-23.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:121ca7f10b4a01fd99951234abdbd97728e1240be89fde18480ffac16503d481"}, - {url = "https://files.pythonhosted.org/packages/77/11/db2ae5bf93af5185086d9b1baf4ce369ca34c3a01835230873aa9163d52d/black-23.1.0-cp39-cp39-macosx_10_16_universal2.whl", hash = "sha256:bb460c8561c8c1bec7824ecbc3ce085eb50005883a6203dcfb0122e95797ee06"}, - {url = "https://files.pythonhosted.org/packages/7e/fe/6c05c3f9255b7b498cfb88faa85b45329f1b7b0ecb444ebdc6b74ffa1457/black-23.1.0-cp311-cp311-macosx_10_16_universal2.whl", hash = "sha256:c1c476bc7b7d021321e7d93dc2cbd78ce103b84d5a4cf97ed535fbc0d6660648"}, - {url = "https://files.pythonhosted.org/packages/96/af/3361b34907efbfd9d55af453488be2282f831d98b7d201248b38d4c44346/black-23.1.0-cp37-cp37m-macosx_10_16_x86_64.whl", hash = "sha256:a8471939da5e824b891b25751955be52ee7f8a30a916d570a5ba8e0f2eb2ecad"}, - {url = "https://files.pythonhosted.org/packages/9a/ee/549e8be7f635cabcc3c7c3f2c3b27971dc32735155631b9ef2dcb1bd861f/black-23.1.0-cp311-cp311-macosx_10_16_arm64.whl", hash = "sha256:bfffba28dc52a58f04492181392ee380e95262af14ee01d4bc7bb1b1c6ca8d27"}, - {url = "https://files.pythonhosted.org/packages/a4/ec/934e89820289e6952922fa5965aec0e046ed65da168ffb0515af1e3364e1/black-23.1.0-cp38-cp38-macosx_10_16_arm64.whl", hash = "sha256:a59db0a2094d2259c554676403fa2fac3473ccf1354c1c63eccf7ae65aac8ab6"}, - {url = "https://files.pythonhosted.org/packages/ae/93/1e62fe94ab531bdc3f6cbbbd5b518727277bf40f695777b4097db5da2a38/black-23.1.0-cp39-cp39-macosx_10_16_x86_64.whl", hash = "sha256:c91dfc2c2a4e50df0026f88d2215e166616e0c80e86004d0003ece0488db2739"}, - {url = "https://files.pythonhosted.org/packages/b1/7e/c368e9c795387a01bc181d8acbfd178278cc9960c5e7ef1059222a4419f9/black-23.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:a436e7881d33acaf2536c46a454bb964a50eff59b21b51c6ccf5a40601fbef24"}, - {url = "https://files.pythonhosted.org/packages/b7/33/8e074fd8b86a1c8668f5493ed28929d87bdccb6aa68c2975b47a02f92287/black-23.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2a951cc83ab535d248c89f300eccbd625e80ab880fbcfb5ac8afb5f01a258ac9"}, - {url = "https://files.pythonhosted.org/packages/be/f9/11e401323cd5b4e53d138fc880564765466a86acd2d4b50d7c8cdd048c18/black-23.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e6663f91b6feca5d06f2ccd49a10f254f9298cc1f7f49c46e498a0771b507104"}, - {url = "https://files.pythonhosted.org/packages/c0/1d/8dac412cf5cc4120a438969a4fafefdc3de8fa13d411f317a9f9f1e268a4/black-23.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:0680d4380db3719ebcfb2613f34e86c8e6d15ffeabcf8ec59355c5e7b85bb555"}, - {url = "https://files.pythonhosted.org/packages/cf/fe/dda4b7eedb9d4dc46e306b814f7838cd9026907fdc889f75eb9f6d47d414/black-23.1.0-cp310-cp310-macosx_10_16_x86_64.whl", hash = "sha256:9880d7d419bb7e709b37e28deb5e68a49227713b623c72b2b931028ea65f619b"}, - {url = "https://files.pythonhosted.org/packages/d0/cb/0a38ffdafbb4b3f337adaf1b79aeaf4b8a21ed18835acad6349e46c78c80/black-23.1.0-cp310-cp310-macosx_10_16_arm64.whl", hash = "sha256:b6a92a41ee34b883b359998f0c8e6eb8e99803aa8bf3123bf2b2e6fec505a221"}, - {url = "https://files.pythonhosted.org/packages/dd/19/875b5006e40ea69a4120671f50317100b24732f2b877203722c91bc4eef3/black-23.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:162e37d49e93bd6eb6f1afc3e17a3d23a823042530c37c3c42eeeaf026f38468"}, - {url = "https://files.pythonhosted.org/packages/e6/0a/9a5fca4a2ca07d4dbc3b00445c9353f05ea182b000f68c9ad6ba1da87a47/black-23.1.0-cp38-cp38-macosx_10_16_universal2.whl", hash = "sha256:0052dba51dec07ed029ed61b18183942043e00008ec65d5028814afaab9a22fd"}, - {url = "https://files.pythonhosted.org/packages/f1/89/ccc28cb74a66c094b609295b009b5e0350c10b75661d2450eeed2f60ce37/black-23.1.0-cp311-cp311-macosx_10_16_x86_64.whl", hash = "sha256:382998821f58e5c8238d3166c492139573325287820963d2f7de4d518bd76958"}, +"black 23.3.0" = [ + {url = "https://files.pythonhosted.org/packages/06/1e/273d610249f0335afb1ddb03664a03223f4826e3d1a95170a0142cb19fb4/black-23.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:6b39abdfb402002b8a7d030ccc85cf5afff64ee90fa4c5aebc531e3ad0175ddb"}, + {url = "https://files.pythonhosted.org/packages/12/4b/99c71d1cf1353edd5aff2700b8960f92e9b805c9dab72639b67dbb449d3a/black-23.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:562bd3a70495facf56814293149e51aa1be9931567474993c7942ff7d3533961"}, + {url = "https://files.pythonhosted.org/packages/13/0a/ed8b66c299e896780e4528eed4018f5b084da3b9ba4ee48328550567d866/black-23.3.0-cp38-cp38-macosx_10_16_x86_64.whl", hash = "sha256:064101748afa12ad2291c2b91c960be28b817c0c7eaa35bec09cc63aa56493c5"}, + {url = "https://files.pythonhosted.org/packages/13/25/cfa06788d0a936f2445af88f13604b5bcd5c9d050db618c718e6ebe66f74/black-23.3.0-cp39-cp39-macosx_10_16_arm64.whl", hash = "sha256:3238f2aacf827d18d26db07524e44741233ae09a584273aa059066d644ca7b30"}, + {url = "https://files.pythonhosted.org/packages/21/14/d5a2bec5fb15f9118baab7123d344646fac0b1c6939d51c2b05259cd2d9c/black-23.3.0-cp38-cp38-macosx_10_16_universal2.whl", hash = "sha256:714290490c18fb0126baa0fca0a54ee795f7502b44177e1ce7624ba1c00f2331"}, + {url = "https://files.pythonhosted.org/packages/24/eb/2d2d2c27cb64cfd073896f62a952a802cd83cf943a692a2f278525b57ca9/black-23.3.0-cp37-cp37m-macosx_10_16_x86_64.whl", hash = "sha256:1d06691f1eb8de91cd1b322f21e3bfc9efe0c7ca1f0e1eb1db44ea367dff656b"}, + {url = "https://files.pythonhosted.org/packages/27/70/07aab2623cfd3789786f17e051487a41d5657258c7b1ef8f780512ffea9c/black-23.3.0-cp310-cp310-macosx_10_16_universal2.whl", hash = "sha256:67de8d0c209eb5b330cce2469503de11bca4085880d62f1628bd9972cc3366b9"}, + {url = "https://files.pythonhosted.org/packages/29/b1/b584fc863c155653963039664a592b3327b002405043b7e761b9b0212337/black-23.3.0-cp310-cp310-macosx_10_16_x86_64.whl", hash = "sha256:7c3eb7cea23904399866c55826b31c1f55bbcd3890ce22ff70466b907b6775c2"}, + {url = "https://files.pythonhosted.org/packages/3c/d7/85f3d79f9e543402de2244c4d117793f262149e404ea0168841613c33e07/black-23.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a150542a204124ed00683f0db1f5cf1c2aaaa9cc3495b7a3b5976fb136090ab"}, + {url = "https://files.pythonhosted.org/packages/3f/0d/81dd4194ce7057c199d4f28e4c2a885082d9d929e7a55c514b23784f7787/black-23.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:11c410f71b876f961d1de77b9699ad19f939094c3a677323f43d7a29855fe326"}, + {url = "https://files.pythonhosted.org/packages/49/36/15d2122f90ff1cd70f06892ebda777b650218cf84b56b5916a993dc1359a/black-23.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50cb33cac881766a5cd9913e10ff75b1e8eb71babf4c7104f2e9c52da1fb7de2"}, + {url = "https://files.pythonhosted.org/packages/49/d7/f3b7da6c772800f5375aeb050a3dcf682f0bbeb41d313c9c2820d0156e4e/black-23.3.0-cp39-cp39-macosx_10_16_x86_64.whl", hash = "sha256:92c543f6854c28a3c7f39f4d9b7694f9a6eb9d3c5e2ece488c327b6e7ea9b266"}, + {url = "https://files.pythonhosted.org/packages/69/49/7e1f0cf585b0d607aad3f971f95982cc4208fc77f92363d632d23021ee57/black-23.3.0-cp311-cp311-macosx_10_16_x86_64.whl", hash = "sha256:a6f6886c9869d4daae2d1715ce34a19bbc4b95006d20ed785ca00fa03cba312d"}, + {url = "https://files.pythonhosted.org/packages/6d/b4/0f13ab7f5e364795ff82b76b0f9a4c9c50afda6f1e2feeb8b03fdd7ec57d/black-23.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:32daa9783106c28815d05b724238e30718f34155653d4d6e125dc7daec8e260c"}, + {url = "https://files.pythonhosted.org/packages/ad/e7/4642b7f462381799393fbad894ba4b32db00870a797f0616c197b07129a9/black-23.3.0-py3-none-any.whl", hash = "sha256:ec751418022185b0c1bb7d7736e6933d40bbb14c14a0abcf9123d1b159f98dd4"}, + {url = "https://files.pythonhosted.org/packages/c0/53/42e312c17cfda5c8fc4b6b396a508218807a3fcbb963b318e49d3ddd11d5/black-23.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f3c333ea1dd6771b2d3777482429864f8e258899f6ff05826c3a4fcc5ce3f70"}, + {url = "https://files.pythonhosted.org/packages/ca/44/eb41edd3f558a6139f09eee052dead4a7a464e563b822ddf236f5a8ee286/black-23.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:e114420bf26b90d4b9daa597351337762b63039752bdf72bf361364c1aa05925"}, + {url = "https://files.pythonhosted.org/packages/ce/f4/2b0c6ac9e1f8584296747f66dd511898b4ebd51d6510dba118279bff53b6/black-23.3.0-cp38-cp38-macosx_10_16_arm64.whl", hash = "sha256:48f9d345675bb7fbc3dd85821b12487e1b9a75242028adad0333ce36ed2a6d27"}, + {url = "https://files.pythonhosted.org/packages/d1/6e/5810b6992ed70403124c67e8b3f62858a32b35405177553f1a78ed6b6e31/black-23.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:e198cf27888ad6f4ff331ca1c48ffc038848ea9f031a3b40ba36aced7e22f2c8"}, + {url = "https://files.pythonhosted.org/packages/d6/36/66370f5017b100225ec4950a60caeef60201a10080da57ddb24124453fba/black-23.3.0.tar.gz", hash = "sha256:1c7b8d606e728a41ea1ccbd7264677e494e87cf630e399262ced92d4a8dac940"}, + {url = "https://files.pythonhosted.org/packages/d7/6f/d3832960a3b646b333b7f0d80d336a3c123012e9d9d5dba4a622b2b6181d/black-23.3.0-cp311-cp311-macosx_10_16_arm64.whl", hash = "sha256:a8a968125d0a6a404842fa1bf0b349a568634f856aa08ffaff40ae0dfa52e7c6"}, + {url = "https://files.pythonhosted.org/packages/db/f4/7908f71cc71da08df1317a3619f002cbf91927fb5d3ffc7723905a2113f7/black-23.3.0-cp310-cp310-macosx_10_16_arm64.whl", hash = "sha256:0945e13506be58bf7db93ee5853243eb368ace1c08a24c65ce108986eac65915"}, + {url = "https://files.pythonhosted.org/packages/de/b4/76f152c5eb0be5471c22cd18380d31d188930377a1a57969073b89d6615d/black-23.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:35d1381d7a22cc5b2be2f72c7dfdae4072a3336060635718cc7e1ede24221d6c"}, + {url = "https://files.pythonhosted.org/packages/eb/a5/17b40bfd9b607b69fa726b0b3a473d14b093dcd5191ea1a1dd664eccfee3/black-23.3.0-cp311-cp311-macosx_10_16_universal2.whl", hash = "sha256:c7ab5790333c448903c4b721b59c0d80b11fe5e9803d8703e84dcb8da56fec1b"}, + {url = "https://files.pythonhosted.org/packages/fd/5b/fc2d7922c1a6bb49458d424b5be71d251f2d0dc97be9534e35d171bdc653/black-23.3.0-cp39-cp39-macosx_10_16_universal2.whl", hash = "sha256:f0bd2f4a58d6666500542b26354978218a9babcdc972722f4bf90779524515f3"}, ] "chispa 0.9.2" = [ {url = "https://files.pythonhosted.org/packages/03/8a/060df449d0a53a9c8480ccadfbcbba000a009d6e357f183a88adbe23f458/chispa-0.9.2.tar.gz", hash = "sha256:621ad2e64fd27e7372c7b90ab2d5ad1f8dd69b737a3421ba5b6f84b113a18b84"}, @@ -548,6 +592,25 @@ content_hash = "sha256:295dabb866e124f2b45815873bec9a5be9a1fe6893514bc26faa25675 "future 0.18.3" = [ {url = "https://files.pythonhosted.org/packages/8f/2e/cf6accf7415237d6faeeebdc7832023c90e0282aa16fd3263db0eb4715ec/future-0.18.3.tar.gz", hash = "sha256:34a17436ed1e96697a86f9de3d15a3b0be01d8bc8de9c1dffd59fb8234ed5307"}, ] +"gensim 4.1.0" = [ + {url = "https://files.pythonhosted.org/packages/0d/6d/06e6d747575caadd70af4f08f88cc0f8e9db34bb1f60cdf99ad38aaa5c8b/gensim-4.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:28f6e97d9a6bb32c44c53602d990a12e9fd199719c46f89f9221fe6cb6109bcd"}, + {url = "https://files.pythonhosted.org/packages/14/93/676d62ce2431fda03adf061f024ca4902e387db0098beb65de0f31d9a68a/gensim-4.1.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:504c129224953f75940402143ecce1383be0784ea89912042c7b7e1e1107cbbc"}, + {url = "https://files.pythonhosted.org/packages/2d/b3/655ea3d7c3c6601913ed8fb8d8a6db14fee2120681eb5175003f184a18d2/gensim-4.1.0-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:068a4bc698f3b2844609dcb8f693b033d81c9d03f82ffd56d80a62fecc9347e9"}, + {url = "https://files.pythonhosted.org/packages/2d/f5/3c1b8fbfdb41c787bf0b8b26c9170084b7e9b09af0dee0526bffcfb66790/gensim-4.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:eb577ec45ef72cc213e8e1301b251a6b5798e51f5085f88a0b5527d281acbcec"}, + {url = "https://files.pythonhosted.org/packages/33/88/24b7eb1d3a1db62eac8ef34a2825f72ab6cd21c7ef9f9c0a0cb77e657f82/gensim-4.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:382c821e256040c9763d8ae356d851bbc58590bc45aa8c70ee067f037349e3b3"}, + {url = "https://files.pythonhosted.org/packages/39/2c/5eeb200d2f9af2caca2b2dfdb86cadcc9cf978b66bc8568fe2ea6714f2ef/gensim-4.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9a37a941d3b618520225f0f3e1ea2e2c1971385cca1d740db707c10421a7319"}, + {url = "https://files.pythonhosted.org/packages/3c/1a/e9474318d49d396055c45e436d85428c0aefd5d6801f247431bdc1bdeb0f/gensim-4.1.0-cp36-cp36m-win_amd64.whl", hash = "sha256:093415811d56af0f70cd8ed51ef21f927937dc59ed7b341a5b5ebb887906851a"}, + {url = "https://files.pythonhosted.org/packages/55/31/5a8f52f29232d8a6aa5d8fc75531a029bbc24a78869a0a0a4566b9f8c13a/gensim-4.1.0.tar.gz", hash = "sha256:0b09983048a97c7915ab50500bc53eeec438d26366041598709ec156db3eef1f"}, + {url = "https://files.pythonhosted.org/packages/7b/6e/307f8957ce37449b5135044de8d5cb25ad72a9a5ee56578f3110b008f300/gensim-4.1.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f6c4c8bf0c07d1f05f1c718fbc88e85eced222ceba03216eb6e5763098e8c2da"}, + {url = "https://files.pythonhosted.org/packages/99/b8/288bbf52a661647e8728626baf2de88b38cb79875610d858bac81b7552a8/gensim-4.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:905617489a72bd51c7cf516db0defb4d6c04d500a0b054976e816df7f7397a90"}, + {url = "https://files.pythonhosted.org/packages/a8/0a/b4e6b17557823e4d916789c4b8871452f9e24848bfd25fb4675f319a3852/gensim-4.1.0-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:96a4bae3e7523a9e28049e0b85b9e8525924f6ae9313467734556afecf120bb9"}, + {url = "https://files.pythonhosted.org/packages/cb/a1/fa7cd9adc106dba2d18273089cfd60d0f5a16ea3b319dafb957260dfb03f/gensim-4.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d2dee416383591b4fef9cdf3810f9818a2de4af5e588133822e2629a0ed2dc79"}, + {url = "https://files.pythonhosted.org/packages/d8/c5/0f77e384a056634410c165e49ca1cfa6d48a7a6a8be976373e3bca39687b/gensim-4.1.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9866744a57e6001dd8115328bc7e306fedc746e37e8d1720f3811b5e76359e67"}, + {url = "https://files.pythonhosted.org/packages/e0/ad/4cf91087ad50f95ce9c8fa9845c25eeaebaa375600b698a0660a296f710b/gensim-4.1.0-cp36-cp36m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:932a3d372d4b795ce56efbec8f7b074e8c97a043e2f846b2f721a76c5cc80cc3"}, + {url = "https://files.pythonhosted.org/packages/f1/a0/adec6df0a00b22d22b5842da47fe209fdb2a7b3aa946bf042e708107904a/gensim-4.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:344ca542b7d8aca249ac3b4ad952f5c360341d4656cb0d25f9893a68ed4fd473"}, + {url = "https://files.pythonhosted.org/packages/fd/3c/91b345adf405cbc00fb1c90be5477ccc12b3d291f24b2572991d95b93041/gensim-4.1.0-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:57c3b947b352e0637d810a5651db568e61844617ff07fec5fb56d55275b04578"}, + {url = "https://files.pythonhosted.org/packages/fe/55/96f90e06d6d16d33a755b915fee34a0c67b7ee313a4f41f5af4e6a656822/gensim-4.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:1f3f37b2836d00b92e75f33695ee0c901286b71ced325e3e7510918e06b8144e"}, +] "httplib2 0.22.0" = [ {url = "https://files.pythonhosted.org/packages/3d/ad/2371116b22d616c194aa25ec410c9c6c37f23599dcd590502b74db197584/httplib2-0.22.0.tar.gz", hash = "sha256:d7a10bc5ef5ab08322488bde8c726eeee5c8618723fdb399597ec58f3d82df81"}, {url = "https://files.pythonhosted.org/packages/a8/6c/d2fbdaaa5959339d53ba38e94c123e4e84b8fbc4b84beb0e70d7c1608486/httplib2-0.22.0-py3-none-any.whl", hash = "sha256:14ae0a53c1ba8f3d37e9e27cf37eabb0fb9980f435ba405d546948b009dd64dc"}, @@ -580,6 +643,10 @@ content_hash = "sha256:295dabb866e124f2b45815873bec9a5be9a1fe6893514bc26faa25675 {url = "https://files.pythonhosted.org/packages/7a/ff/75c28576a1d900e87eb6335b063fab47a8ef3c8b4d88524c4bf78f670cce/Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, {url = "https://files.pythonhosted.org/packages/bc/c3/f068337a370801f372f2f8f6bad74a5c140f6fda3d9de154052708dd3c65/Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, ] +"joblib 1.2.0" = [ + {url = "https://files.pythonhosted.org/packages/45/dd/a5435a6902d6315241c48a5343e6e6675b007e05d3738ed97a7a47864e53/joblib-1.2.0.tar.gz", hash = "sha256:e1cee4a79e4af22881164f218d4311f60074197fb707e082e803b61f6d137018"}, + {url = "https://files.pythonhosted.org/packages/91/d4/3b4c8e5a30604df4c7518c562d4bf0502f2fa29221459226e140cf846512/joblib-1.2.0-py3-none-any.whl", hash = "sha256:091138ed78f800342968c523bdde947e7a305b8594b910a0fea2ab83c3c6d385"}, +] "markupsafe 2.1.2" = [ {url = "https://files.pythonhosted.org/packages/02/2c/18d55e5df6a9ea33709d6c33e08cb2e07d39e20ad05d8c6fbf9c9bcafd54/MarkupSafe-2.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:835fb5e38fd89328e9c81067fd642b3593c33e1e17e2fdbf77f5676abb14a156"}, {url = "https://files.pythonhosted.org/packages/04/cf/9464c3c41b7cdb8df660cda75676697e7fb49ce1be7691a1162fc88da078/MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:085fd3201e7b12809f9e6e9bc1e5c96a368c8523fad5afb02afe3c051ae4afcc"}, @@ -656,6 +723,9 @@ content_hash = "sha256:295dabb866e124f2b45815873bec9a5be9a1fe6893514bc26faa25675 {url = "https://files.pythonhosted.org/packages/27/fc/fd86d6e55d4413b71f5d0c24233f81e1b621404bcceb6ca2de768cb1f8c0/ng_nx-0.1.8-py3-none-any.whl", hash = "sha256:cd221fcf97b2faeaed12aa1cf4b9f982877ad1018a6ccb2b6e81f5a865dabca0"}, {url = "https://files.pythonhosted.org/packages/99/82/fb70b87fc591f4cdf904e676f0652c9986fc617b84ec7d66c1fc28199b9d/ng-nx-0.1.8.tar.gz", hash = "sha256:a3a0ad3ef56c49c1ef68dcb503a279a9d8345ee103ac8066d23985ac0df933c5"}, ] +"node2vec 0.4.3" = [ + {url = "https://files.pythonhosted.org/packages/87/c4/8e859a1099d78dbb00b25c6832b8ee9fe11110cc7f2f3a6a4bd37ada3185/node2vec-0.4.3.tar.gz", hash = "sha256:7107757177b0c7730215c4b40524ca63420cf2c23e0af290a45c6e3bc0dc24e2"}, +] "numpy 1.21.6" = [ {url = "https://files.pythonhosted.org/packages/06/78/b184f13f5461812a17a90b380d70a93fa3532460f0af9d72b0d93d8bc4ff/numpy-1.21.6-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:67c261d6c0a9981820c3a149d255a76918278a6b03b6a036800359aba1256d46"}, {url = "https://files.pythonhosted.org/packages/0d/21/036363516c06737135ee58741e9c0af4899348ce3c5f5e04379240edd090/numpy-1.21.6-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:357768c2e4451ac241465157a3e929b265dfac85d9214074985b1786244f2ef3"}, @@ -829,6 +899,10 @@ content_hash = "sha256:295dabb866e124f2b45815873bec9a5be9a1fe6893514bc26faa25675 {url = "https://files.pythonhosted.org/packages/71/39/171f1c67cd00715f190ba0b100d606d440a28c93c7714febeca8b79af85e/six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, {url = "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, ] +"smart-open 6.3.0" = [ + {url = "https://files.pythonhosted.org/packages/47/80/c2d1bdd36c6b64ae566d9a29724291510e4f3796ce99639d3c2999286284/smart_open-6.3.0-py3-none-any.whl", hash = "sha256:b4c9ae193ad6d3e7add50944b86afa0d150bd821ab8ec21edb26d9a06b66f6a8"}, + {url = "https://files.pythonhosted.org/packages/b0/2b/ebc6d835bb354eb6d7f5f560be53dc746dab84d0958c363a082bfdf1e862/smart_open-6.3.0.tar.gz", hash = "sha256:d5238825fe9a9340645fac3d75b287c08fbb99fb2b422477de781c9f5f09e019"}, +] "tokenize-rt 5.0.0" = [ {url = "https://files.pythonhosted.org/packages/40/01/fb40ea8c465f680bf7aa3f5bee39c62ba8b7f52c38048c27aa95aff4f779/tokenize_rt-5.0.0.tar.gz", hash = "sha256:3160bc0c3e8491312d0485171dea861fc160a240f5f5766b72a1165408d10740"}, {url = "https://files.pythonhosted.org/packages/8d/12/4c7495f25b4c9131706f3aaffb185d4de32c02a6ee49d875e929c5b7c919/tokenize_rt-5.0.0-py2.py3-none-any.whl", hash = "sha256:c67772c662c6b3dc65edf66808577968fb10badfc2042e3027196bed4daf9e5a"}, @@ -837,6 +911,10 @@ content_hash = "sha256:295dabb866e124f2b45815873bec9a5be9a1fe6893514bc26faa25675 {url = "https://files.pythonhosted.org/packages/97/75/10a9ebee3fd790d20926a90a2547f0bf78f371b2f13aa822c759680ca7b9/tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, {url = "https://files.pythonhosted.org/packages/c0/3f/d7af728f075fb08564c5949a9c95e44352e23dee646869fa104a3b2060a3/tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] +"tqdm 4.65.0" = [ + {url = "https://files.pythonhosted.org/packages/3d/78/81191f56abb7d3d56963337dbdff6aa4f55805c8afd8bad64b0a34199e9b/tqdm-4.65.0.tar.gz", hash = "sha256:1871fb68a86b8fb3b59ca4cdd3dcccbc7e6d613eeed31f4c332531977b89beb5"}, + {url = "https://files.pythonhosted.org/packages/e6/02/a2cff6306177ae6bc73bc0665065de51dfb3b9db7373e122e2735faf0d97/tqdm-4.65.0-py3-none-any.whl", hash = "sha256:c4f53a17fe37e132815abceec022631be8ffe1b9381c2e6e30aa70edc99e9671"}, +] "traitlets 5.9.0" = [ {url = "https://files.pythonhosted.org/packages/39/c3/205e88f02959712b62008502952707313640369144a7fded4cbc61f48321/traitlets-5.9.0.tar.gz", hash = "sha256:f6cde21a9c68cf756af02035f72d5a723bf607e862e7be33ece505abf4a3bad9"}, {url = "https://files.pythonhosted.org/packages/77/75/c28e9ef7abec2b7e9ff35aea3e0be6c1aceaf7873c26c95ae1f0d594de71/traitlets-5.9.0-py3-none-any.whl", hash = "sha256:9e6ec080259b9a5940c797d58b613b5e31441c2257b87c2e795c5228ae80d2d8"}, diff --git a/pyproject.toml b/pyproject.toml index fb84877..26116fb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ shell = """ docker-compose -f 'tests/integration/setup/docker-compose.yaml' up -d --remove-orphans echo "Waiting for NebulaGraph to start..." - sleep 10 + sleep 20 for i in {1..50}; do docker-compose -f 'tests/integration/setup/docker-compose.yaml' ps | grep "unhealthy\\|starting" | wc -l | grep -q 0 && break; echo Waiting for another 5 sec; sleep 5; done echo "NebulaGraph is up and running, removing console container" docker-compose -f 'tests/integration/setup/docker-compose.yaml' stop console @@ -111,7 +111,7 @@ filter_files = true [project] name = "ng_ai" -version = "0.2.10.0" +version = "0.2.10.1" description = "NebulaGraph AI Suite" authors = [ {name = "Wey Gu", email = "weyl.gu@gmail.com"}, @@ -144,11 +144,14 @@ build-backend = "pdm.pep517.api" [project.optional-dependencies] # pyspark 2.4.8 doesn't work with python 3.8+, so we use 3.2.3 spark = ["pyspark>=3.2.3"] +# gensim 4.2.0 doesn't work on m1 mac, 4.3.x requires python 3.8+, so we use 4.1.0 networkx = [ "ng_nx>=0.1.8", "pandas>=1.3.5", "numpy>=1.21.6", "scipy>=1.7.3", + "gensim==4.1.0", + "node2vec>=0.4.3", ] all = ["ng_ai[spark,networkx]"] lint = [ diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py new file mode 100644 index 0000000..77a7c51 --- /dev/null +++ b/tests/integration/conftest.py @@ -0,0 +1,104 @@ +import os +import subprocess +from time import sleep + +import pytest + + +@pytest.fixture(scope="session", autouse=True) +def prepare_data(): + print("Setup data...") + subprocess.run( + "docker run --rm --network setup_ngai-net vesoft/nebula-console:v3" + ' -addr graphd -port 9669 -u root -p nebula -e " ' + 'DROP SPACE basketballplayer;" ', + shell=True, + check=True, + ) + sleep(4) + result = subprocess.run( + "docker run --rm --network setup_ngai-net vesoft/nebula-console:v3" + ' -addr graphd -port 9669 -u root -p nebula -e " ' + ':play basketballplayer" ', + shell=True, + check=True, + capture_output=True, + timeout=300, + ) + print(f":play basketballplayer: {result}") + + subprocess.run( + "docker run --rm --network setup_ngai-net vesoft/nebula-console:v3" + ' -addr graphd -port 9669 -u root -p nebula -e " ' + 'USE basketballplayer; SUBMIT JOB STATS" ', + shell=True, + check=True, + capture_output=True, + ) + + result = subprocess.run( + "docker run --rm --network setup_ngai-net vesoft/nebula-console:v3" + ' -addr graphd -port 9669 -u root -p nebula -e " ' + "USE basketballplayer; " + "CREATE TAG IF NOT EXISTS label_propagation " + '(cluster_id string NOT NULL);" ', + shell=True, + check=True, + capture_output=True, + ) + sleep(4) + assert ( + b"ERROR" not in result.stdout + ), f"ERROR during create tag: {result.stdout.decode('utf-8')}" + + result = subprocess.run( + "docker run --rm --network setup_ngai-net vesoft/nebula-console:v3" + ' -addr graphd -port 9669 -u root -p nebula -e " ' + "USE basketballplayer; " + 'SHOW STATS;" ', + shell=True, + check=True, + capture_output=True, + ) + print(f"Show stats:\n{result.stdout.decode('utf-8')}") + + os.system("mkdir -p tests/integration/setup/run") + os.system("rm -rf tests/integration/setup/run/*") + + os.system( + "cp -r tests/integration/spark_engine_cases/* tests/integration/setup/run/" + ) + + # create louvain schema for networkx engine writer test + # RUN: + # CREATE TAG IF NOT EXISTS louvain ( + # cluster_id string NOT NULL + # ); + + result = subprocess.run( + "docker run --rm --network setup_ngai-net vesoft/nebula-console:v3" + ' -addr graphd -port 9669 -u root -p nebula -e " ' + "USE basketballplayer; " + "CREATE TAG IF NOT EXISTS louvain " + '(cluster_id string NOT NULL);" ', + shell=True, + check=True, + capture_output=True, + ) + sleep(10) + # check the schema existence with DESC TAG louvain: + result = subprocess.run( + "docker run --rm --network setup_ngai-net vesoft/nebula-console:v3" + ' -addr graphd -port 9669 -u root -p nebula -e " ' + "USE basketballplayer; " + 'DESC TAG louvain;" ', + shell=True, + check=True, + capture_output=True, + ) + print(f"DESC TAG louvain:\n{result.stdout.decode('utf-8')}") + assert ( + b"ERROR" not in result.stdout + ), f"ERROR during create tag: {result.stdout.decode('utf-8')}" + + print("Setup data done...") diff --git a/tests/integration/test_e2e_networkx_engine.py b/tests/integration/test_e2e_networkx_engine.py new file mode 100644 index 0000000..b29bfaa --- /dev/null +++ b/tests/integration/test_e2e_networkx_engine.py @@ -0,0 +1,115 @@ +import subprocess + +from nebula3.Config import Config +from nebula3.gclient.net import ConnectionPool + + +def test_networkx_engine_query_reader(): + import networkx as nx + + from ng_ai import NebulaReader + from ng_ai.config import NebulaGraphConfig + + # read data with spark engine, query mode + config_dict = { + "graphd_hosts": "127.0.0.1:39669", + "user": "root", + "password": "nebula", + "space": "basketballplayer", + } + config = NebulaGraphConfig(**config_dict) + reader = NebulaReader(engine="nebula", config=config) + reader.query(edges=["follow", "serve"], props=[["degree"], []]) + g = reader.read() + assert isinstance(g.data, nx.MultiDiGraph) + g.show(10) + + +def test_networkx_engine_algo(): + """ + Test networkx engine with all algorithms + """ + import networkx as nx + + from ng_ai import NebulaReader + from ng_ai.config import NebulaGraphConfig + + # read data with spark engine, query mode + config_dict = { + "graphd_hosts": "127.0.0.1:39669", + "user": "root", + "password": "nebula", + "space": "basketballplayer", + } + config = NebulaGraphConfig(**config_dict) + reader = NebulaReader(engine="nebula", config=config) + reader.query(edges=["follow", "serve"], props=[["degree"], []]) + g = reader.read() + + for algo_name in g.algo.get_all_algo(): + algo_func = getattr(g.algo, algo_name) + print(f"Running {algo_name}...") + result = algo_func() + if hasattr(result, "__iter__"): + for x in result: + print(x) + else: + print(result) + print("=" * 20) + + +def test_networkx_engine_writer(): + from ng_ai import NebulaReader, NebulaWriter + from ng_ai.config import NebulaGraphConfig + + # read data with spark engine, query mode + config_dict = { + "graphd_hosts": "127.0.0.1:39669", + "user": "root", + "password": "nebula", + "space": "basketballplayer", + } + config = NebulaGraphConfig(**config_dict) + reader = NebulaReader(engine="nebula", config=config) + reader.query(edges=["follow", "serve"], props=[["degree"], []]) + g = reader.read() + + graph_result = g.algo.louvain() + + writer = NebulaWriter( + data=graph_result, + sink="nebulagraph_vertex", + config=config, + engine="nebula", + ) + + # properties to write + properties = ["cluster_id"] + + writer.set_options( + tag="louvain", + properties=properties, + batch_size=256, + write_mode="insert", + ) + # write back to NebulaGraph + writer.write() + + nebula_config = Config() + connection_pool = ConnectionPool() + connection_pool.init([("127.0.0.1", 39669)], nebula_config) + + with connection_pool.session_context("root", "nebula") as session: + session.execute("USE basketballplayer") + result = session.execute( + "MATCH (v:louvain) RETURN id(v), v.louvain.cluster_id LIMIT 10;" + ) + print(result) + connection_pool.close() + + assert result.is_succeeded(), f"ERROR during query NebulaGraph: {result}" + assert ( + not result.is_empty() + ), f"louvain not written to NebulaGraph result: {result}" + + print(f"Label propagation result:\n{result}") diff --git a/tests/integration/test_e2e_spark_engine.py b/tests/integration/test_e2e_spark_engine.py index 44d4974..44066a5 100644 --- a/tests/integration/test_e2e_spark_engine.py +++ b/tests/integration/test_e2e_spark_engine.py @@ -1,79 +1,9 @@ -import os import subprocess -from time import sleep -import pytest from nebula3.Config import Config from nebula3.gclient.net import ConnectionPool -@pytest.fixture(scope="session", autouse=True) -def prepare_data(): - print("Setup data...") - subprocess.run( - "docker run --rm --network setup_ngai-net vesoft/nebula-console:v3" - ' -addr graphd -port 9669 -u root -p nebula -e " ' - 'DROP SPACE basketballplayer;" ', - shell=True, - check=True, - ) - sleep(4) - result = subprocess.run( - "docker run --rm --network setup_ngai-net vesoft/nebula-console:v3" - ' -addr graphd -port 9669 -u root -p nebula -e " ' - ':play basketballplayer" ', - shell=True, - check=True, - capture_output=True, - timeout=300, - ) - print(f":play basketballplayer: {result}") - - subprocess.run( - "docker run --rm --network setup_ngai-net vesoft/nebula-console:v3" - ' -addr graphd -port 9669 -u root -p nebula -e " ' - 'USE basketballplayer; SUBMIT JOB STATS" ', - shell=True, - check=True, - capture_output=True, - ) - - result = subprocess.run( - "docker run --rm --network setup_ngai-net vesoft/nebula-console:v3" - ' -addr graphd -port 9669 -u root -p nebula -e " ' - "USE basketballplayer; " - "CREATE TAG IF NOT EXISTS label_propagation " - '(cluster_id string NOT NULL);" ', - shell=True, - check=True, - capture_output=True, - ) - sleep(4) - assert ( - b"ERROR" not in result.stdout - ), f"ERROR during create tag: {result.stdout.decode('utf-8')}" - - result = subprocess.run( - "docker run --rm --network setup_ngai-net vesoft/nebula-console:v3" - ' -addr graphd -port 9669 -u root -p nebula -e " ' - "USE basketballplayer; " - 'SHOW STATS;" ', - shell=True, - check=True, - capture_output=True, - ) - print(f"Show stats:\n{result.stdout.decode('utf-8')}") - - os.system("mkdir -p tests/integration/setup/run") - os.system("rm -rf tests/integration/setup/run/*") - - os.system( - "cp -r tests/integration/spark_engine_cases/* tests/integration/setup/run/" - ) - sleep(10) - print("Setup data done...") - - def test_scan_reader_spark_engine(): """ Just call: diff --git a/tests/unit/test_nebula_algo.py b/tests/unit/spark_engine/test_nebula_algo.py similarity index 100% rename from tests/unit/test_nebula_algo.py rename to tests/unit/spark_engine/test_nebula_algo.py diff --git a/tests/unit/test_nebula_data.py b/tests/unit/spark_engine/test_nebula_data.py similarity index 100% rename from tests/unit/test_nebula_data.py rename to tests/unit/spark_engine/test_nebula_data.py diff --git a/tests/unit/test_nebula_reader.py b/tests/unit/spark_engine/test_nebula_reader.py similarity index 100% rename from tests/unit/test_nebula_reader.py rename to tests/unit/spark_engine/test_nebula_reader.py diff --git a/tests/unit/test_nebula_writer.py b/tests/unit/spark_engine/test_nebula_writer.py similarity index 100% rename from tests/unit/test_nebula_writer.py rename to tests/unit/spark_engine/test_nebula_writer.py diff --git a/udf/ng_ai.cpp b/udf/ng_ai.cpp index 74a5b49..fd06986 100644 --- a/udf/ng_ai.cpp +++ b/udf/ng_ai.cpp @@ -155,7 +155,7 @@ nebula::Value ng_ai::call_ng_ai_api( return nebula::Value(response); } // validate name exists in algo_context, and the value is in ["label_propagation", "louvain", - // "k_core", "degree_statics", "betweenness_centrality", "coefficient_centrality", "bfs", "hanp", + // "k_core", "degree_statics", "betweenness_centrality", "clustering_coefficient", "bfs", "hanp", // "jaccard", "strong_connected_components", "triangle_coun", "pagerank"] if not valid, return response MAP // with error message: "Invalid algo_name: {algo_name}" auto algo_name = algo_context.kvs.find("name"); @@ -168,7 +168,7 @@ nebula::Value ng_ai::call_ng_ai_api( auto algo_name_value = algo_name->second.getStr(); if (algo_name_value != "label_propagation" && algo_name_value != "louvain" && algo_name_value != "k_core" && algo_name_value != "degree_statics" && - algo_name_value != "betweenness_centrality" && algo_name_value != "coefficient_centrality" && + algo_name_value != "betweenness_centrality" && algo_name_value != "clustering_coefficient" && algo_name_value != "bfs" && algo_name_value != "hanp" && algo_name_value != "jaccard" && algo_name_value != "strong_connected_components" && algo_name_value != "triangle_coun" && algo_name_value != "pagerank") { @@ -177,7 +177,7 @@ nebula::Value ng_ai::call_ng_ai_api( response.kvs.emplace("algo_name", algo_name_value); response.kvs.emplace("hint", nebula::Value( "Valid algo_name: label_propagation, louvain, k_core, degree_statics, betweenness_centrality, " - "coefficient_centrality, bfs, hanp, jaccard, strong_connected_components, triangle_coun, " + "clustering_coefficient, bfs, hanp, jaccard, strong_connected_components, triangle_coun, " "pagerank")); return nebula::Value(response); }