diff --git a/.ci/test.py b/.ci/test.py index 946210143..b7edaf54f 100644 --- a/.ci/test.py +++ b/.ci/test.py @@ -1,15 +1,14 @@ -from pathlib import Path -import subprocess -from pprint import pprint -from collections import namedtuple -from typing import Generator - +import json import logging +import os +import shlex import shutil +import subprocess import sys import tempfile -import shlex -import os +from collections import namedtuple +from pathlib import Path, PosixPath +from typing import Generator, List logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) @@ -18,8 +17,8 @@ '.ci', '.git', '.github', + 'archived', 'lightning', - 'feeadjuster' ] global_dependencies = [ @@ -50,13 +49,15 @@ def enumerate_plugins(basedir: Path) -> Generator[Plugin, None, None]: pip_pytest = [ x for x in plugins if (x / Path('requirements.txt')).exists() ] + print(f'Pip plugins: {", ".join([p.name for p in sorted(pip_pytest)])}') poetry_pytest = [ x for x in plugins if (x / Path("pyproject.toml")).exists() ] - print(poetry_pytest) + print(f'Poetry plugins: {", ".join([p.name for p in sorted(poetry_pytest)])}') other_plugins = [x for x in plugins if x not in pip_pytest and x not in poetry_pytest] + print(f'Other plugins: {", ".join([p.name for p in sorted(other_plugins)])}') for p in sorted(pip_pytest): yield Plugin( @@ -227,20 +228,24 @@ def install_pyln_testing(pip_path): stderr=subprocess.STDOUT, ) -def run_one(p: Plugin) -> bool: - print("Running tests on plugin {p.name}".format(p=p)) - - testfiles = [ +def get_testfiles(p: Plugin) -> List[PosixPath]: + return [ x for x in p.path.iterdir() if (x.is_dir() and x.name == 'tests') or (x.name.startswith("test_") and x.name.endswith('.py')) ] - if len(testfiles) == 0: +def has_testfiles(p: Plugin) -> bool: + return len(get_testfiles(p)) > 0 + +def run_one(p: Plugin) -> bool: + print("Running tests on plugin {p.name}".format(p=p)) + + if not has_testfiles(p): print("No test files found, skipping plugin {p.name}".format(p=p)) return True - print("Found {ctestfiles} test files, creating virtualenv and running tests".format(ctestfiles=len(testfiles))) + print("Found {ctestfiles} test files, creating virtualenv and running tests".format(ctestfiles=len(get_testfiles(p)))) print("##[group]{p.name}".format(p=p)) # Create a virtual env @@ -296,7 +301,7 @@ def run_one(p: Plugin) -> bool: print("##[endgroup]") -def run_all(args): +def run_all(workflow, update_badges, plugin_names): root_path = subprocess.check_output([ 'git', 'rev-parse', @@ -306,8 +311,8 @@ def run_all(args): root = Path(root_path) plugins = list(enumerate_plugins(root)) - if args != []: - plugins = [p for p in plugins if p.name in args] + if plugin_names != []: + plugins = [p for p in plugins if p.name in plugin_names] print("Testing the following plugins: {names}".format(names=[p.name for p in plugins])) else: print("Testing all plugins in {root}".format(root=root)) @@ -315,11 +320,73 @@ def run_all(args): results = [(p, run_one(p)) for p in plugins] success = all([t[1] for t in results]) + if sys.version_info[0:2] == (3, 12) and update_badges: + push_badges_data(collect_badges_data(results, success), workflow) + if not success: print("The following tests failed:") for t in filter(lambda t: not t[1], results): print(" - {p.name} ({p.path})".format(p=t[0])) sys.exit(1) + else: + print("All tests passed.") + + +def collect_badges_data(results, success): + badges_data = {} + for t in results: + p = t[0] + if has_testfiles(p): + if success or t[1]: + badges_data[p.name] = True + else: + badges_data[p.name] = False + return badges_data + + +def configure_git(): + subprocess.run(["git", "config", "--global", "user.email", '"lightningd@plugins.repo"']) + subprocess.run(["git", "config", "--global", "user.name", '"lightningd"']) + + +def update_and_commit_badge(plugin_name, passed, workflow): + json_data = { "schemaVersion": 1, "label": "", "message": " ✔ ", "color": "green" } + if not passed: + json_data.update({"message": "✗", "color": "red"}) + + filename = os.path.join("badges", f"{plugin_name}_{workflow}.json") + with open(filename, "w") as file: + file.write(json.dumps(json_data)) + + output = subprocess.check_output(["git", "add", "-v", filename]).decode("utf-8") + if output != "": + subprocess.run(["git", "commit", "-m", f'Update {plugin_name} badge to {"passed" if passed else "failed"} ({workflow})']) + return True + return False + + +def push_badges_data(data, workflow): + print("Pushing badge data...") + configure_git() + subprocess.run(["git", "fetch"]) + subprocess.run(["git", "checkout", "badges"]) + + any_changes = False + for plugin_name, passed in data.items(): + any_changes |= update_and_commit_badge(plugin_name, passed, workflow) + + if any_changes: + subprocess.run(["git", "push", "origin", "badges"]) + print("Done.") + if __name__ == "__main__": - run_all(sys.argv[1:]) + import argparse + + parser = argparse.ArgumentParser(description='Plugins test script') + parser.add_argument("workflow", type=str, help="Name of the GitHub workflow") + parser.add_argument("--update-badges", action='store_true', help="Whether badges data should be updated") + parser.add_argument("plugins", nargs="*", default=[], help="List of plugins") + args = parser.parse_args() + + run_all(args.workflow, args.update_badges, args.plugins) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e8d8c8ed4..3b3a9cf80 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -98,7 +98,14 @@ jobs: # Run the tests: In the case of a 'pull_request' event only the plugins in `plugin_dirs` # are going to be tested; otherwise ('push' event) we test all plugins. - python3 .ci/test.py $(echo "$plugin_dirs") + + update_badges='' + if [[ "${{ github.event_name }}" == 'push' && "${{ github.ref }}" == 'refs/heads/master' ]] || [[ "${{ github.event_name }}" == 'schedule' ]] + then + update_badges='--update-badges' + fi + + python3 .ci/test.py main $update_badges $(echo "$plugin_dirs") gather: # A dummy task that depends on the full matrix of tests, and diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 8abfde298..ce3859ea4 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -91,7 +91,7 @@ jobs: export CLN_PATH=${{ github.workspace }}/lightning pip3 install --upgrade pip pip3 install --user -U virtualenv pip > /dev/null - python3 .ci/test.py + python3 .ci/test.py nightly --update-badges gather: # A dummy task that depends on the full matrix of tests, and diff --git a/README.md b/README.md index a0ef9d566..320f5d646 100644 --- a/README.md +++ b/README.md @@ -7,43 +7,44 @@ Community curated plugins for Core-Lightning. ## Available plugins -| Name | Short description | -| ------------------------------------ | ------------------------------------------------------------------------------------------- | -| [backup][backup] | A simple and reliable backup plugin | -| [boltz-channel-creation][boltz] | A Core-Lightning plugin for Boltz Channel Creation Swaps | -| [btcli4j][btcli4j] | A Bitcoin Backend to enable safely the pruning mode, and support also rest APIs. | -| [circular][circular] | A smart rebalancing plugin for Core Lightning routing nodes | -| [clearnet][clearnet] | A plugin that can be used to enforce clearnet connections when possible | -| [cln-ntfy][cln-ntfy] | Core Lightning plugin for sending `ntfy` alerts. | -| [csvexportpays][csvexportpays] | A plugin that exports all payments to a CSV file | -| [currencyrate][currencyrate] | A plugin to convert other currencies to BTC using web requests | -| [donations][donations] | A simple donations page to accept donations from the web | -| [event-websocket][event-websocket] | Exposes notifications over a Websocket | -| [feeadjuster][feeadjuster] | Dynamic fees to keep your channels more balanced | -| [go-lnmetrics.reporter][reporter] | Collect and report of the lightning node metrics | -| [graphql][graphql] | Exposes the Core-Lightning API over [graphql][graphql-spec] | -| [holdinvoice][holdinvoice] | Holds htlcs for invoices until settle or cancel is called (aka Hodlinvoices) via RPC/GRPC | -| [invoice-queue][invoice-queue] | Listen to lightning invoices from multiple nodes and send to a redis queue for processing | -| [lightning-qt][lightning-qt] | A bitcoin-qt-like GUI for lightningd | -| [listmempoolfunds][listmempoolfunds] | Track unconfirmed wallet deposits | -| [monitor][monitor] | helps you analyze the health of your peers and channels | -| [nloop][nloop] | Generic Lightning Loop for boltz | -| [paythrough][paythrough] | Pay an invoice through a specific channel, regardless of better routes | -| [persistent-channels][pers-chans] | Maintains a number of channels to peers | -| [poncho][poncho] | Turns CLN into a [hosted channels][blip12] provider | -| [pruning][pruning] | This plugin manages pruning of bitcoind such that it can always sync | -| [python-teos][python-teos] | The Eye of Satoshi - Lightning Watchtower | -| [rebalance][rebalance] | Keeps your channels balanced | -| [reckless][reckless] | An **experimental** plugin manager (search/install plugins) | -| [sauron][sauron] | A Bitcoin backend relying on [Esplora][esplora]'s API | -| [sitzprobe][sitzprobe] | A Lightning Network payment rehearsal utility | -| [sling][sling] | Rebalance your channels with smart rules and built-in background tasks | -| [sparko][sparko] | RPC over HTTP with fine-grained permissions, SSE and spark-wallet support | -| [summars][summars] | Print configurable summary of node, channels and optionally forwards, invoices, payments | -| [trustedcoin][trustedcoin] | Replace your Bitcoin Core with data from public block explorers | -| [watchtower][watchtower-client] | Watchtower client for The Eye of Satoshi | -| [webhook][webhook] | Dispatches webhooks based from [event notifications][event-notifications] | -| [zmq][zmq] | Publishes notifications via [ZeroMQ][zmq-home] to configured endpoints | +| Name | Short description | Main ![GitHub Release](https://img.shields.io/github/v/release/ElementsProject/lightning?label=%20&color=393D47) | Nigthly ![Static Badge](https://img.shields.io/badge/master-master?color=393D47) | +| ------------------------------------ | ------------------------------------------------------------------------------------------- | :----: | :-----: | +| [backup][backup] | A simple and reliable backup plugin | ![](https://img.shields.io/endpoint?url=https%3A%2F%2Flightningd.github.io%2Fplugins%2Fbadges%2Fbackup_main.json) | ![](https://img.shields.io/endpoint?url=https%3A%2F%2Flightningd.github.io%2Fplugins%2Fbadges%2Fbackup_nightly.json) | +| [boltz-channel-creation][boltz] | A Core-Lightning plugin for Boltz Channel Creation Swaps | | | +| [btcli4j][btcli4j] | A Bitcoin Backend to enable safely the pruning mode, and support also rest APIs. | | | +| [circular][circular] | A smart rebalancing plugin for Core Lightning routing nodes | | | +| [clearnet][clearnet] | A plugin that can be used to enforce clearnet connections when possible | ![](https://img.shields.io/endpoint?url=https%3A%2F%2Flightningd.github.io%2Fplugins%2Fbadges%2Fclearnet_main.json) | ![](https://img.shields.io/endpoint?url=https%3A%2F%2Flightningd.github.io%2Fplugins%2Fbadges%2Fclearnet_nightly.json) | +| [cln-ntfy][cln-ntfy] | Core Lightning plugin for sending `ntfy` alerts. | | | +| [csvexportpays][csvexportpays] | A plugin that exports all payments to a CSV file | | | +| [currencyrate][currencyrate] | A plugin to convert other currencies to BTC using web requests | | | +| [datastore][datastore] | The Datastore Plugin | ![](https://img.shields.io/endpoint?url=https%3A%2F%2Flightningd.github.io%2Fplugins%2Fbadges%2Fdatastore_main.json) | ![](https://img.shields.io/endpoint?url=https%3A%2F%2Flightningd.github.io%2Fplugins%2Fbadges%2Fdatastore_nightly.json) | +| [donations][donations] | A simple donations page to accept donations from the web | ![](https://img.shields.io/endpoint?url=https%3A%2F%2Flightningd.github.io%2Fplugins%2Fbadges%2Fdonations_main.json) | ![](https://img.shields.io/endpoint?url=https%3A%2F%2Flightningd.github.io%2Fplugins%2Fbadges%2Fdonations_nightly.json) | +| [event-websocket][event-websocket] | Exposes notifications over a Websocket | | | +| [feeadjuster][feeadjuster] | Dynamic fees to keep your channels more balanced | ![](https://img.shields.io/endpoint?url=https%3A%2F%2Flightningd.github.io%2Fplugins%2Fbadges%2Ffeeadjuster_main.json) | ![](https://img.shields.io/endpoint?url=https%3A%2F%2Flightningd.github.io%2Fplugins%2Fbadges%2Ffeeadjuster_nightly.json) | +| [go-lnmetrics.reporter][reporter] | Collect and report of the lightning node metrics | | | +| [graphql][graphql] | Exposes the Core-Lightning API over [graphql][graphql-spec] | | | +| [holdinvoice][holdinvoice] | Holds htlcs for invoices until settle or cancel is called (aka Hodlinvoices) via RPC/GRPC | ![](https://img.shields.io/endpoint?url=https%3A%2F%2Flightningd.github.io%2Fplugins%2Fbadges%2Fholdinvoice_main.json) | ![](https://img.shields.io/endpoint?url=https%3A%2F%2Flightningd.github.io%2Fplugins%2Fbadges%2Fholdinvoice_nightly.json) | +| [invoice-queue][invoice-queue] | Listen to lightning invoices from multiple nodes and send to a redis queue for processing | | | +| [lightning-qt][lightning-qt] | A bitcoin-qt-like GUI for lightningd | | | +| [listmempoolfunds][listmempoolfunds] | Track unconfirmed wallet deposits | | | +| [monitor][monitor] | helps you analyze the health of your peers and channels | ![](https://img.shields.io/endpoint?url=https%3A%2F%2Flightningd.github.io%2Fplugins%2Fbadges%2Fmonitor_main.json) | ![](https://img.shields.io/endpoint?url=https%3A%2F%2Flightningd.github.io%2Fplugins%2Fbadges%2Fmonitor_nightly.json) | +| [nloop][nloop] | Generic Lightning Loop for boltz | | | +| [paythrough][paythrough] | Pay an invoice through a specific channel, regardless of better routes | | | +| [persistent-channels][pers-chans] | Maintains a number of channels to peers | ![](https://img.shields.io/endpoint?url=https%3A%2F%2Flightningd.github.io%2Fplugins%2Fbadges%2Fpersistent-channels_main.json) | ![](https://img.shields.io/endpoint?url=https%3A%2F%2Flightningd.github.io%2Fplugins%2Fbadges%2Fpersistent-channels_nightly.json) | +| [poncho][poncho] | Turns CLN into a [hosted channels][blip12] provider | | | +| [pruning][pruning] | This plugin manages pruning of bitcoind such that it can always sync | | | +| [python-teos][python-teos] | The Eye of Satoshi - Lightning Watchtower | | | +| [rebalance][rebalance] | Keeps your channels balanced | ![](https://img.shields.io/endpoint?url=https%3A%2F%2Flightningd.github.io%2Fplugins%2Fbadges%2Frebalance_main.json) | ![](https://img.shields.io/endpoint?url=https%3A%2F%2Flightningd.github.io%2Fplugins%2Fbadges%2Frebalance_nightly.json) | +| [reckless][reckless] | An **experimental** plugin manager (search/install plugins) | | | +| [sauron][sauron] | A Bitcoin backend relying on [Esplora][esplora]'s API | | | +| [sitzprobe][sitzprobe] | A Lightning Network payment rehearsal utility | | | +| [sling][sling] | Rebalance your channels with smart rules and built-in background tasks | ![](https://img.shields.io/endpoint?url=https%3A%2F%2Flightningd.github.io%2Fplugins%2Fbadges%2Fsling_main.json) | ![](https://img.shields.io/endpoint?url=https%3A%2F%2Flightningd.github.io%2Fplugins%2Fbadges%2Fsling_nightly.json) | +| [sparko][sparko] | RPC over HTTP with fine-grained permissions, SSE and spark-wallet support | | | +| [summars][summars] | Print configurable summary of node, channels and optionally forwards, invoices, payments | ![](https://img.shields.io/endpoint?url=https%3A%2F%2Flightningd.github.io%2Fplugins%2Fbadges%2Fsummars_main.json) | ![](https://img.shields.io/endpoint?url=https%3A%2F%2Flightningd.github.io%2Fplugins%2Fbadges%2Fsummars_nightly.json) | +| [trustedcoin][trustedcoin] | Replace your Bitcoin Core with data from public block explorers | | | +| [watchtower][watchtower-client] | Watchtower client for The Eye of Satoshi | | | +| [webhook][webhook] | Dispatches webhooks based from [event notifications][event-notifications] | | | +| [zmq][zmq] | Publishes notifications via [ZeroMQ][zmq-home] to configured endpoints | ![](https://img.shields.io/endpoint?url=https%3A%2F%2Flightningd.github.io%2Fplugins%2Fbadges%2Fzmq_main.json) | ![](https://img.shields.io/endpoint?url=https%3A%2F%2Flightningd.github.io%2Fplugins%2Fbadges%2Fzmq_nightly.json) | ## Archived plugins @@ -197,7 +198,7 @@ Python plugins developers must ensure their plugin to work with all Python versi - [Kotlin plugin guideline and example][kotlin-example] by @vincenzopalazzo [autopilot]: https://github.com/lightningd/plugins/tree/master/archived/autopilot -[backup]: https://github.com/lightningd/plugins/tree/master/archived/backup +[backup]: https://github.com/lightningd/plugins/tree/master/backup [blip12]: https://github.com/lightning/blips/blob/42cec1d0f66eb68c840443abb609a5a9acb34f8e/blip-0012.md [boltz]: https://github.com/BoltzExchange/channel-creation-plugin [btcli4j]: https://github.com/clightning4j/btcli4j @@ -210,7 +211,8 @@ Python plugins developers must ensure their plugin to work with all Python versi [csharp-example]: https://github.com/joemphilips/DotNetLightning/tree/master/examples/HelloWorldPlugin [csvexportpays]: https://github.com/0xB10C/c-lightning-plugin-csvexportpays [currencyrate]: https://github.com/lightningd/plugins/tree/master/currencyrate -[donations]: https://github.com/lightningd/plugins/tree/master/archived/donations +[datastore]: https://github.com/lightningd/plugins/tree/master/datastore +[donations]: https://github.com/lightningd/plugins/tree/master/donations [drain]: https://github.com/lightningd/plugins/tree/master/archived/drain [esplora]: https://github.com/Blockstream/esplora [event-notifications]: https://lightning.readthedocs.io/PLUGINS.html#event-notifications