From 63db567703998238a770134fc632a2cbdac40d1e Mon Sep 17 00:00:00 2001
From: daywalker90 <admin@noserver4u.de>
Date: Tue, 11 Apr 2023 17:19:18 +0200
Subject: [PATCH 01/18] init

---
 api/lightning/node_cln.py | 386 ++++++++++++++++++++++++++++++++++++++
 generate_grpc.sh          |   6 +
 2 files changed, 392 insertions(+)
 create mode 100755 api/lightning/node_cln.py

diff --git a/api/lightning/node_cln.py b/api/lightning/node_cln.py
new file mode 100755
index 000000000..80d9b077a
--- /dev/null
+++ b/api/lightning/node_cln.py
@@ -0,0 +1,386 @@
+import hashlib
+import os
+import secrets
+import time
+from base64 import b64decode
+from datetime import datetime, timedelta
+
+import grpc
+import ring
+from decouple import config
+from django.utils import timezone
+
+from . import node_pb2 as noderpc
+from . import node_pb2_grpc as nodestub
+from . import primitives_pb2 as primitivesrpc
+from . import primitives_pb2_grpc as primitivesstub
+
+#######
+# Works with CLN
+#######
+
+# Load the client's certificate and key
+with open(os.path.join(config("CLN_DIR"),'client.pem'), 'rb') as f:
+    client_cert = f.read()
+with open(os.path.join(config("CLN_DIR"),'client-key.pem'), 'rb') as f:
+    client_key = f.read()
+
+# Load the server's certificate
+with open(os.path.join(config("CLN_DIR"),'server.pem'), 'rb') as f:
+    server_cert = f.read()
+
+
+CLN_GRPC_HOST = config("CLN_GRPC_HOST")
+DISABLE_ONCHAIN = config("DISABLE_ONCHAIN", cast=bool, default=True)
+MAX_SWAP_AMOUNT = config("MAX_SWAP_AMOUNT", cast=int, default=500000)
+
+
+class LNNode:
+
+    os.environ["GRPC_SSL_CIPHER_SUITES"] = "HIGH+ECDSA"
+
+    # Create the SSL credentials object
+    creds = grpc.ssl_channel_credentials(root_certificates=server_cert, private_key=client_key, certificate_chain=client_cert)
+    # Create the gRPC channel using the SSL credentials
+    channel = grpc.secure_channel(CLN_GRPC_HOST, creds)
+
+    # Create the gRPC stub
+    stub = nodestub.NodeStub(channel)
+
+    noderpc = noderpc
+    # invoicesrpc = invoicesrpc
+    # routerrpc = routerrpc
+
+    payment_failure_context = {
+        -1: "Catchall nonspecific error.",
+        201: "Already paid with this hash using different amount or destination.",
+        203: "Permanent failure at destination. The data field of the error will be routing failure object.",
+        205: "Unable to find a route.",
+        206: "Route too expensive. Either the fee or the needed total locktime for the route exceeds your maxfeepercent or maxdelay settings, respectively. The data field of the error will indicate the actual fee as well as the feepercent percentage that the fee has of the destination payment amount. It will also indicate the actual delay along the route.",
+        207: "Invoice expired. Payment took too long before expiration, or already expired at the time you initiated payment. The data field of the error indicates now (the current time) and expiry (the invoice expiration) as UNIX epoch time in seconds.",
+        210: "Payment timed out without a payment in progress.",
+    }
+
+    @classmethod
+    def decode_payreq(cls, invoice):
+        """Decodes a lightning payment request (invoice)"""
+        # TODO: no decodepay in cln-grpc yet
+        return "response"
+
+    @classmethod
+    def estimate_fee(cls, amount_sats, target_conf=2, min_confs=1):
+        """Returns estimated fee for onchain payouts"""
+        # feerate estimaes work a bit differently in cln see https://lightning.readthedocs.io/lightning-feerates.7.html
+        request = noderpc.FeeratesRequest(style="PERKB")
+
+        response = cls.stub.Feerates(request)
+
+        # TODO missing "mining_fee_sats" because no weight
+        return {
+            "mining_fee_rate": response.perkb.opening/1000,
+        }
+
+    wallet_balance_cache = {}
+
+    @ring.dict(wallet_balance_cache, expire=10)  # keeps in cache for 10 seconds
+    @classmethod
+    def wallet_balance(cls):
+        """Returns onchain balance"""
+        request = noderpc.ListfundsRequest()
+
+        response = cls.stub.ListFunds(request)
+
+        unconfirmed_balance = 0
+        confirmed_balance = 0
+        total_balance = 0
+        for utxo in response.outputs:
+                if not utxo.reserved:
+                    if utxo.status == 0: # UNCONFIRMED
+                        unconfirmed_balance += utxo.amount_msat.msat // 1_000
+                        total_balance += utxo.amount_msat.msat // 1_000
+                    elif utxo.status == 1: # CONFIRMED
+                        confirmed_balance += utxo.amount_msat.msat // 1_000
+                        total_balance += utxo.amount_msat.msat // 1_000
+
+        return {
+            "total_balance": total_balance,
+            "confirmed_balance": confirmed_balance,
+            "unconfirmed_balance": unconfirmed_balance,
+        }
+
+    channel_balance_cache = {}
+
+    @ring.dict(channel_balance_cache, expire=10)  # keeps in cache for 10 seconds
+    @classmethod
+    def channel_balance(cls):
+        """Returns channels balance"""
+        request = noderpc.ListpeersRequest()
+
+        response = cls.stub.ListPeers(request)
+
+        local_balance_sat = 0
+        remote_balance_sat = 0
+        for peer in response.peers:
+            for channel in peer.channels:
+                local_balance_sat += channel.to_us_msat.msat // 1_000
+                remote_balance_sat += (channel.total_msat.msat-channel.to_us_msat.msat) // 1_000
+        
+        # TODO no idea what is meant by unsettled/pending balance exactly
+        return {
+            "local_balance": local_balance_sat,
+            "remote_balance": remote_balance_sat,
+            "unsettled_local_balance": response.unsettled_local_balance.sat,
+            "unsettled_remote_balance": response.unsettled_remote_balance.sat,
+        }
+
+    @classmethod
+    def pay_onchain(cls, onchainpayment, queue_code=5, on_mempool_code=2):
+        """Send onchain transaction for buyer payouts"""
+
+        if DISABLE_ONCHAIN or onchainpayment.sent_satoshis > MAX_SWAP_AMOUNT:
+            return False
+
+        request = noderpc.WithdrawRequest(destination=onchainpayment.address,satoshi=int(onchainpayment.sent_satoshis), feerate=str(str(int(onchainpayment.mining_fee_rate)*1_000)+"perkb"), minconf=TODO)
+        
+        # Cheap security measure to ensure there has been some non-deterministic time between request and DB check
+        delay = (
+            secrets.randbelow(2**256) / (2**256) * 10
+        )  # Random uniform 0 to 5 secs with good entropy
+        time.sleep(3 + delay)
+
+        if onchainpayment.status == queue_code:
+            # Changing the state to "MEMPO" should be atomic with SendCoins.
+            onchainpayment.status = on_mempool_code
+            onchainpayment.save()
+            response = cls.stub.Withdraw(request)
+
+            if response.txid:
+                onchainpayment.txid = response.txid
+                onchainpayment.broadcasted = True
+            onchainpayment.save()
+            return True
+
+        elif onchainpayment.status == on_mempool_code:
+            # Bug, double payment attempted
+            return True
+
+    @classmethod
+    def cancel_return_hold_invoice(cls, payment_hash):
+        """Cancels or returns a hold invoice"""
+        request = noderpc.HodlInvoiceCancelRequest(payment_hash=bytes.fromhex(payment_hash))
+        response = cls.stub.HodlInvoiceCancel(request)
+        
+        return response.state == 1  # True if state is CANCELED, false otherwise.
+
+    @classmethod
+    def settle_hold_invoice(cls, preimage):
+        """settles a hold invoice"""
+        request = noderpc.HodlInvoiceSettleRequest(payment_hash=hashlib.sha256(bytes.fromhex(preimage)).digest())
+        response = cls.stub.HodlInvoiceSettle(request)
+        
+        return response.state == 2  # True if state is SETTLED, false otherwise.
+
+    @classmethod
+    def gen_hold_invoice(
+        cls, num_satoshis, description, invoice_expiry, cltv_expiry_blocks
+    ):
+        """Generates hold invoice"""
+
+        hold_payment = {}
+        # The preimage is a random hash of 256 bits entropy
+        preimage = hashlib.sha256(secrets.token_bytes(nbytes=32)).digest()
+
+        # Its hash is used to generate the hold invoice
+        r_hash = hashlib.sha256(preimage).digest()
+
+        request = noderpc.InvoiceRequest(
+            description=description,
+            amount_msat=num_satoshis * 1_000,
+            label=TODO, # needs to be a unique string
+            expiry=int(
+                invoice_expiry * 1.5
+            ),  # actual expiry is padded by 50%, if tight, wrong client system clock will say invoice is expired.
+            cltv=cltv_expiry_blocks,
+        )
+        response = cls.stub.HodlInvoice(request)
+
+        hold_payment["invoice"] = response.bolt11
+        payreq_decoded = cls.decode_payreq(hold_payment["invoice"])
+        hold_payment["preimage"] = preimage.hex()
+        hold_payment["payment_hash"] = response.payment_hash
+        hold_payment["created_at"] = timezone.make_aware(
+            datetime.fromtimestamp(payreq_decoded.timestamp)
+        )
+        hold_payment["expires_at"] = response.expires_at
+        hold_payment["cltv_expiry"] = cltv_expiry_blocks
+
+        return hold_payment
+
+    @classmethod
+    def validate_hold_invoice_locked(cls, lnpayment):
+        """Checks if hold invoice is locked"""
+        from api.models import LNPayment
+
+        request = noderpc.HodlInvoiceLookupRequest(
+            payment_hash=bytes.fromhex(lnpayment.payment_hash)
+        )
+        response = cls.stub.HodlInvoiceLookup(request)
+
+        # Will fail if 'unable to locate invoice'. Happens if invoice expiry
+        # time has passed (but these are 15% padded at the moment). Should catch it
+        # and report back that the invoice has expired (better robustness)
+        if response.state == 0:  # OPEN
+            pass
+        if response.state == 1:  # SETTLED
+            pass
+        if response.state == 2:  # CANCELLED
+            pass
+        if response.state == 3:  # ACCEPTED (LOCKED)
+            lnpayment.expiry_height = response.htlcs[0].expiry_height #TODO
+            lnpayment.status = LNPayment.Status.LOCKED
+            lnpayment.save()
+            return True
+
+    @classmethod
+    def resetmc(cls):
+        # don't think an equivalent exists for cln
+        return True
+
+    @classmethod
+    def validate_ln_invoice(cls, invoice, num_satoshis, routing_budget_ppm):
+        """Checks if the submited LN invoice comforms to expectations"""
+
+        payout = {
+            "valid": False,
+            "context": None,
+            "description": None,
+            "payment_hash": None,
+            "created_at": None,
+            "expires_at": None,
+        }
+
+        try:
+            payreq_decoded = cls.decode_payreq(invoice)
+        except Exception:
+            payout["context"] = {
+                "bad_invoice": "Does not look like a valid lightning invoice"
+            }
+            return payout
+
+        # Some wallet providers (e.g. Muun) force routing through a private channel with high fees >1500ppm
+        # These payments will fail. So it is best to let the user know in advance this invoice is not valid.
+        route_hints = payreq_decoded.route_hints
+
+        # Max amount RoboSats will pay for routing
+        if routing_budget_ppm == 0:
+            max_routing_fee_sats = max(
+                num_satoshis * float(config("PROPORTIONAL_ROUTING_FEE_LIMIT")),
+                float(config("MIN_FLAT_ROUTING_FEE_LIMIT_REWARD")),
+            )
+        else:
+            max_routing_fee_sats = int(
+                float(num_satoshis) * float(routing_budget_ppm) / 1000000
+            )
+
+        if route_hints:
+            routes_cost = []
+            # For every hinted route...
+            for hinted_route in route_hints:
+                route_cost = 0
+                # ...add up the cost of every hinted hop...
+                for hop_hint in hinted_route.hop_hints:
+                    route_cost += hop_hint.fee_base_msat / 1000
+                    route_cost += (
+                        hop_hint.fee_proportional_millionths * num_satoshis / 1000000
+                    )
+
+                # ...and store the cost of the route to the array
+                routes_cost.append(route_cost)
+
+            # If the cheapest possible private route is more expensive than what RoboSats is willing to pay
+            if min(routes_cost) >= max_routing_fee_sats:
+                payout["context"] = {
+                    "bad_invoice": "The invoice hinted private routes are not payable within the submitted routing budget."
+                }
+                return payout
+
+        if payreq_decoded.num_satoshis == 0:
+            payout["context"] = {
+                "bad_invoice": "The invoice provided has no explicit amount"
+            }
+            return payout
+
+        if not payreq_decoded.num_satoshis == num_satoshis:
+            payout["context"] = {
+                "bad_invoice": "The invoice provided is not for "
+                + "{:,}".format(num_satoshis)
+                + " Sats"
+            }
+            return payout
+
+        payout["created_at"] = timezone.make_aware(
+            datetime.fromtimestamp(payreq_decoded.timestamp)
+        )
+        payout["expires_at"] = payout["created_at"] + timedelta(
+            seconds=payreq_decoded.expiry
+        )
+
+        if payout["expires_at"] < timezone.now():
+            payout["context"] = {
+                "bad_invoice": "The invoice provided has already expired"
+            }
+            return payout
+
+        payout["valid"] = True
+        payout["description"] = payreq_decoded.description
+        payout["payment_hash"] = payreq_decoded.payment_hash
+
+        return payout
+
+    @classmethod
+    def pay_invoice(cls, lnpayment):
+        """Sends sats. Used for rewards payouts"""
+        from api.models import LNPayment
+
+        fee_limit_sat = int(
+            max(
+                lnpayment.num_satoshis
+                * float(config("PROPORTIONAL_ROUTING_FEE_LIMIT")),
+                float(config("MIN_FLAT_ROUTING_FEE_LIMIT_REWARD")),
+            )
+        )  # 200 ppm or 10 sats
+        timeout_seconds = int(config("REWARDS_TIMEOUT_SECONDS"))
+        request = noderpc.PayRequest(
+            bolt11=lnpayment.invoice,
+            maxfee=fee_limit_sat,
+            retry_for=timeout_seconds,
+        )
+
+        #maybe use waitsendpay here?
+        try: 
+            response = cls.stub.Pay(request)
+            
+            lnpayment.status = LNPayment.Status.SUCCED
+            lnpayment.fee = float(response.amount_sent_msat - response.amount_msat) / 1000
+            lnpayment.preimage = response.payment_preimage
+            lnpayment.save()
+            return True, None
+        except grpc._channel._InactiveRpcError as e:
+            status_code = int(e.details().split('code: Some(')[1].split(')')[0])
+            failure_reason = cls.payment_failure_context[status_code]
+            lnpayment.failure_reason = status_code # or failure_reason ?
+            lnpayment.status = LNPayment.Status.FAILRO
+            lnpayment.save()
+            return False, failure_reason
+
+
+    @classmethod
+    def double_check_htlc_is_settled(cls, payment_hash):
+        """Just as it sounds. Better safe than sorry!"""
+        request = noderpc.ListinvoicesRequest(payment_hash=bytes.fromhex(payment_hash))
+        response = cls.stub.ListInvoices(request)
+
+        return (
+            response.status == 1
+        )   # CLN states: UNPAID = 0, PAID = 1, EXPIRED = 2, this is clns own invoice-lookup
+            # so just a check for paid/unpaid/expired not hodl-invoice related states like ACCEPTED/CANCELED
diff --git a/generate_grpc.sh b/generate_grpc.sh
index 5c5685b5c..6035174e7 100755
--- a/generate_grpc.sh
+++ b/generate_grpc.sh
@@ -20,6 +20,12 @@ python3 -m grpc_tools.protoc --proto_path=googleapis:. --python_out=. --grpc_pyt
 curl -o verrpc.proto -s https://raw.githubusercontent.com/lightningnetwork/lnd/master/lnrpc/verrpc/verrpc.proto
 python3 -m grpc_tools.protoc --proto_path=googleapis:. --python_out=. --grpc_python_out=. verrpc.proto
 
+# generate cln grpc definitions
+curl -o node.proto -s https://raw.githubusercontent.com/daywalker90/lightning/master/cln-grpc/proto/node.proto
+python3 -m grpc_tools.protoc --proto_path=. --python_out=. --grpc_python_out=. node.proto
+curl -o primitives.proto -s https://raw.githubusercontent.com/daywalker90/lightning/master/cln-grpc/proto/primitives.proto
+python3 -m grpc_tools.protoc --proto_path=. --python_out=. --grpc_python_out=. primitives.proto
+
 # patch generated files relative imports
 sed -i 's/^import .*_pb2 as/from . \0/' router_pb2.py
 sed -i 's/^import .*_pb2 as/from . \0/' invoices_pb2.py

From fb2c1186ab328be26bfb2703dbbc977743daf102 Mon Sep 17 00:00:00 2001
From: daywalker90 <admin@noserver4u.de>
Date: Wed, 12 Apr 2023 00:16:17 +0200
Subject: [PATCH 02/18] first draft

---
 api/lightning/node_cln.py | 70 +++++++++++++++++++++++++--------------
 generate_grpc.sh          |  7 ++--
 2 files changed, 49 insertions(+), 28 deletions(-)

diff --git a/api/lightning/node_cln.py b/api/lightning/node_cln.py
index 80d9b077a..34c2d8282 100755
--- a/api/lightning/node_cln.py
+++ b/api/lightning/node_cln.py
@@ -64,8 +64,10 @@ class LNNode:
     @classmethod
     def decode_payreq(cls, invoice):
         """Decodes a lightning payment request (invoice)"""
-        # TODO: no decodepay in cln-grpc yet
-        return "response"
+        request = noderpc.DecodeBolt11Request(bolt11=invoice)
+
+        response = cls.stub.DecodeBolt11(request)
+        return response
 
     @classmethod
     def estimate_fee(cls, amount_sats, target_conf=2, min_confs=1):
@@ -75,8 +77,9 @@ def estimate_fee(cls, amount_sats, target_conf=2, min_confs=1):
 
         response = cls.stub.Feerates(request)
 
-        # TODO missing "mining_fee_sats" because no weight
+        # "opening" -> ~12 block target
         return {
+            "mining_fee_sats": response.onchain_fee_estimates.opening_channel_satoshis,
             "mining_fee_rate": response.perkb.opening/1000,
         }
 
@@ -122,15 +125,16 @@ def channel_balance(cls):
         remote_balance_sat = 0
         for peer in response.peers:
             for channel in peer.channels:
-                local_balance_sat += channel.to_us_msat.msat // 1_000
-                remote_balance_sat += (channel.total_msat.msat-channel.to_us_msat.msat) // 1_000
+                if channel.state == 2: # CHANNELD_NORMAL
+                    local_balance_sat += channel.to_us_msat.msat // 1_000
+                    remote_balance_sat += (channel.total_msat.msat-channel.to_us_msat.msat) // 1_000
         
-        # TODO no idea what is meant by unsettled/pending balance exactly
+        # TODO no idea what is meant by unsettled/pending balance here exactly
         return {
             "local_balance": local_balance_sat,
             "remote_balance": remote_balance_sat,
-            "unsettled_local_balance": response.unsettled_local_balance.sat,
-            "unsettled_remote_balance": response.unsettled_remote_balance.sat,
+            "unsettled_local_balance": 0,
+            "unsettled_remote_balance": 0,
         }
 
     @classmethod
@@ -140,7 +144,11 @@ def pay_onchain(cls, onchainpayment, queue_code=5, on_mempool_code=2):
         if DISABLE_ONCHAIN or onchainpayment.sent_satoshis > MAX_SWAP_AMOUNT:
             return False
 
-        request = noderpc.WithdrawRequest(destination=onchainpayment.address,satoshi=int(onchainpayment.sent_satoshis), feerate=str(str(int(onchainpayment.mining_fee_rate)*1_000)+"perkb"), minconf=TODO)
+        request = noderpc.WithdrawRequest(
+            destination=onchainpayment.address,satoshi=int(onchainpayment.sent_satoshis),
+            feerate=str(int(onchainpayment.mining_fee_rate)*1_000)+"perkb",
+            minconf=int(not config("SPEND_UNCONFIRMED", default=False, cast=bool))
+        )
         
         # Cheap security measure to ensure there has been some non-deterministic time between request and DB check
         delay = (
@@ -196,11 +204,12 @@ def gen_hold_invoice(
         request = noderpc.InvoiceRequest(
             description=description,
             amount_msat=num_satoshis * 1_000,
-            label=TODO, # needs to be a unique string
+            label="TODO", # TODO needs to be a unique string
             expiry=int(
                 invoice_expiry * 1.5
             ),  # actual expiry is padded by 50%, if tight, wrong client system clock will say invoice is expired.
             cltv=cltv_expiry_blocks,
+            preimage=preimage, # preimage is actually optional in cln, as cln would generate one by default
         )
         response = cls.stub.HodlInvoice(request)
 
@@ -236,15 +245,15 @@ def validate_hold_invoice_locked(cls, lnpayment):
         if response.state == 2:  # CANCELLED
             pass
         if response.state == 3:  # ACCEPTED (LOCKED)
-            lnpayment.expiry_height = response.htlcs[0].expiry_height #TODO
+            lnpayment.expiry_height = response.htlc_cltv
             lnpayment.status = LNPayment.Status.LOCKED
             lnpayment.save()
             return True
 
     @classmethod
     def resetmc(cls):
-        # don't think an equivalent exists for cln
-        return True
+        # don't think an equivalent exists for cln, maybe deleting gossip_store file?
+        return False
 
     @classmethod
     def validate_ln_invoice(cls, invoice, num_satoshis, routing_budget_ppm):
@@ -269,7 +278,7 @@ def validate_ln_invoice(cls, invoice, num_satoshis, routing_budget_ppm):
 
         # Some wallet providers (e.g. Muun) force routing through a private channel with high fees >1500ppm
         # These payments will fail. So it is best to let the user know in advance this invoice is not valid.
-        route_hints = payreq_decoded.route_hints
+        route_hints = payreq_decoded.route_hints.hints
 
         # Max amount RoboSats will pay for routing
         if routing_budget_ppm == 0:
@@ -288,10 +297,10 @@ def validate_ln_invoice(cls, invoice, num_satoshis, routing_budget_ppm):
             for hinted_route in route_hints:
                 route_cost = 0
                 # ...add up the cost of every hinted hop...
-                for hop_hint in hinted_route.hop_hints:
-                    route_cost += hop_hint.fee_base_msat / 1000
+                for hop_hint in hinted_route.hops:
+                    route_cost += hop_hint.feebase.msat / 1_000
                     route_cost += (
-                        hop_hint.fee_proportional_millionths * num_satoshis / 1000000
+                        hop_hint.feeprop * num_satoshis / 1_000_000
                     )
 
                 # ...and store the cost of the route to the array
@@ -304,13 +313,13 @@ def validate_ln_invoice(cls, invoice, num_satoshis, routing_budget_ppm):
                 }
                 return payout
 
-        if payreq_decoded.num_satoshis == 0:
+        if payreq_decoded.amount_msat == 0:
             payout["context"] = {
                 "bad_invoice": "The invoice provided has no explicit amount"
             }
             return payout
 
-        if not payreq_decoded.num_satoshis == num_satoshis:
+        if not payreq_decoded.amount_msat // 1_000 == num_satoshis:
             payout["context"] = {
                 "bad_invoice": "The invoice provided is not for "
                 + "{:,}".format(num_satoshis)
@@ -360,11 +369,24 @@ def pay_invoice(cls, lnpayment):
         try: 
             response = cls.stub.Pay(request)
             
-            lnpayment.status = LNPayment.Status.SUCCED
-            lnpayment.fee = float(response.amount_sent_msat - response.amount_msat) / 1000
-            lnpayment.preimage = response.payment_preimage
-            lnpayment.save()
-            return True, None
+            if response.status == 0: #COMPLETE
+                lnpayment.status = LNPayment.Status.SUCCED
+                lnpayment.fee = float(response.amount_sent_msat - response.amount_msat) / 1000
+                lnpayment.preimage = response.payment_preimage
+                lnpayment.save()
+                return True, None
+            elif response.status == 1: #PENDING
+                failure_reason = str("PENDING")
+                lnpayment.failure_reason = failure_reason
+                lnpayment.status = LNPayment.Status.FLIGHT
+                lnpayment.save()
+                return False, failure_reason
+            else: # status == 2 FAILED
+                failure_reason = str("FAILED")
+                lnpayment.failure_reason = failure_reason
+                lnpayment.status = LNPayment.Status.FAILRO
+                lnpayment.save()
+                return False, failure_reason
         except grpc._channel._InactiveRpcError as e:
             status_code = int(e.details().split('code: Some(')[1].split(')')[0])
             failure_reason = cls.payment_failure_context[status_code]
diff --git a/generate_grpc.sh b/generate_grpc.sh
index 6035174e7..f1d25d01f 100755
--- a/generate_grpc.sh
+++ b/generate_grpc.sh
@@ -21,10 +21,9 @@ curl -o verrpc.proto -s https://raw.githubusercontent.com/lightningnetwork/lnd/m
 python3 -m grpc_tools.protoc --proto_path=googleapis:. --python_out=. --grpc_python_out=. verrpc.proto
 
 # generate cln grpc definitions
-curl -o node.proto -s https://raw.githubusercontent.com/daywalker90/lightning/master/cln-grpc/proto/node.proto
-python3 -m grpc_tools.protoc --proto_path=. --python_out=. --grpc_python_out=. node.proto
-curl -o primitives.proto -s https://raw.githubusercontent.com/daywalker90/lightning/master/cln-grpc/proto/primitives.proto
-python3 -m grpc_tools.protoc --proto_path=. --python_out=. --grpc_python_out=. primitives.proto
+curl -o node.proto -s https://raw.githubusercontent.com/daywalker90/lightning/hodlvoice/cln-grpc/proto/node.proto
+curl -o primitives.proto -s https://raw.githubusercontent.com/daywalker90/lightning/hodlvoice/cln-grpc/proto/primitives.proto
+python3 -m grpc_tools.protoc --proto_path=. --python_out=. --grpc_python_out=. node.proto primitives.proto
 
 # patch generated files relative imports
 sed -i 's/^import .*_pb2 as/from . \0/' router_pb2.py

From d282c3631d3481c5027d0951ba67c0472e4e6bef Mon Sep 17 00:00:00 2001
From: daywalker90 <admin@noserver4u.de>
Date: Mon, 17 Apr 2023 13:56:27 +0200
Subject: [PATCH 03/18] add unsettled_local_balance and
 unsettled_remote_balance

---
 api/lightning/node_cln.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/api/lightning/node_cln.py b/api/lightning/node_cln.py
index 34c2d8282..f326436c6 100755
--- a/api/lightning/node_cln.py
+++ b/api/lightning/node_cln.py
@@ -123,18 +123,24 @@ def channel_balance(cls):
 
         local_balance_sat = 0
         remote_balance_sat = 0
+        unsettled_local_balance = 0
+        unsettled_remote_balance = 0
         for peer in response.peers:
             for channel in peer.channels:
                 if channel.state == 2: # CHANNELD_NORMAL
                     local_balance_sat += channel.to_us_msat.msat // 1_000
                     remote_balance_sat += (channel.total_msat.msat-channel.to_us_msat.msat) // 1_000
+                for htlc in channel.htlcs:
+                    if htlc.direction == 0: #IN
+                        unsettled_local_balance += htlc.amount_msat // 1_000
+                    elif htlc.direction == 1: #OUT
+                        unsettled_remote_balance += htlc.amount_msat // 1_000
         
-        # TODO no idea what is meant by unsettled/pending balance here exactly
         return {
             "local_balance": local_balance_sat,
             "remote_balance": remote_balance_sat,
-            "unsettled_local_balance": 0,
-            "unsettled_remote_balance": 0,
+            "unsettled_local_balance": unsettled_local_balance,
+            "unsettled_remote_balance": unsettled_remote_balance,
         }
 
     @classmethod

From e91e5f634f27d1f88d0b6010605aeaf607ab2fb5 Mon Sep 17 00:00:00 2001
From: daywalker90 <admin@noserver4u.de>
Date: Mon, 17 Apr 2023 14:05:27 +0200
Subject: [PATCH 04/18] gen_hold_invoice now takes 3 more variables to build a
 label for cln

---
 api/lightning/node_cln.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/api/lightning/node_cln.py b/api/lightning/node_cln.py
index f326436c6..9a8cf80d1 100755
--- a/api/lightning/node_cln.py
+++ b/api/lightning/node_cln.py
@@ -196,7 +196,7 @@ def settle_hold_invoice(cls, preimage):
 
     @classmethod
     def gen_hold_invoice(
-        cls, num_satoshis, description, invoice_expiry, cltv_expiry_blocks
+        cls, num_satoshis, description, invoice_expiry, cltv_expiry_blocks, order_id , receiver_robot, time 
     ):
         """Generates hold invoice"""
 
@@ -210,7 +210,7 @@ def gen_hold_invoice(
         request = noderpc.InvoiceRequest(
             description=description,
             amount_msat=num_satoshis * 1_000,
-            label="TODO", # TODO needs to be a unique string
+            label=str(order_id) + "_" + str(receiver_robot) + "_" + str(time),
             expiry=int(
                 invoice_expiry * 1.5
             ),  # actual expiry is padded by 50%, if tight, wrong client system clock will say invoice is expired.

From 267371caeb9442c1287107194357cb6e21b20501 Mon Sep 17 00:00:00 2001
From: daywalker90 <admin@noserver4u.de>
Date: Mon, 17 Apr 2023 14:06:47 +0200
Subject: [PATCH 05/18] remove unneeded payment_hash from gen_hold_invoice

---
 api/lightning/node_cln.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/api/lightning/node_cln.py b/api/lightning/node_cln.py
index 9a8cf80d1..7c95b713a 100755
--- a/api/lightning/node_cln.py
+++ b/api/lightning/node_cln.py
@@ -204,9 +204,6 @@ def gen_hold_invoice(
         # The preimage is a random hash of 256 bits entropy
         preimage = hashlib.sha256(secrets.token_bytes(nbytes=32)).digest()
 
-        # Its hash is used to generate the hold invoice
-        r_hash = hashlib.sha256(preimage).digest()
-
         request = noderpc.InvoiceRequest(
             description=description,
             amount_msat=num_satoshis * 1_000,

From ccac832be5d543b43b46e939460fb942511dad94 Mon Sep 17 00:00:00 2001
From: daywalker90 <admin@noserver4u.de>
Date: Mon, 17 Apr 2023 14:08:49 +0200
Subject: [PATCH 06/18] remove comment

---
 api/lightning/node_cln.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/api/lightning/node_cln.py b/api/lightning/node_cln.py
index 7c95b713a..04952a8bc 100755
--- a/api/lightning/node_cln.py
+++ b/api/lightning/node_cln.py
@@ -368,7 +368,6 @@ def pay_invoice(cls, lnpayment):
             retry_for=timeout_seconds,
         )
 
-        #maybe use waitsendpay here?
         try: 
             response = cls.stub.Pay(request)
             

From 1aeb1ec8a9ae54cc16e6d484ac09640e4e8bebcd Mon Sep 17 00:00:00 2001
From: daywalker90 <admin@noserver4u.de>
Date: Mon, 17 Apr 2023 14:40:31 +0200
Subject: [PATCH 07/18] add get_cln_version

---
 api/lightning/node_cln.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/api/lightning/node_cln.py b/api/lightning/node_cln.py
index 04952a8bc..113026496 100755
--- a/api/lightning/node_cln.py
+++ b/api/lightning/node_cln.py
@@ -408,3 +408,11 @@ def double_check_htlc_is_settled(cls, payment_hash):
             response.status == 1
         )   # CLN states: UNPAID = 0, PAID = 1, EXPIRED = 2, this is clns own invoice-lookup
             # so just a check for paid/unpaid/expired not hodl-invoice related states like ACCEPTED/CANCELED
+
+    @classmethod
+    def get_lnd_version(cls):
+        request = noderpc.GetinfoRequest()
+        response = cls.stub.Getinfo(request)
+
+        return response.version
+    
\ No newline at end of file

From 0e03822374fd5f1831360eace5791d00bf23d1bd Mon Sep 17 00:00:00 2001
From: daywalker90 <admin@noserver4u.de>
Date: Tue, 18 Apr 2023 15:50:51 +0200
Subject: [PATCH 08/18] first draft of clns follow_send_payment

---
 api/lightning/node_cln.py | 174 +++++++++++++++++++++++++++++++++++---
 1 file changed, 163 insertions(+), 11 deletions(-)

diff --git a/api/lightning/node_cln.py b/api/lightning/node_cln.py
index 113026496..43151c347 100755
--- a/api/lightning/node_cln.py
+++ b/api/lightning/node_cln.py
@@ -4,6 +4,7 @@
 import time
 from base64 import b64decode
 from datetime import datetime, timedelta
+from celery import shared_task
 
 import grpc
 import ring
@@ -54,10 +55,10 @@ class LNNode:
     payment_failure_context = {
         -1: "Catchall nonspecific error.",
         201: "Already paid with this hash using different amount or destination.",
-        203: "Permanent failure at destination. The data field of the error will be routing failure object.",
+        203: "Permanent failure at destination.",
         205: "Unable to find a route.",
-        206: "Route too expensive. Either the fee or the needed total locktime for the route exceeds your maxfeepercent or maxdelay settings, respectively. The data field of the error will indicate the actual fee as well as the feepercent percentage that the fee has of the destination payment amount. It will also indicate the actual delay along the route.",
-        207: "Invoice expired. Payment took too long before expiration, or already expired at the time you initiated payment. The data field of the error indicates now (the current time) and expiry (the invoice expiration) as UNIX epoch time in seconds.",
+        206: "Route too expensive.",
+        207: "Invoice expired.",
         210: "Payment timed out without a payment in progress.",
     }
 
@@ -135,7 +136,7 @@ def channel_balance(cls):
                         unsettled_local_balance += htlc.amount_msat // 1_000
                     elif htlc.direction == 1: #OUT
                         unsettled_remote_balance += htlc.amount_msat // 1_000
-        
+
         return {
             "local_balance": local_balance_sat,
             "remote_balance": remote_balance_sat,
@@ -155,7 +156,7 @@ def pay_onchain(cls, onchainpayment, queue_code=5, on_mempool_code=2):
             feerate=str(int(onchainpayment.mining_fee_rate)*1_000)+"perkb",
             minconf=int(not config("SPEND_UNCONFIRMED", default=False, cast=bool))
         )
-        
+
         # Cheap security measure to ensure there has been some non-deterministic time between request and DB check
         delay = (
             secrets.randbelow(2**256) / (2**256) * 10
@@ -183,7 +184,7 @@ def cancel_return_hold_invoice(cls, payment_hash):
         """Cancels or returns a hold invoice"""
         request = noderpc.HodlInvoiceCancelRequest(payment_hash=bytes.fromhex(payment_hash))
         response = cls.stub.HodlInvoiceCancel(request)
-        
+
         return response.state == 1  # True if state is CANCELED, false otherwise.
 
     @classmethod
@@ -191,12 +192,12 @@ def settle_hold_invoice(cls, preimage):
         """settles a hold invoice"""
         request = noderpc.HodlInvoiceSettleRequest(payment_hash=hashlib.sha256(bytes.fromhex(preimage)).digest())
         response = cls.stub.HodlInvoiceSettle(request)
-        
+
         return response.state == 2  # True if state is SETTLED, false otherwise.
 
     @classmethod
     def gen_hold_invoice(
-        cls, num_satoshis, description, invoice_expiry, cltv_expiry_blocks, order_id , receiver_robot, time 
+        cls, num_satoshis, description, invoice_expiry, cltv_expiry_blocks, order_id , receiver_robot, time
     ):
         """Generates hold invoice"""
 
@@ -368,9 +369,9 @@ def pay_invoice(cls, lnpayment):
             retry_for=timeout_seconds,
         )
 
-        try: 
+        try:
             response = cls.stub.Pay(request)
-            
+
             if response.status == 0: #COMPLETE
                 lnpayment.status = LNPayment.Status.SUCCED
                 lnpayment.fee = float(response.amount_sent_msat - response.amount_msat) / 1000
@@ -415,4 +416,155 @@ def get_lnd_version(cls):
         response = cls.stub.Getinfo(request)
 
         return response.version
-    
\ No newline at end of file
+
+    @classmethod
+    @shared_task(name="follow_send_payment", time_limit=180)
+    def follow_send_payment(cls, hash):
+        """Sends sats to buyer, continuous update"""
+
+        from datetime import timedelta
+
+        from decouple import config
+        from django.utils import timezone
+
+        from api.models import LNPayment, Order
+
+        lnpayment = LNPayment.objects.get(payment_hash=hash)
+        lnpayment.last_routing_time = timezone.now()
+        lnpayment.save()
+
+        # Default is 0ppm. Set by the user over API. Client's default is 1000 ppm.
+        fee_limit_sat = int(
+            float(lnpayment.num_satoshis) * float(lnpayment.routing_budget_ppm) / 1000000
+        )
+        timeout_seconds = int(config("PAYOUT_TIMEOUT_SECONDS"))
+
+        request = noderpc.PayRequest(
+            bolt11=lnpayment.invoice,
+            maxfee=fee_limit_sat,
+            retry_for=timeout_seconds, #retry_for is not quite the same as a timeout. Pay can still take SIGNIFICANTLY longer to return if htlcs are stuck!
+            # allow_self_payment=True, No such thing in pay command and self_payments do not work with pay!
+        )
+
+        order = lnpayment.order_paid_LN
+        if order.trade_escrow.num_satoshis < lnpayment.num_satoshis:
+            print(f"Order: {order.id} Payout is larger than collateral !?")
+            return
+
+        def handle_response(response, was_in_transit=False):
+            lnpayment.status = LNPayment.Status.FLIGHT
+            lnpayment.in_flight = True
+            lnpayment.save()
+            order.status = Order.Status.PAY
+            order.save()
+
+            if response.status == 1:  # Status 1 'PENDING'
+                print(f"Order: {order.id} IN_FLIGHT. Hash {hash}")
+
+                # If payment was already "payment is in transition" we do not
+                # want to spawn a new thread every 3 minutes to check on it.
+                # in case this thread dies, let's move the last_routing_time
+                # 20 minutes in the future so another thread spawns.
+                if was_in_transit:
+                    lnpayment.last_routing_time = timezone.now() + timedelta(minutes=20)
+                    lnpayment.save()
+
+            if response.status == 2:  # Status 3 'FAILED'
+                lnpayment.status = LNPayment.Status.FAILRO
+                lnpayment.last_routing_time = timezone.now()
+                lnpayment.routing_attempts += 1
+                lnpayment.failure_reason = -1 #no failure_reason in non-error pay response with stauts FAILED
+                lnpayment.in_flight = False
+                if lnpayment.routing_attempts > 2:
+                    lnpayment.status = LNPayment.Status.EXPIRE
+                    lnpayment.routing_attempts = 0
+                lnpayment.save()
+
+                order.status = Order.Status.FAI
+                order.expires_at = timezone.now() + timedelta(
+                    seconds=order.t_to_expire(Order.Status.FAI)
+                )
+                order.save()
+                print(
+                    f"Order: {order.id} FAILED. Hash: {hash} Reason: {cls.payment_failure_context[-1]}"
+                )
+                return {
+                    "succeded": False,
+                    "context": f"payment failure reason: {cls.payment_failure_context[-1]}",
+                }
+
+            if response.status == 0:  # Status 2 'COMPLETE'
+                print(f"Order: {order.id} SUCCEEDED. Hash: {hash}")
+                lnpayment.status = LNPayment.Status.SUCCED
+                lnpayment.fee = float(response.amount_sent_msat.msat - response.amount_msat.msat) / 1000
+                lnpayment.preimage = response.payment_preimage
+                lnpayment.save()
+                order.status = Order.Status.SUC
+                order.expires_at = timezone.now() + timedelta(
+                    seconds=order.t_to_expire(Order.Status.SUC)
+                )
+                order.save()
+                results = {"succeded": True}
+                return results
+
+        try:
+            response = cls.stub.Pay(request)
+            handle_response(response)
+
+        except grpc._channel._InactiveRpcError as e:
+            if "code: Some" in str(e):
+                status_code = int(e.details().split('code: Some(')[1].split(')')[0])
+                if status_code == 201: #Already paid with this hash using different amount or destination
+                    # i don't think this can happen really, since we don't use the amount_msat in request and if you just try 'pay' 2x where the first time it succeeds you get the same non-error result the 2nd time.
+                    # Listpays has some different fields as pay aswell, so not sure this makes sense
+                    print(f"Order: {order.id} ALREADY PAID. Hash: {hash}.")
+
+                    request = noderpc.ListpaysRequest(
+                        payment_hash=bytes.fromhex(hash), status="complete"
+                    )
+
+                    for response in cls.stub.ListPays(request):
+                        handle_response(response)
+                elif status_code == 203 or status_code == 205 or status_code == 206:  #Permanent failure at destination. or Unable to find a route. or Route too expensive.
+                    lnpayment.status = LNPayment.Status.FAILRO
+                    lnpayment.last_routing_time = timezone.now()
+                    lnpayment.routing_attempts += 1
+                    lnpayment.failure_reason = status_code
+                    lnpayment.in_flight = False
+                    if lnpayment.routing_attempts > 2:
+                        lnpayment.status = LNPayment.Status.EXPIRE
+                        lnpayment.routing_attempts = 0
+                    lnpayment.save()
+
+                    order.status = Order.Status.FAI
+                    order.expires_at = timezone.now() + timedelta(
+                        seconds=order.t_to_expire(Order.Status.FAI)
+                    )
+                    order.save()
+                    print(
+                        f"Order: {order.id} FAILED. Hash: {hash} Reason: {cls.payment_failure_context[status_code]}"
+                    )
+                    return {
+                        "succeded": False,
+                        "context": f"payment failure reason: {cls.payment_failure_context[status_code]}",
+                    }
+                elif status_code == 207:  #invoice expired
+                    print(f"Order: {order.id}. INVOICE EXPIRED. Hash: {hash}")
+                    lnpayment.status = LNPayment.Status.EXPIRE
+                    lnpayment.last_routing_time = timezone.now()
+                    lnpayment.in_flight = False
+                    lnpayment.save()
+                    order.status = Order.Status.FAI
+                    order.expires_at = timezone.now() + timedelta(
+                        seconds=order.t_to_expire(Order.Status.FAI)
+                    )
+                    order.save()
+                    results = {"succeded": False, "context": "The payout invoice has expired"}
+                    return results
+                else: #-1 and 210 (don't know when 210 happens exactly)
+                    print(str(e))
+            else:
+                print(str(e))
+
+        except Exception as e:
+            print(str(e))

From 9eb15cabc7bd8b5c2747db3381c9acb70109ac81 Mon Sep 17 00:00:00 2001
From: daywalker90 <admin@noserver4u.de>
Date: Tue, 18 Apr 2023 15:51:20 +0200
Subject: [PATCH 09/18] fix name of get_lnd_version

---
 api/lightning/node_cln.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/api/lightning/node_cln.py b/api/lightning/node_cln.py
index 43151c347..374d3b649 100755
--- a/api/lightning/node_cln.py
+++ b/api/lightning/node_cln.py
@@ -411,7 +411,7 @@ def double_check_htlc_is_settled(cls, payment_hash):
             # so just a check for paid/unpaid/expired not hodl-invoice related states like ACCEPTED/CANCELED
 
     @classmethod
-    def get_lnd_version(cls):
+    def get_cln_version(cls):
         request = noderpc.GetinfoRequest()
         response = cls.stub.Getinfo(request)
 

From d3e0c113587e4cc35fcadce5253e6dd531605fde Mon Sep 17 00:00:00 2001
From: daywalker90 <admin@noserver4u.de>
Date: Tue, 18 Apr 2023 16:06:45 +0200
Subject: [PATCH 10/18] enable flake8

---
 api/lightning/node_cln.py | 82 ++++++++++++++++++++++-----------------
 1 file changed, 46 insertions(+), 36 deletions(-)

diff --git a/api/lightning/node_cln.py b/api/lightning/node_cln.py
index 374d3b649..a0fad9e50 100755
--- a/api/lightning/node_cln.py
+++ b/api/lightning/node_cln.py
@@ -21,13 +21,13 @@
 #######
 
 # Load the client's certificate and key
-with open(os.path.join(config("CLN_DIR"),'client.pem'), 'rb') as f:
+with open(os.path.join(config("CLN_DIR"), 'client.pem'), 'rb') as f:
     client_cert = f.read()
-with open(os.path.join(config("CLN_DIR"),'client-key.pem'), 'rb') as f:
+with open(os.path.join(config("CLN_DIR"), 'client-key.pem'), 'rb') as f:
     client_key = f.read()
 
 # Load the server's certificate
-with open(os.path.join(config("CLN_DIR"),'server.pem'), 'rb') as f:
+with open(os.path.join(config("CLN_DIR"), 'server.pem'), 'rb') as f:
     server_cert = f.read()
 
 
@@ -41,7 +41,8 @@ class LNNode:
     os.environ["GRPC_SSL_CIPHER_SUITES"] = "HIGH+ECDSA"
 
     # Create the SSL credentials object
-    creds = grpc.ssl_channel_credentials(root_certificates=server_cert, private_key=client_key, certificate_chain=client_cert)
+    creds = grpc.ssl_channel_credentials(
+        root_certificates=server_cert, private_key=client_key, certificate_chain=client_cert)
     # Create the gRPC channel using the SSL credentials
     channel = grpc.secure_channel(CLN_GRPC_HOST, creds)
 
@@ -98,13 +99,13 @@ def wallet_balance(cls):
         confirmed_balance = 0
         total_balance = 0
         for utxo in response.outputs:
-                if not utxo.reserved:
-                    if utxo.status == 0: # UNCONFIRMED
-                        unconfirmed_balance += utxo.amount_msat.msat // 1_000
-                        total_balance += utxo.amount_msat.msat // 1_000
-                    elif utxo.status == 1: # CONFIRMED
-                        confirmed_balance += utxo.amount_msat.msat // 1_000
-                        total_balance += utxo.amount_msat.msat // 1_000
+            if not utxo.reserved:
+                if utxo.status == 0:  # UNCONFIRMED
+                    unconfirmed_balance += utxo.amount_msat.msat // 1_000
+                    total_balance += utxo.amount_msat.msat // 1_000
+                elif utxo.status == 1:  # CONFIRMED
+                    confirmed_balance += utxo.amount_msat.msat // 1_000
+                    total_balance += utxo.amount_msat.msat // 1_000
 
         return {
             "total_balance": total_balance,
@@ -128,13 +129,14 @@ def channel_balance(cls):
         unsettled_remote_balance = 0
         for peer in response.peers:
             for channel in peer.channels:
-                if channel.state == 2: # CHANNELD_NORMAL
+                if channel.state == 2:  # CHANNELD_NORMAL
                     local_balance_sat += channel.to_us_msat.msat // 1_000
-                    remote_balance_sat += (channel.total_msat.msat-channel.to_us_msat.msat) // 1_000
+                    remote_balance_sat += (channel.total_msat.msat -
+                                           channel.to_us_msat.msat) // 1_000
                 for htlc in channel.htlcs:
-                    if htlc.direction == 0: #IN
+                    if htlc.direction == 0:  # IN
                         unsettled_local_balance += htlc.amount_msat // 1_000
-                    elif htlc.direction == 1: #OUT
+                    elif htlc.direction == 1:  # OUT
                         unsettled_remote_balance += htlc.amount_msat // 1_000
 
         return {
@@ -152,7 +154,8 @@ def pay_onchain(cls, onchainpayment, queue_code=5, on_mempool_code=2):
             return False
 
         request = noderpc.WithdrawRequest(
-            destination=onchainpayment.address,satoshi=int(onchainpayment.sent_satoshis),
+            destination=onchainpayment.address, satoshi=int(
+                onchainpayment.sent_satoshis),
             feerate=str(int(onchainpayment.mining_fee_rate)*1_000)+"perkb",
             minconf=int(not config("SPEND_UNCONFIRMED", default=False, cast=bool))
         )
@@ -182,7 +185,8 @@ def pay_onchain(cls, onchainpayment, queue_code=5, on_mempool_code=2):
     @classmethod
     def cancel_return_hold_invoice(cls, payment_hash):
         """Cancels or returns a hold invoice"""
-        request = noderpc.HodlInvoiceCancelRequest(payment_hash=bytes.fromhex(payment_hash))
+        request = noderpc.HodlInvoiceCancelRequest(
+            payment_hash=bytes.fromhex(payment_hash))
         response = cls.stub.HodlInvoiceCancel(request)
 
         return response.state == 1  # True if state is CANCELED, false otherwise.
@@ -190,14 +194,15 @@ def cancel_return_hold_invoice(cls, payment_hash):
     @classmethod
     def settle_hold_invoice(cls, preimage):
         """settles a hold invoice"""
-        request = noderpc.HodlInvoiceSettleRequest(payment_hash=hashlib.sha256(bytes.fromhex(preimage)).digest())
+        request = noderpc.HodlInvoiceSettleRequest(
+            payment_hash=hashlib.sha256(bytes.fromhex(preimage)).digest())
         response = cls.stub.HodlInvoiceSettle(request)
 
         return response.state == 2  # True if state is SETTLED, false otherwise.
 
     @classmethod
     def gen_hold_invoice(
-        cls, num_satoshis, description, invoice_expiry, cltv_expiry_blocks, order_id , receiver_robot, time
+        cls, num_satoshis, description, invoice_expiry, cltv_expiry_blocks, order_id, receiver_robot, time
     ):
         """Generates hold invoice"""
 
@@ -213,7 +218,7 @@ def gen_hold_invoice(
                 invoice_expiry * 1.5
             ),  # actual expiry is padded by 50%, if tight, wrong client system clock will say invoice is expired.
             cltv=cltv_expiry_blocks,
-            preimage=preimage, # preimage is actually optional in cln, as cln would generate one by default
+            preimage=preimage,  # preimage is actually optional in cln, as cln would generate one by default
         )
         response = cls.stub.HodlInvoice(request)
 
@@ -372,19 +377,20 @@ def pay_invoice(cls, lnpayment):
         try:
             response = cls.stub.Pay(request)
 
-            if response.status == 0: #COMPLETE
+            if response.status == 0:  # COMPLETE
                 lnpayment.status = LNPayment.Status.SUCCED
-                lnpayment.fee = float(response.amount_sent_msat - response.amount_msat) / 1000
+                lnpayment.fee = float(response.amount_sent_msat -
+                                      response.amount_msat) / 1000
                 lnpayment.preimage = response.payment_preimage
                 lnpayment.save()
                 return True, None
-            elif response.status == 1: #PENDING
+            elif response.status == 1:  # PENDING
                 failure_reason = str("PENDING")
                 lnpayment.failure_reason = failure_reason
                 lnpayment.status = LNPayment.Status.FLIGHT
                 lnpayment.save()
                 return False, failure_reason
-            else: # status == 2 FAILED
+            else:  # status == 2 FAILED
                 failure_reason = str("FAILED")
                 lnpayment.failure_reason = failure_reason
                 lnpayment.status = LNPayment.Status.FAILRO
@@ -393,12 +399,11 @@ def pay_invoice(cls, lnpayment):
         except grpc._channel._InactiveRpcError as e:
             status_code = int(e.details().split('code: Some(')[1].split(')')[0])
             failure_reason = cls.payment_failure_context[status_code]
-            lnpayment.failure_reason = status_code # or failure_reason ?
+            lnpayment.failure_reason = status_code  # or failure_reason ?
             lnpayment.status = LNPayment.Status.FAILRO
             lnpayment.save()
             return False, failure_reason
 
-
     @classmethod
     def double_check_htlc_is_settled(cls, payment_hash):
         """Just as it sounds. Better safe than sorry!"""
@@ -408,7 +413,7 @@ def double_check_htlc_is_settled(cls, payment_hash):
         return (
             response.status == 1
         )   # CLN states: UNPAID = 0, PAID = 1, EXPIRED = 2, this is clns own invoice-lookup
-            # so just a check for paid/unpaid/expired not hodl-invoice related states like ACCEPTED/CANCELED
+        # so just a check for paid/unpaid/expired not hodl-invoice related states like ACCEPTED/CANCELED
 
     @classmethod
     def get_cln_version(cls):
@@ -435,14 +440,16 @@ def follow_send_payment(cls, hash):
 
         # Default is 0ppm. Set by the user over API. Client's default is 1000 ppm.
         fee_limit_sat = int(
-            float(lnpayment.num_satoshis) * float(lnpayment.routing_budget_ppm) / 1000000
+            float(lnpayment.num_satoshis) *
+            float(lnpayment.routing_budget_ppm) / 1000000
         )
         timeout_seconds = int(config("PAYOUT_TIMEOUT_SECONDS"))
 
         request = noderpc.PayRequest(
             bolt11=lnpayment.invoice,
             maxfee=fee_limit_sat,
-            retry_for=timeout_seconds, #retry_for is not quite the same as a timeout. Pay can still take SIGNIFICANTLY longer to return if htlcs are stuck!
+            # retry_for is not quite the same as a timeout. Pay can still take SIGNIFICANTLY longer to return if htlcs are stuck!
+            retry_for=timeout_seconds,
             # allow_self_payment=True, No such thing in pay command and self_payments do not work with pay!
         )
 
@@ -473,7 +480,7 @@ def handle_response(response, was_in_transit=False):
                 lnpayment.status = LNPayment.Status.FAILRO
                 lnpayment.last_routing_time = timezone.now()
                 lnpayment.routing_attempts += 1
-                lnpayment.failure_reason = -1 #no failure_reason in non-error pay response with stauts FAILED
+                lnpayment.failure_reason = -1  # no failure_reason in non-error pay response with stauts FAILED
                 lnpayment.in_flight = False
                 if lnpayment.routing_attempts > 2:
                     lnpayment.status = LNPayment.Status.EXPIRE
@@ -496,7 +503,8 @@ def handle_response(response, was_in_transit=False):
             if response.status == 0:  # Status 2 'COMPLETE'
                 print(f"Order: {order.id} SUCCEEDED. Hash: {hash}")
                 lnpayment.status = LNPayment.Status.SUCCED
-                lnpayment.fee = float(response.amount_sent_msat.msat - response.amount_msat.msat) / 1000
+                lnpayment.fee = float(response.amount_sent_msat.msat -
+                                      response.amount_msat.msat) / 1000
                 lnpayment.preimage = response.payment_preimage
                 lnpayment.save()
                 order.status = Order.Status.SUC
@@ -514,7 +522,7 @@ def handle_response(response, was_in_transit=False):
         except grpc._channel._InactiveRpcError as e:
             if "code: Some" in str(e):
                 status_code = int(e.details().split('code: Some(')[1].split(')')[0])
-                if status_code == 201: #Already paid with this hash using different amount or destination
+                if status_code == 201:  # Already paid with this hash using different amount or destination
                     # i don't think this can happen really, since we don't use the amount_msat in request and if you just try 'pay' 2x where the first time it succeeds you get the same non-error result the 2nd time.
                     # Listpays has some different fields as pay aswell, so not sure this makes sense
                     print(f"Order: {order.id} ALREADY PAID. Hash: {hash}.")
@@ -525,7 +533,8 @@ def handle_response(response, was_in_transit=False):
 
                     for response in cls.stub.ListPays(request):
                         handle_response(response)
-                elif status_code == 203 or status_code == 205 or status_code == 206:  #Permanent failure at destination. or Unable to find a route. or Route too expensive.
+                # Permanent failure at destination. or Unable to find a route. or Route too expensive.
+                elif status_code == 203 or status_code == 205 or status_code == 206:
                     lnpayment.status = LNPayment.Status.FAILRO
                     lnpayment.last_routing_time = timezone.now()
                     lnpayment.routing_attempts += 1
@@ -548,7 +557,7 @@ def handle_response(response, was_in_transit=False):
                         "succeded": False,
                         "context": f"payment failure reason: {cls.payment_failure_context[status_code]}",
                     }
-                elif status_code == 207:  #invoice expired
+                elif status_code == 207:  # invoice expired
                     print(f"Order: {order.id}. INVOICE EXPIRED. Hash: {hash}")
                     lnpayment.status = LNPayment.Status.EXPIRE
                     lnpayment.last_routing_time = timezone.now()
@@ -559,9 +568,10 @@ def handle_response(response, was_in_transit=False):
                         seconds=order.t_to_expire(Order.Status.FAI)
                     )
                     order.save()
-                    results = {"succeded": False, "context": "The payout invoice has expired"}
+                    results = {"succeded": False,
+                               "context": "The payout invoice has expired"}
                     return results
-                else: #-1 and 210 (don't know when 210 happens exactly)
+                else:  # -1 and 210 (don't know when 210 happens exactly)
                     print(str(e))
             else:
                 print(str(e))

From e0f406ccb84544929f61e2d8338dfdf140c63686 Mon Sep 17 00:00:00 2001
From: daywalker90 <admin@noserver4u.de>
Date: Tue, 18 Apr 2023 16:14:36 +0200
Subject: [PATCH 11/18] flake8 fixes

---
 api/lightning/node_cln.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/api/lightning/node_cln.py b/api/lightning/node_cln.py
index a0fad9e50..13e9e6f88 100755
--- a/api/lightning/node_cln.py
+++ b/api/lightning/node_cln.py
@@ -2,7 +2,6 @@
 import os
 import secrets
 import time
-from base64 import b64decode
 from datetime import datetime, timedelta
 from celery import shared_task
 
@@ -13,8 +12,8 @@
 
 from . import node_pb2 as noderpc
 from . import node_pb2_grpc as nodestub
-from . import primitives_pb2 as primitivesrpc
-from . import primitives_pb2_grpc as primitivesstub
+# from . import primitives_pb2 as primitivesrpc
+# from . import primitives_pb2_grpc as primitivesstub
 
 #######
 # Works with CLN

From 2820a39b2f669c6ecfde7a2c6d7eff59d6861b27 Mon Sep 17 00:00:00 2001
From: daywalker90 <admin@noserver4u.de>
Date: Sun, 23 Apr 2023 01:24:10 +0200
Subject: [PATCH 12/18] renaming cln file, class and get_version

---
 api/lightning/cln.py      | 577 ++++++++++++++++++++++++++++++++++++-
 api/lightning/node_cln.py | 579 --------------------------------------
 2 files changed, 576 insertions(+), 580 deletions(-)
 mode change 100644 => 100755 api/lightning/cln.py
 delete mode 100755 api/lightning/node_cln.py

diff --git a/api/lightning/cln.py b/api/lightning/cln.py
old mode 100644
new mode 100755
index 12cd7b593..c05144b7f
--- a/api/lightning/cln.py
+++ b/api/lightning/cln.py
@@ -1,4 +1,579 @@
+import hashlib
+import os
+import secrets
+import time
+from datetime import datetime, timedelta
+from celery import shared_task
+
+import grpc
+import ring
+from decouple import config
+from django.utils import timezone
+
+from . import node_pb2 as noderpc
+from . import node_pb2_grpc as nodestub
+# from . import primitives_pb2 as primitivesrpc
+# from . import primitives_pb2_grpc as primitivesstub
+
+#######
+# Works with CLN
+#######
+
+# Load the client's certificate and key
+with open(os.path.join(config("CLN_DIR"), 'client.pem'), 'rb') as f:
+    client_cert = f.read()
+with open(os.path.join(config("CLN_DIR"), 'client-key.pem'), 'rb') as f:
+    client_key = f.read()
+
+# Load the server's certificate
+with open(os.path.join(config("CLN_DIR"), 'server.pem'), 'rb') as f:
+    server_cert = f.read()
+
+
+CLN_GRPC_HOST = config("CLN_GRPC_HOST")
+DISABLE_ONCHAIN = config("DISABLE_ONCHAIN", cast=bool, default=True)
+MAX_SWAP_AMOUNT = config("MAX_SWAP_AMOUNT", cast=int, default=500000)
+
+
 class CLNNode:
+
+    os.environ["GRPC_SSL_CIPHER_SUITES"] = "HIGH+ECDSA"
+
+    # Create the SSL credentials object
+    creds = grpc.ssl_channel_credentials(
+        root_certificates=server_cert, private_key=client_key, certificate_chain=client_cert)
+    # Create the gRPC channel using the SSL credentials
+    channel = grpc.secure_channel(CLN_GRPC_HOST, creds)
+
+    # Create the gRPC stub
+    stub = nodestub.NodeStub(channel)
+
+    noderpc = noderpc
+    # invoicesrpc = invoicesrpc
+    # routerrpc = routerrpc
+
+    payment_failure_context = {
+        -1: "Catchall nonspecific error.",
+        201: "Already paid with this hash using different amount or destination.",
+        203: "Permanent failure at destination.",
+        205: "Unable to find a route.",
+        206: "Route too expensive.",
+        207: "Invoice expired.",
+        210: "Payment timed out without a payment in progress.",
+    }
+
+    @classmethod
+    def decode_payreq(cls, invoice):
+        """Decodes a lightning payment request (invoice)"""
+        request = noderpc.DecodeBolt11Request(bolt11=invoice)
+
+        response = cls.stub.DecodeBolt11(request)
+        return response
+
+    @classmethod
+    def estimate_fee(cls, amount_sats, target_conf=2, min_confs=1):
+        """Returns estimated fee for onchain payouts"""
+        # feerate estimaes work a bit differently in cln see https://lightning.readthedocs.io/lightning-feerates.7.html
+        request = noderpc.FeeratesRequest(style="PERKB")
+
+        response = cls.stub.Feerates(request)
+
+        # "opening" -> ~12 block target
+        return {
+            "mining_fee_sats": response.onchain_fee_estimates.opening_channel_satoshis,
+            "mining_fee_rate": response.perkb.opening/1000,
+        }
+
+    wallet_balance_cache = {}
+
+    @ring.dict(wallet_balance_cache, expire=10)  # keeps in cache for 10 seconds
+    @classmethod
+    def wallet_balance(cls):
+        """Returns onchain balance"""
+        request = noderpc.ListfundsRequest()
+
+        response = cls.stub.ListFunds(request)
+
+        unconfirmed_balance = 0
+        confirmed_balance = 0
+        total_balance = 0
+        for utxo in response.outputs:
+            if not utxo.reserved:
+                if utxo.status == 0:  # UNCONFIRMED
+                    unconfirmed_balance += utxo.amount_msat.msat // 1_000
+                    total_balance += utxo.amount_msat.msat // 1_000
+                elif utxo.status == 1:  # CONFIRMED
+                    confirmed_balance += utxo.amount_msat.msat // 1_000
+                    total_balance += utxo.amount_msat.msat // 1_000
+
+        return {
+            "total_balance": total_balance,
+            "confirmed_balance": confirmed_balance,
+            "unconfirmed_balance": unconfirmed_balance,
+        }
+
+    channel_balance_cache = {}
+
+    @ring.dict(channel_balance_cache, expire=10)  # keeps in cache for 10 seconds
+    @classmethod
+    def channel_balance(cls):
+        """Returns channels balance"""
+        request = noderpc.ListpeersRequest()
+
+        response = cls.stub.ListPeers(request)
+
+        local_balance_sat = 0
+        remote_balance_sat = 0
+        unsettled_local_balance = 0
+        unsettled_remote_balance = 0
+        for peer in response.peers:
+            for channel in peer.channels:
+                if channel.state == 2:  # CHANNELD_NORMAL
+                    local_balance_sat += channel.to_us_msat.msat // 1_000
+                    remote_balance_sat += (channel.total_msat.msat -
+                                           channel.to_us_msat.msat) // 1_000
+                for htlc in channel.htlcs:
+                    if htlc.direction == 0:  # IN
+                        unsettled_local_balance += htlc.amount_msat // 1_000
+                    elif htlc.direction == 1:  # OUT
+                        unsettled_remote_balance += htlc.amount_msat // 1_000
+
+        return {
+            "local_balance": local_balance_sat,
+            "remote_balance": remote_balance_sat,
+            "unsettled_local_balance": unsettled_local_balance,
+            "unsettled_remote_balance": unsettled_remote_balance,
+        }
+
+    @classmethod
+    def pay_onchain(cls, onchainpayment, queue_code=5, on_mempool_code=2):
+        """Send onchain transaction for buyer payouts"""
+
+        if DISABLE_ONCHAIN or onchainpayment.sent_satoshis > MAX_SWAP_AMOUNT:
+            return False
+
+        request = noderpc.WithdrawRequest(
+            destination=onchainpayment.address, satoshi=int(
+                onchainpayment.sent_satoshis),
+            feerate=str(int(onchainpayment.mining_fee_rate)*1_000)+"perkb",
+            minconf=int(not config("SPEND_UNCONFIRMED", default=False, cast=bool))
+        )
+
+        # Cheap security measure to ensure there has been some non-deterministic time between request and DB check
+        delay = (
+            secrets.randbelow(2**256) / (2**256) * 10
+        )  # Random uniform 0 to 5 secs with good entropy
+        time.sleep(3 + delay)
+
+        if onchainpayment.status == queue_code:
+            # Changing the state to "MEMPO" should be atomic with SendCoins.
+            onchainpayment.status = on_mempool_code
+            onchainpayment.save()
+            response = cls.stub.Withdraw(request)
+
+            if response.txid:
+                onchainpayment.txid = response.txid
+                onchainpayment.broadcasted = True
+            onchainpayment.save()
+            return True
+
+        elif onchainpayment.status == on_mempool_code:
+            # Bug, double payment attempted
+            return True
+
+    @classmethod
+    def cancel_return_hold_invoice(cls, payment_hash):
+        """Cancels or returns a hold invoice"""
+        request = noderpc.HodlInvoiceCancelRequest(
+            payment_hash=bytes.fromhex(payment_hash))
+        response = cls.stub.HodlInvoiceCancel(request)
+
+        return response.state == 1  # True if state is CANCELED, false otherwise.
+
+    @classmethod
+    def settle_hold_invoice(cls, preimage):
+        """settles a hold invoice"""
+        request = noderpc.HodlInvoiceSettleRequest(
+            payment_hash=hashlib.sha256(bytes.fromhex(preimage)).digest())
+        response = cls.stub.HodlInvoiceSettle(request)
+
+        return response.state == 2  # True if state is SETTLED, false otherwise.
+
+    @classmethod
+    def gen_hold_invoice(
+        cls, num_satoshis, description, invoice_expiry, cltv_expiry_blocks, order_id, receiver_robot, time
+    ):
+        """Generates hold invoice"""
+
+        hold_payment = {}
+        # The preimage is a random hash of 256 bits entropy
+        preimage = hashlib.sha256(secrets.token_bytes(nbytes=32)).digest()
+
+        request = noderpc.InvoiceRequest(
+            description=description,
+            amount_msat=num_satoshis * 1_000,
+            label=str(order_id) + "_" + str(receiver_robot) + "_" + str(time),
+            expiry=int(
+                invoice_expiry * 1.5
+            ),  # actual expiry is padded by 50%, if tight, wrong client system clock will say invoice is expired.
+            cltv=cltv_expiry_blocks,
+            preimage=preimage,  # preimage is actually optional in cln, as cln would generate one by default
+        )
+        response = cls.stub.HodlInvoice(request)
+
+        hold_payment["invoice"] = response.bolt11
+        payreq_decoded = cls.decode_payreq(hold_payment["invoice"])
+        hold_payment["preimage"] = preimage.hex()
+        hold_payment["payment_hash"] = response.payment_hash
+        hold_payment["created_at"] = timezone.make_aware(
+            datetime.fromtimestamp(payreq_decoded.timestamp)
+        )
+        hold_payment["expires_at"] = response.expires_at
+        hold_payment["cltv_expiry"] = cltv_expiry_blocks
+
+        return hold_payment
+
+    @classmethod
+    def validate_hold_invoice_locked(cls, lnpayment):
+        """Checks if hold invoice is locked"""
+        from api.models import LNPayment
+
+        request = noderpc.HodlInvoiceLookupRequest(
+            payment_hash=bytes.fromhex(lnpayment.payment_hash)
+        )
+        response = cls.stub.HodlInvoiceLookup(request)
+
+        # Will fail if 'unable to locate invoice'. Happens if invoice expiry
+        # time has passed (but these are 15% padded at the moment). Should catch it
+        # and report back that the invoice has expired (better robustness)
+        if response.state == 0:  # OPEN
+            pass
+        if response.state == 1:  # SETTLED
+            pass
+        if response.state == 2:  # CANCELLED
+            pass
+        if response.state == 3:  # ACCEPTED (LOCKED)
+            lnpayment.expiry_height = response.htlc_cltv
+            lnpayment.status = LNPayment.Status.LOCKED
+            lnpayment.save()
+            return True
+
+    @classmethod
+    def resetmc(cls):
+        # don't think an equivalent exists for cln, maybe deleting gossip_store file?
+        return False
+
+    @classmethod
+    def validate_ln_invoice(cls, invoice, num_satoshis, routing_budget_ppm):
+        """Checks if the submited LN invoice comforms to expectations"""
+
+        payout = {
+            "valid": False,
+            "context": None,
+            "description": None,
+            "payment_hash": None,
+            "created_at": None,
+            "expires_at": None,
+        }
+
+        try:
+            payreq_decoded = cls.decode_payreq(invoice)
+        except Exception:
+            payout["context"] = {
+                "bad_invoice": "Does not look like a valid lightning invoice"
+            }
+            return payout
+
+        # Some wallet providers (e.g. Muun) force routing through a private channel with high fees >1500ppm
+        # These payments will fail. So it is best to let the user know in advance this invoice is not valid.
+        route_hints = payreq_decoded.route_hints.hints
+
+        # Max amount RoboSats will pay for routing
+        if routing_budget_ppm == 0:
+            max_routing_fee_sats = max(
+                num_satoshis * float(config("PROPORTIONAL_ROUTING_FEE_LIMIT")),
+                float(config("MIN_FLAT_ROUTING_FEE_LIMIT_REWARD")),
+            )
+        else:
+            max_routing_fee_sats = int(
+                float(num_satoshis) * float(routing_budget_ppm) / 1000000
+            )
+
+        if route_hints:
+            routes_cost = []
+            # For every hinted route...
+            for hinted_route in route_hints:
+                route_cost = 0
+                # ...add up the cost of every hinted hop...
+                for hop_hint in hinted_route.hops:
+                    route_cost += hop_hint.feebase.msat / 1_000
+                    route_cost += (
+                        hop_hint.feeprop * num_satoshis / 1_000_000
+                    )
+
+                # ...and store the cost of the route to the array
+                routes_cost.append(route_cost)
+
+            # If the cheapest possible private route is more expensive than what RoboSats is willing to pay
+            if min(routes_cost) >= max_routing_fee_sats:
+                payout["context"] = {
+                    "bad_invoice": "The invoice hinted private routes are not payable within the submitted routing budget."
+                }
+                return payout
+
+        if payreq_decoded.amount_msat == 0:
+            payout["context"] = {
+                "bad_invoice": "The invoice provided has no explicit amount"
+            }
+            return payout
+
+        if not payreq_decoded.amount_msat // 1_000 == num_satoshis:
+            payout["context"] = {
+                "bad_invoice": "The invoice provided is not for "
+                + "{:,}".format(num_satoshis)
+                + " Sats"
+            }
+            return payout
+
+        payout["created_at"] = timezone.make_aware(
+            datetime.fromtimestamp(payreq_decoded.timestamp)
+        )
+        payout["expires_at"] = payout["created_at"] + timedelta(
+            seconds=payreq_decoded.expiry
+        )
+
+        if payout["expires_at"] < timezone.now():
+            payout["context"] = {
+                "bad_invoice": "The invoice provided has already expired"
+            }
+            return payout
+
+        payout["valid"] = True
+        payout["description"] = payreq_decoded.description
+        payout["payment_hash"] = payreq_decoded.payment_hash
+
+        return payout
+
+    @classmethod
+    def pay_invoice(cls, lnpayment):
+        """Sends sats. Used for rewards payouts"""
+        from api.models import LNPayment
+
+        fee_limit_sat = int(
+            max(
+                lnpayment.num_satoshis
+                * float(config("PROPORTIONAL_ROUTING_FEE_LIMIT")),
+                float(config("MIN_FLAT_ROUTING_FEE_LIMIT_REWARD")),
+            )
+        )  # 200 ppm or 10 sats
+        timeout_seconds = int(config("REWARDS_TIMEOUT_SECONDS"))
+        request = noderpc.PayRequest(
+            bolt11=lnpayment.invoice,
+            maxfee=fee_limit_sat,
+            retry_for=timeout_seconds,
+        )
+
+        try:
+            response = cls.stub.Pay(request)
+
+            if response.status == 0:  # COMPLETE
+                lnpayment.status = LNPayment.Status.SUCCED
+                lnpayment.fee = float(response.amount_sent_msat -
+                                      response.amount_msat) / 1000
+                lnpayment.preimage = response.payment_preimage
+                lnpayment.save()
+                return True, None
+            elif response.status == 1:  # PENDING
+                failure_reason = str("PENDING")
+                lnpayment.failure_reason = failure_reason
+                lnpayment.status = LNPayment.Status.FLIGHT
+                lnpayment.save()
+                return False, failure_reason
+            else:  # status == 2 FAILED
+                failure_reason = str("FAILED")
+                lnpayment.failure_reason = failure_reason
+                lnpayment.status = LNPayment.Status.FAILRO
+                lnpayment.save()
+                return False, failure_reason
+        except grpc._channel._InactiveRpcError as e:
+            status_code = int(e.details().split('code: Some(')[1].split(')')[0])
+            failure_reason = cls.payment_failure_context[status_code]
+            lnpayment.failure_reason = status_code  # or failure_reason ?
+            lnpayment.status = LNPayment.Status.FAILRO
+            lnpayment.save()
+            return False, failure_reason
+
+    @classmethod
+    def double_check_htlc_is_settled(cls, payment_hash):
+        """Just as it sounds. Better safe than sorry!"""
+        request = noderpc.ListinvoicesRequest(payment_hash=bytes.fromhex(payment_hash))
+        response = cls.stub.ListInvoices(request)
+
+        return (
+            response.status == 1
+        )   # CLN states: UNPAID = 0, PAID = 1, EXPIRED = 2, this is clns own invoice-lookup
+        # so just a check for paid/unpaid/expired not hodl-invoice related states like ACCEPTED/CANCELED
+
     @classmethod
     def get_version(cls):
-        return None
+        request = noderpc.GetinfoRequest()
+        response = cls.stub.Getinfo(request)
+
+        return response.version
+
+    @classmethod
+    @shared_task(name="follow_send_payment", time_limit=180)
+    def follow_send_payment(cls, hash):
+        """Sends sats to buyer, continuous update"""
+
+        from datetime import timedelta
+
+        from decouple import config
+        from django.utils import timezone
+
+        from api.models import LNPayment, Order
+
+        lnpayment = LNPayment.objects.get(payment_hash=hash)
+        lnpayment.last_routing_time = timezone.now()
+        lnpayment.save()
+
+        # Default is 0ppm. Set by the user over API. Client's default is 1000 ppm.
+        fee_limit_sat = int(
+            float(lnpayment.num_satoshis) *
+            float(lnpayment.routing_budget_ppm) / 1000000
+        )
+        timeout_seconds = int(config("PAYOUT_TIMEOUT_SECONDS"))
+
+        request = noderpc.PayRequest(
+            bolt11=lnpayment.invoice,
+            maxfee=fee_limit_sat,
+            # retry_for is not quite the same as a timeout. Pay can still take SIGNIFICANTLY longer to return if htlcs are stuck!
+            retry_for=timeout_seconds,
+            # allow_self_payment=True, No such thing in pay command and self_payments do not work with pay!
+        )
+
+        order = lnpayment.order_paid_LN
+        if order.trade_escrow.num_satoshis < lnpayment.num_satoshis:
+            print(f"Order: {order.id} Payout is larger than collateral !?")
+            return
+
+        def handle_response(response, was_in_transit=False):
+            lnpayment.status = LNPayment.Status.FLIGHT
+            lnpayment.in_flight = True
+            lnpayment.save()
+            order.status = Order.Status.PAY
+            order.save()
+
+            if response.status == 1:  # Status 1 'PENDING'
+                print(f"Order: {order.id} IN_FLIGHT. Hash {hash}")
+
+                # If payment was already "payment is in transition" we do not
+                # want to spawn a new thread every 3 minutes to check on it.
+                # in case this thread dies, let's move the last_routing_time
+                # 20 minutes in the future so another thread spawns.
+                if was_in_transit:
+                    lnpayment.last_routing_time = timezone.now() + timedelta(minutes=20)
+                    lnpayment.save()
+
+            if response.status == 2:  # Status 3 'FAILED'
+                lnpayment.status = LNPayment.Status.FAILRO
+                lnpayment.last_routing_time = timezone.now()
+                lnpayment.routing_attempts += 1
+                lnpayment.failure_reason = -1  # no failure_reason in non-error pay response with stauts FAILED
+                lnpayment.in_flight = False
+                if lnpayment.routing_attempts > 2:
+                    lnpayment.status = LNPayment.Status.EXPIRE
+                    lnpayment.routing_attempts = 0
+                lnpayment.save()
+
+                order.status = Order.Status.FAI
+                order.expires_at = timezone.now() + timedelta(
+                    seconds=order.t_to_expire(Order.Status.FAI)
+                )
+                order.save()
+                print(
+                    f"Order: {order.id} FAILED. Hash: {hash} Reason: {cls.payment_failure_context[-1]}"
+                )
+                return {
+                    "succeded": False,
+                    "context": f"payment failure reason: {cls.payment_failure_context[-1]}",
+                }
+
+            if response.status == 0:  # Status 2 'COMPLETE'
+                print(f"Order: {order.id} SUCCEEDED. Hash: {hash}")
+                lnpayment.status = LNPayment.Status.SUCCED
+                lnpayment.fee = float(response.amount_sent_msat.msat -
+                                      response.amount_msat.msat) / 1000
+                lnpayment.preimage = response.payment_preimage
+                lnpayment.save()
+                order.status = Order.Status.SUC
+                order.expires_at = timezone.now() + timedelta(
+                    seconds=order.t_to_expire(Order.Status.SUC)
+                )
+                order.save()
+                results = {"succeded": True}
+                return results
+
+        try:
+            response = cls.stub.Pay(request)
+            handle_response(response)
+
+        except grpc._channel._InactiveRpcError as e:
+            if "code: Some" in str(e):
+                status_code = int(e.details().split('code: Some(')[1].split(')')[0])
+                if status_code == 201:  # Already paid with this hash using different amount or destination
+                    # i don't think this can happen really, since we don't use the amount_msat in request and if you just try 'pay' 2x where the first time it succeeds you get the same non-error result the 2nd time.
+                    # Listpays has some different fields as pay aswell, so not sure this makes sense
+                    print(f"Order: {order.id} ALREADY PAID. Hash: {hash}.")
+
+                    request = noderpc.ListpaysRequest(
+                        payment_hash=bytes.fromhex(hash), status="complete"
+                    )
+
+                    for response in cls.stub.ListPays(request):
+                        handle_response(response)
+                # Permanent failure at destination. or Unable to find a route. or Route too expensive.
+                elif status_code == 203 or status_code == 205 or status_code == 206:
+                    lnpayment.status = LNPayment.Status.FAILRO
+                    lnpayment.last_routing_time = timezone.now()
+                    lnpayment.routing_attempts += 1
+                    lnpayment.failure_reason = status_code
+                    lnpayment.in_flight = False
+                    if lnpayment.routing_attempts > 2:
+                        lnpayment.status = LNPayment.Status.EXPIRE
+                        lnpayment.routing_attempts = 0
+                    lnpayment.save()
+
+                    order.status = Order.Status.FAI
+                    order.expires_at = timezone.now() + timedelta(
+                        seconds=order.t_to_expire(Order.Status.FAI)
+                    )
+                    order.save()
+                    print(
+                        f"Order: {order.id} FAILED. Hash: {hash} Reason: {cls.payment_failure_context[status_code]}"
+                    )
+                    return {
+                        "succeded": False,
+                        "context": f"payment failure reason: {cls.payment_failure_context[status_code]}",
+                    }
+                elif status_code == 207:  # invoice expired
+                    print(f"Order: {order.id}. INVOICE EXPIRED. Hash: {hash}")
+                    lnpayment.status = LNPayment.Status.EXPIRE
+                    lnpayment.last_routing_time = timezone.now()
+                    lnpayment.in_flight = False
+                    lnpayment.save()
+                    order.status = Order.Status.FAI
+                    order.expires_at = timezone.now() + timedelta(
+                        seconds=order.t_to_expire(Order.Status.FAI)
+                    )
+                    order.save()
+                    results = {"succeded": False,
+                               "context": "The payout invoice has expired"}
+                    return results
+                else:  # -1 and 210 (don't know when 210 happens exactly)
+                    print(str(e))
+            else:
+                print(str(e))
+
+        except Exception as e:
+            print(str(e))
diff --git a/api/lightning/node_cln.py b/api/lightning/node_cln.py
deleted file mode 100755
index 13e9e6f88..000000000
--- a/api/lightning/node_cln.py
+++ /dev/null
@@ -1,579 +0,0 @@
-import hashlib
-import os
-import secrets
-import time
-from datetime import datetime, timedelta
-from celery import shared_task
-
-import grpc
-import ring
-from decouple import config
-from django.utils import timezone
-
-from . import node_pb2 as noderpc
-from . import node_pb2_grpc as nodestub
-# from . import primitives_pb2 as primitivesrpc
-# from . import primitives_pb2_grpc as primitivesstub
-
-#######
-# Works with CLN
-#######
-
-# Load the client's certificate and key
-with open(os.path.join(config("CLN_DIR"), 'client.pem'), 'rb') as f:
-    client_cert = f.read()
-with open(os.path.join(config("CLN_DIR"), 'client-key.pem'), 'rb') as f:
-    client_key = f.read()
-
-# Load the server's certificate
-with open(os.path.join(config("CLN_DIR"), 'server.pem'), 'rb') as f:
-    server_cert = f.read()
-
-
-CLN_GRPC_HOST = config("CLN_GRPC_HOST")
-DISABLE_ONCHAIN = config("DISABLE_ONCHAIN", cast=bool, default=True)
-MAX_SWAP_AMOUNT = config("MAX_SWAP_AMOUNT", cast=int, default=500000)
-
-
-class LNNode:
-
-    os.environ["GRPC_SSL_CIPHER_SUITES"] = "HIGH+ECDSA"
-
-    # Create the SSL credentials object
-    creds = grpc.ssl_channel_credentials(
-        root_certificates=server_cert, private_key=client_key, certificate_chain=client_cert)
-    # Create the gRPC channel using the SSL credentials
-    channel = grpc.secure_channel(CLN_GRPC_HOST, creds)
-
-    # Create the gRPC stub
-    stub = nodestub.NodeStub(channel)
-
-    noderpc = noderpc
-    # invoicesrpc = invoicesrpc
-    # routerrpc = routerrpc
-
-    payment_failure_context = {
-        -1: "Catchall nonspecific error.",
-        201: "Already paid with this hash using different amount or destination.",
-        203: "Permanent failure at destination.",
-        205: "Unable to find a route.",
-        206: "Route too expensive.",
-        207: "Invoice expired.",
-        210: "Payment timed out without a payment in progress.",
-    }
-
-    @classmethod
-    def decode_payreq(cls, invoice):
-        """Decodes a lightning payment request (invoice)"""
-        request = noderpc.DecodeBolt11Request(bolt11=invoice)
-
-        response = cls.stub.DecodeBolt11(request)
-        return response
-
-    @classmethod
-    def estimate_fee(cls, amount_sats, target_conf=2, min_confs=1):
-        """Returns estimated fee for onchain payouts"""
-        # feerate estimaes work a bit differently in cln see https://lightning.readthedocs.io/lightning-feerates.7.html
-        request = noderpc.FeeratesRequest(style="PERKB")
-
-        response = cls.stub.Feerates(request)
-
-        # "opening" -> ~12 block target
-        return {
-            "mining_fee_sats": response.onchain_fee_estimates.opening_channel_satoshis,
-            "mining_fee_rate": response.perkb.opening/1000,
-        }
-
-    wallet_balance_cache = {}
-
-    @ring.dict(wallet_balance_cache, expire=10)  # keeps in cache for 10 seconds
-    @classmethod
-    def wallet_balance(cls):
-        """Returns onchain balance"""
-        request = noderpc.ListfundsRequest()
-
-        response = cls.stub.ListFunds(request)
-
-        unconfirmed_balance = 0
-        confirmed_balance = 0
-        total_balance = 0
-        for utxo in response.outputs:
-            if not utxo.reserved:
-                if utxo.status == 0:  # UNCONFIRMED
-                    unconfirmed_balance += utxo.amount_msat.msat // 1_000
-                    total_balance += utxo.amount_msat.msat // 1_000
-                elif utxo.status == 1:  # CONFIRMED
-                    confirmed_balance += utxo.amount_msat.msat // 1_000
-                    total_balance += utxo.amount_msat.msat // 1_000
-
-        return {
-            "total_balance": total_balance,
-            "confirmed_balance": confirmed_balance,
-            "unconfirmed_balance": unconfirmed_balance,
-        }
-
-    channel_balance_cache = {}
-
-    @ring.dict(channel_balance_cache, expire=10)  # keeps in cache for 10 seconds
-    @classmethod
-    def channel_balance(cls):
-        """Returns channels balance"""
-        request = noderpc.ListpeersRequest()
-
-        response = cls.stub.ListPeers(request)
-
-        local_balance_sat = 0
-        remote_balance_sat = 0
-        unsettled_local_balance = 0
-        unsettled_remote_balance = 0
-        for peer in response.peers:
-            for channel in peer.channels:
-                if channel.state == 2:  # CHANNELD_NORMAL
-                    local_balance_sat += channel.to_us_msat.msat // 1_000
-                    remote_balance_sat += (channel.total_msat.msat -
-                                           channel.to_us_msat.msat) // 1_000
-                for htlc in channel.htlcs:
-                    if htlc.direction == 0:  # IN
-                        unsettled_local_balance += htlc.amount_msat // 1_000
-                    elif htlc.direction == 1:  # OUT
-                        unsettled_remote_balance += htlc.amount_msat // 1_000
-
-        return {
-            "local_balance": local_balance_sat,
-            "remote_balance": remote_balance_sat,
-            "unsettled_local_balance": unsettled_local_balance,
-            "unsettled_remote_balance": unsettled_remote_balance,
-        }
-
-    @classmethod
-    def pay_onchain(cls, onchainpayment, queue_code=5, on_mempool_code=2):
-        """Send onchain transaction for buyer payouts"""
-
-        if DISABLE_ONCHAIN or onchainpayment.sent_satoshis > MAX_SWAP_AMOUNT:
-            return False
-
-        request = noderpc.WithdrawRequest(
-            destination=onchainpayment.address, satoshi=int(
-                onchainpayment.sent_satoshis),
-            feerate=str(int(onchainpayment.mining_fee_rate)*1_000)+"perkb",
-            minconf=int(not config("SPEND_UNCONFIRMED", default=False, cast=bool))
-        )
-
-        # Cheap security measure to ensure there has been some non-deterministic time between request and DB check
-        delay = (
-            secrets.randbelow(2**256) / (2**256) * 10
-        )  # Random uniform 0 to 5 secs with good entropy
-        time.sleep(3 + delay)
-
-        if onchainpayment.status == queue_code:
-            # Changing the state to "MEMPO" should be atomic with SendCoins.
-            onchainpayment.status = on_mempool_code
-            onchainpayment.save()
-            response = cls.stub.Withdraw(request)
-
-            if response.txid:
-                onchainpayment.txid = response.txid
-                onchainpayment.broadcasted = True
-            onchainpayment.save()
-            return True
-
-        elif onchainpayment.status == on_mempool_code:
-            # Bug, double payment attempted
-            return True
-
-    @classmethod
-    def cancel_return_hold_invoice(cls, payment_hash):
-        """Cancels or returns a hold invoice"""
-        request = noderpc.HodlInvoiceCancelRequest(
-            payment_hash=bytes.fromhex(payment_hash))
-        response = cls.stub.HodlInvoiceCancel(request)
-
-        return response.state == 1  # True if state is CANCELED, false otherwise.
-
-    @classmethod
-    def settle_hold_invoice(cls, preimage):
-        """settles a hold invoice"""
-        request = noderpc.HodlInvoiceSettleRequest(
-            payment_hash=hashlib.sha256(bytes.fromhex(preimage)).digest())
-        response = cls.stub.HodlInvoiceSettle(request)
-
-        return response.state == 2  # True if state is SETTLED, false otherwise.
-
-    @classmethod
-    def gen_hold_invoice(
-        cls, num_satoshis, description, invoice_expiry, cltv_expiry_blocks, order_id, receiver_robot, time
-    ):
-        """Generates hold invoice"""
-
-        hold_payment = {}
-        # The preimage is a random hash of 256 bits entropy
-        preimage = hashlib.sha256(secrets.token_bytes(nbytes=32)).digest()
-
-        request = noderpc.InvoiceRequest(
-            description=description,
-            amount_msat=num_satoshis * 1_000,
-            label=str(order_id) + "_" + str(receiver_robot) + "_" + str(time),
-            expiry=int(
-                invoice_expiry * 1.5
-            ),  # actual expiry is padded by 50%, if tight, wrong client system clock will say invoice is expired.
-            cltv=cltv_expiry_blocks,
-            preimage=preimage,  # preimage is actually optional in cln, as cln would generate one by default
-        )
-        response = cls.stub.HodlInvoice(request)
-
-        hold_payment["invoice"] = response.bolt11
-        payreq_decoded = cls.decode_payreq(hold_payment["invoice"])
-        hold_payment["preimage"] = preimage.hex()
-        hold_payment["payment_hash"] = response.payment_hash
-        hold_payment["created_at"] = timezone.make_aware(
-            datetime.fromtimestamp(payreq_decoded.timestamp)
-        )
-        hold_payment["expires_at"] = response.expires_at
-        hold_payment["cltv_expiry"] = cltv_expiry_blocks
-
-        return hold_payment
-
-    @classmethod
-    def validate_hold_invoice_locked(cls, lnpayment):
-        """Checks if hold invoice is locked"""
-        from api.models import LNPayment
-
-        request = noderpc.HodlInvoiceLookupRequest(
-            payment_hash=bytes.fromhex(lnpayment.payment_hash)
-        )
-        response = cls.stub.HodlInvoiceLookup(request)
-
-        # Will fail if 'unable to locate invoice'. Happens if invoice expiry
-        # time has passed (but these are 15% padded at the moment). Should catch it
-        # and report back that the invoice has expired (better robustness)
-        if response.state == 0:  # OPEN
-            pass
-        if response.state == 1:  # SETTLED
-            pass
-        if response.state == 2:  # CANCELLED
-            pass
-        if response.state == 3:  # ACCEPTED (LOCKED)
-            lnpayment.expiry_height = response.htlc_cltv
-            lnpayment.status = LNPayment.Status.LOCKED
-            lnpayment.save()
-            return True
-
-    @classmethod
-    def resetmc(cls):
-        # don't think an equivalent exists for cln, maybe deleting gossip_store file?
-        return False
-
-    @classmethod
-    def validate_ln_invoice(cls, invoice, num_satoshis, routing_budget_ppm):
-        """Checks if the submited LN invoice comforms to expectations"""
-
-        payout = {
-            "valid": False,
-            "context": None,
-            "description": None,
-            "payment_hash": None,
-            "created_at": None,
-            "expires_at": None,
-        }
-
-        try:
-            payreq_decoded = cls.decode_payreq(invoice)
-        except Exception:
-            payout["context"] = {
-                "bad_invoice": "Does not look like a valid lightning invoice"
-            }
-            return payout
-
-        # Some wallet providers (e.g. Muun) force routing through a private channel with high fees >1500ppm
-        # These payments will fail. So it is best to let the user know in advance this invoice is not valid.
-        route_hints = payreq_decoded.route_hints.hints
-
-        # Max amount RoboSats will pay for routing
-        if routing_budget_ppm == 0:
-            max_routing_fee_sats = max(
-                num_satoshis * float(config("PROPORTIONAL_ROUTING_FEE_LIMIT")),
-                float(config("MIN_FLAT_ROUTING_FEE_LIMIT_REWARD")),
-            )
-        else:
-            max_routing_fee_sats = int(
-                float(num_satoshis) * float(routing_budget_ppm) / 1000000
-            )
-
-        if route_hints:
-            routes_cost = []
-            # For every hinted route...
-            for hinted_route in route_hints:
-                route_cost = 0
-                # ...add up the cost of every hinted hop...
-                for hop_hint in hinted_route.hops:
-                    route_cost += hop_hint.feebase.msat / 1_000
-                    route_cost += (
-                        hop_hint.feeprop * num_satoshis / 1_000_000
-                    )
-
-                # ...and store the cost of the route to the array
-                routes_cost.append(route_cost)
-
-            # If the cheapest possible private route is more expensive than what RoboSats is willing to pay
-            if min(routes_cost) >= max_routing_fee_sats:
-                payout["context"] = {
-                    "bad_invoice": "The invoice hinted private routes are not payable within the submitted routing budget."
-                }
-                return payout
-
-        if payreq_decoded.amount_msat == 0:
-            payout["context"] = {
-                "bad_invoice": "The invoice provided has no explicit amount"
-            }
-            return payout
-
-        if not payreq_decoded.amount_msat // 1_000 == num_satoshis:
-            payout["context"] = {
-                "bad_invoice": "The invoice provided is not for "
-                + "{:,}".format(num_satoshis)
-                + " Sats"
-            }
-            return payout
-
-        payout["created_at"] = timezone.make_aware(
-            datetime.fromtimestamp(payreq_decoded.timestamp)
-        )
-        payout["expires_at"] = payout["created_at"] + timedelta(
-            seconds=payreq_decoded.expiry
-        )
-
-        if payout["expires_at"] < timezone.now():
-            payout["context"] = {
-                "bad_invoice": "The invoice provided has already expired"
-            }
-            return payout
-
-        payout["valid"] = True
-        payout["description"] = payreq_decoded.description
-        payout["payment_hash"] = payreq_decoded.payment_hash
-
-        return payout
-
-    @classmethod
-    def pay_invoice(cls, lnpayment):
-        """Sends sats. Used for rewards payouts"""
-        from api.models import LNPayment
-
-        fee_limit_sat = int(
-            max(
-                lnpayment.num_satoshis
-                * float(config("PROPORTIONAL_ROUTING_FEE_LIMIT")),
-                float(config("MIN_FLAT_ROUTING_FEE_LIMIT_REWARD")),
-            )
-        )  # 200 ppm or 10 sats
-        timeout_seconds = int(config("REWARDS_TIMEOUT_SECONDS"))
-        request = noderpc.PayRequest(
-            bolt11=lnpayment.invoice,
-            maxfee=fee_limit_sat,
-            retry_for=timeout_seconds,
-        )
-
-        try:
-            response = cls.stub.Pay(request)
-
-            if response.status == 0:  # COMPLETE
-                lnpayment.status = LNPayment.Status.SUCCED
-                lnpayment.fee = float(response.amount_sent_msat -
-                                      response.amount_msat) / 1000
-                lnpayment.preimage = response.payment_preimage
-                lnpayment.save()
-                return True, None
-            elif response.status == 1:  # PENDING
-                failure_reason = str("PENDING")
-                lnpayment.failure_reason = failure_reason
-                lnpayment.status = LNPayment.Status.FLIGHT
-                lnpayment.save()
-                return False, failure_reason
-            else:  # status == 2 FAILED
-                failure_reason = str("FAILED")
-                lnpayment.failure_reason = failure_reason
-                lnpayment.status = LNPayment.Status.FAILRO
-                lnpayment.save()
-                return False, failure_reason
-        except grpc._channel._InactiveRpcError as e:
-            status_code = int(e.details().split('code: Some(')[1].split(')')[0])
-            failure_reason = cls.payment_failure_context[status_code]
-            lnpayment.failure_reason = status_code  # or failure_reason ?
-            lnpayment.status = LNPayment.Status.FAILRO
-            lnpayment.save()
-            return False, failure_reason
-
-    @classmethod
-    def double_check_htlc_is_settled(cls, payment_hash):
-        """Just as it sounds. Better safe than sorry!"""
-        request = noderpc.ListinvoicesRequest(payment_hash=bytes.fromhex(payment_hash))
-        response = cls.stub.ListInvoices(request)
-
-        return (
-            response.status == 1
-        )   # CLN states: UNPAID = 0, PAID = 1, EXPIRED = 2, this is clns own invoice-lookup
-        # so just a check for paid/unpaid/expired not hodl-invoice related states like ACCEPTED/CANCELED
-
-    @classmethod
-    def get_cln_version(cls):
-        request = noderpc.GetinfoRequest()
-        response = cls.stub.Getinfo(request)
-
-        return response.version
-
-    @classmethod
-    @shared_task(name="follow_send_payment", time_limit=180)
-    def follow_send_payment(cls, hash):
-        """Sends sats to buyer, continuous update"""
-
-        from datetime import timedelta
-
-        from decouple import config
-        from django.utils import timezone
-
-        from api.models import LNPayment, Order
-
-        lnpayment = LNPayment.objects.get(payment_hash=hash)
-        lnpayment.last_routing_time = timezone.now()
-        lnpayment.save()
-
-        # Default is 0ppm. Set by the user over API. Client's default is 1000 ppm.
-        fee_limit_sat = int(
-            float(lnpayment.num_satoshis) *
-            float(lnpayment.routing_budget_ppm) / 1000000
-        )
-        timeout_seconds = int(config("PAYOUT_TIMEOUT_SECONDS"))
-
-        request = noderpc.PayRequest(
-            bolt11=lnpayment.invoice,
-            maxfee=fee_limit_sat,
-            # retry_for is not quite the same as a timeout. Pay can still take SIGNIFICANTLY longer to return if htlcs are stuck!
-            retry_for=timeout_seconds,
-            # allow_self_payment=True, No such thing in pay command and self_payments do not work with pay!
-        )
-
-        order = lnpayment.order_paid_LN
-        if order.trade_escrow.num_satoshis < lnpayment.num_satoshis:
-            print(f"Order: {order.id} Payout is larger than collateral !?")
-            return
-
-        def handle_response(response, was_in_transit=False):
-            lnpayment.status = LNPayment.Status.FLIGHT
-            lnpayment.in_flight = True
-            lnpayment.save()
-            order.status = Order.Status.PAY
-            order.save()
-
-            if response.status == 1:  # Status 1 'PENDING'
-                print(f"Order: {order.id} IN_FLIGHT. Hash {hash}")
-
-                # If payment was already "payment is in transition" we do not
-                # want to spawn a new thread every 3 minutes to check on it.
-                # in case this thread dies, let's move the last_routing_time
-                # 20 minutes in the future so another thread spawns.
-                if was_in_transit:
-                    lnpayment.last_routing_time = timezone.now() + timedelta(minutes=20)
-                    lnpayment.save()
-
-            if response.status == 2:  # Status 3 'FAILED'
-                lnpayment.status = LNPayment.Status.FAILRO
-                lnpayment.last_routing_time = timezone.now()
-                lnpayment.routing_attempts += 1
-                lnpayment.failure_reason = -1  # no failure_reason in non-error pay response with stauts FAILED
-                lnpayment.in_flight = False
-                if lnpayment.routing_attempts > 2:
-                    lnpayment.status = LNPayment.Status.EXPIRE
-                    lnpayment.routing_attempts = 0
-                lnpayment.save()
-
-                order.status = Order.Status.FAI
-                order.expires_at = timezone.now() + timedelta(
-                    seconds=order.t_to_expire(Order.Status.FAI)
-                )
-                order.save()
-                print(
-                    f"Order: {order.id} FAILED. Hash: {hash} Reason: {cls.payment_failure_context[-1]}"
-                )
-                return {
-                    "succeded": False,
-                    "context": f"payment failure reason: {cls.payment_failure_context[-1]}",
-                }
-
-            if response.status == 0:  # Status 2 'COMPLETE'
-                print(f"Order: {order.id} SUCCEEDED. Hash: {hash}")
-                lnpayment.status = LNPayment.Status.SUCCED
-                lnpayment.fee = float(response.amount_sent_msat.msat -
-                                      response.amount_msat.msat) / 1000
-                lnpayment.preimage = response.payment_preimage
-                lnpayment.save()
-                order.status = Order.Status.SUC
-                order.expires_at = timezone.now() + timedelta(
-                    seconds=order.t_to_expire(Order.Status.SUC)
-                )
-                order.save()
-                results = {"succeded": True}
-                return results
-
-        try:
-            response = cls.stub.Pay(request)
-            handle_response(response)
-
-        except grpc._channel._InactiveRpcError as e:
-            if "code: Some" in str(e):
-                status_code = int(e.details().split('code: Some(')[1].split(')')[0])
-                if status_code == 201:  # Already paid with this hash using different amount or destination
-                    # i don't think this can happen really, since we don't use the amount_msat in request and if you just try 'pay' 2x where the first time it succeeds you get the same non-error result the 2nd time.
-                    # Listpays has some different fields as pay aswell, so not sure this makes sense
-                    print(f"Order: {order.id} ALREADY PAID. Hash: {hash}.")
-
-                    request = noderpc.ListpaysRequest(
-                        payment_hash=bytes.fromhex(hash), status="complete"
-                    )
-
-                    for response in cls.stub.ListPays(request):
-                        handle_response(response)
-                # Permanent failure at destination. or Unable to find a route. or Route too expensive.
-                elif status_code == 203 or status_code == 205 or status_code == 206:
-                    lnpayment.status = LNPayment.Status.FAILRO
-                    lnpayment.last_routing_time = timezone.now()
-                    lnpayment.routing_attempts += 1
-                    lnpayment.failure_reason = status_code
-                    lnpayment.in_flight = False
-                    if lnpayment.routing_attempts > 2:
-                        lnpayment.status = LNPayment.Status.EXPIRE
-                        lnpayment.routing_attempts = 0
-                    lnpayment.save()
-
-                    order.status = Order.Status.FAI
-                    order.expires_at = timezone.now() + timedelta(
-                        seconds=order.t_to_expire(Order.Status.FAI)
-                    )
-                    order.save()
-                    print(
-                        f"Order: {order.id} FAILED. Hash: {hash} Reason: {cls.payment_failure_context[status_code]}"
-                    )
-                    return {
-                        "succeded": False,
-                        "context": f"payment failure reason: {cls.payment_failure_context[status_code]}",
-                    }
-                elif status_code == 207:  # invoice expired
-                    print(f"Order: {order.id}. INVOICE EXPIRED. Hash: {hash}")
-                    lnpayment.status = LNPayment.Status.EXPIRE
-                    lnpayment.last_routing_time = timezone.now()
-                    lnpayment.in_flight = False
-                    lnpayment.save()
-                    order.status = Order.Status.FAI
-                    order.expires_at = timezone.now() + timedelta(
-                        seconds=order.t_to_expire(Order.Status.FAI)
-                    )
-                    order.save()
-                    results = {"succeded": False,
-                               "context": "The payout invoice has expired"}
-                    return results
-                else:  # -1 and 210 (don't know when 210 happens exactly)
-                    print(str(e))
-            else:
-                print(str(e))
-
-        except Exception as e:
-            print(str(e))

From f4525d97f669e6f34c6520cf737f029866e0c9ec Mon Sep 17 00:00:00 2001
From: daywalker90 <admin@noserver4u.de>
Date: Sun, 23 Apr 2023 01:44:33 +0200
Subject: [PATCH 13/18] remove lnd specific commented code

---
 api/lightning/cln.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/api/lightning/cln.py b/api/lightning/cln.py
index c05144b7f..7f66a3c83 100755
--- a/api/lightning/cln.py
+++ b/api/lightning/cln.py
@@ -49,8 +49,6 @@ class CLNNode:
     stub = nodestub.NodeStub(channel)
 
     noderpc = noderpc
-    # invoicesrpc = invoicesrpc
-    # routerrpc = routerrpc
 
     payment_failure_context = {
         -1: "Catchall nonspecific error.",

From a05befd93a78f424ad68b8aa4b43b2d3b501d114 Mon Sep 17 00:00:00 2001
From: daywalker90 <admin@noserver4u.de>
Date: Sun, 23 Apr 2023 01:47:12 +0200
Subject: [PATCH 14/18] get_version: add try/except, refactor to top to mimic
 lnd.py

---
 api/lightning/cln.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/api/lightning/cln.py b/api/lightning/cln.py
index 7f66a3c83..ea7eba833 100755
--- a/api/lightning/cln.py
+++ b/api/lightning/cln.py
@@ -60,6 +60,16 @@ class CLNNode:
         210: "Payment timed out without a payment in progress.",
     }
 
+    @classmethod
+    def get_version(cls):
+        try:
+            request = noderpc.GetinfoRequest()
+            response = cls.stub.Getinfo(request)
+            return response.version
+        except Exception as e:
+            print(e)
+            return None
+
     @classmethod
     def decode_payreq(cls, invoice):
         """Decodes a lightning payment request (invoice)"""
@@ -412,13 +422,6 @@ def double_check_htlc_is_settled(cls, payment_hash):
         )   # CLN states: UNPAID = 0, PAID = 1, EXPIRED = 2, this is clns own invoice-lookup
         # so just a check for paid/unpaid/expired not hodl-invoice related states like ACCEPTED/CANCELED
 
-    @classmethod
-    def get_version(cls):
-        request = noderpc.GetinfoRequest()
-        response = cls.stub.Getinfo(request)
-
-        return response.version
-
     @classmethod
     @shared_task(name="follow_send_payment", time_limit=180)
     def follow_send_payment(cls, hash):

From 408f304ca02944abb5b0e1fb946cc6abbcfd1bfd Mon Sep 17 00:00:00 2001
From: daywalker90 <admin@noserver4u.de>
Date: Sun, 23 Apr 2023 01:57:53 +0200
Subject: [PATCH 15/18] rename htlc_cltv to htlc_expiry

---
 api/lightning/cln.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/api/lightning/cln.py b/api/lightning/cln.py
index ea7eba833..ee7fec63a 100755
--- a/api/lightning/cln.py
+++ b/api/lightning/cln.py
@@ -261,7 +261,7 @@ def validate_hold_invoice_locked(cls, lnpayment):
         if response.state == 2:  # CANCELLED
             pass
         if response.state == 3:  # ACCEPTED (LOCKED)
-            lnpayment.expiry_height = response.htlc_cltv
+            lnpayment.expiry_height = response.htlc_expiry
             lnpayment.status = LNPayment.Status.LOCKED
             lnpayment.save()
             return True

From 1a038232c9d172090ae9296fd24aba355a12be51 Mon Sep 17 00:00:00 2001
From: daywalker90 <admin@noserver4u.de>
Date: Sun, 23 Apr 2023 02:25:20 +0200
Subject: [PATCH 16/18] add clns lookup_invoice_status

---
 api/lightning/cln.py | 65 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 65 insertions(+)

diff --git a/api/lightning/cln.py b/api/lightning/cln.py
index ee7fec63a..be68ce733 100755
--- a/api/lightning/cln.py
+++ b/api/lightning/cln.py
@@ -266,6 +266,71 @@ def validate_hold_invoice_locked(cls, lnpayment):
             lnpayment.save()
             return True
 
+    @classmethod
+    def lookup_invoice_status(cls, lnpayment):
+        """
+        Returns the status (as LNpayment.Status) of the given payment_hash
+        If unchanged, returns the previous status
+        """
+        from api.models import LNPayment
+
+        status = lnpayment.status
+
+        cln_response_state_to_lnpayment_status = {
+            0: LNPayment.Status.INVGEN,  # OPEN
+            1: LNPayment.Status.SETLED,  # SETTLED
+            2: LNPayment.Status.CANCEL,  # CANCELLED
+            3: LNPayment.Status.LOCKED,  # ACCEPTED
+        }
+
+        try:
+            # this is similar to LNNnode.validate_hold_invoice_locked
+            request = noderpc.HodlInvoiceLookupRequest(
+                payment_hash=bytes.fromhex(lnpayment.payment_hash)
+            )
+            response = cls.stub.HodlInvoiceLookup(request)
+
+            # try saving expiry height
+            if hasattr(response, "htlc_expiry"):
+                try:
+                    lnpayment.expiry_height = response.htlc_expiry
+                except Exception:
+                    pass
+
+            status = cln_response_state_to_lnpayment_status[response.state]
+            lnpayment.status = status
+            lnpayment.save()
+
+        except Exception as e:
+            # If it fails at finding the invoice: it has been expired for more than an hour (and could be paid or just expired).
+            # In RoboSats DB we make a distinction between cancelled and returned
+            #  (cln-grpc-hodl has separate state for hodl-invoices, which it forgets after an invoice expired more than an hour ago)
+            if "empty result for listdatastore_state" in str(e):
+                print(str(e))
+                request2 = noderpc.ListinvoicesRequest(
+                    payment_hash=bytes.fromhex(lnpayment.payment_hash))
+                try:
+                    response2 = cls.stub.ListInvoices(request2).invoices
+                except Exception as e:
+                    print(str(e))
+
+                if response2[0].status == "paid":
+                    status = LNPayment.Status.SETLED
+                    lnpayment.status = status
+                    lnpayment.save()
+                elif response2[0].status == "expired":
+                    status = LNPayment.Status.CANCEL
+                    lnpayment.status = status
+                    lnpayment.save()
+                else:
+                    print(str(e))
+
+            # Other write to logs
+            else:
+                print(str(e))
+
+        return status
+
     @classmethod
     def resetmc(cls):
         # don't think an equivalent exists for cln, maybe deleting gossip_store file?

From ac02ecad65a1e22adc8fbe5a94d71eaba8931c63 Mon Sep 17 00:00:00 2001
From: daywalker90 <admin@noserver4u.de>
Date: Sun, 23 Apr 2023 02:28:04 +0200
Subject: [PATCH 17/18] refactored double_check_htlc_is_settled to the end to
 match lnds file

---
 api/lightning/cln.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/api/lightning/cln.py b/api/lightning/cln.py
index be68ce733..17684367c 100755
--- a/api/lightning/cln.py
+++ b/api/lightning/cln.py
@@ -476,17 +476,6 @@ def pay_invoice(cls, lnpayment):
             lnpayment.save()
             return False, failure_reason
 
-    @classmethod
-    def double_check_htlc_is_settled(cls, payment_hash):
-        """Just as it sounds. Better safe than sorry!"""
-        request = noderpc.ListinvoicesRequest(payment_hash=bytes.fromhex(payment_hash))
-        response = cls.stub.ListInvoices(request)
-
-        return (
-            response.status == 1
-        )   # CLN states: UNPAID = 0, PAID = 1, EXPIRED = 2, this is clns own invoice-lookup
-        # so just a check for paid/unpaid/expired not hodl-invoice related states like ACCEPTED/CANCELED
-
     @classmethod
     @shared_task(name="follow_send_payment", time_limit=180)
     def follow_send_payment(cls, hash):
@@ -643,3 +632,14 @@ def handle_response(response, was_in_transit=False):
 
         except Exception as e:
             print(str(e))
+
+    @classmethod
+    def double_check_htlc_is_settled(cls, payment_hash):
+        """Just as it sounds. Better safe than sorry!"""
+        request = noderpc.ListinvoicesRequest(payment_hash=bytes.fromhex(payment_hash))
+        response = cls.stub.ListInvoices(request)
+
+        return (
+            response.status == 1
+        )   # CLN states: UNPAID = 0, PAID = 1, EXPIRED = 2, this is clns own invoice-lookup
+        # so just a check for paid/unpaid/expired not hodl-invoice related states like ACCEPTED/CANCELED

From 052ea2c2d21bd5ba142f1dd623c8ec8c5bac5b68 Mon Sep 17 00:00:00 2001
From: daywalker90 <admin@noserver4u.de>
Date: Sun, 23 Apr 2023 14:40:56 +0200
Subject: [PATCH 18/18] fix generate_rpc

---
 generate_grpc.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/generate_grpc.sh b/generate_grpc.sh
index f1d25d01f..ef2954a1d 100755
--- a/generate_grpc.sh
+++ b/generate_grpc.sh
@@ -33,3 +33,6 @@ sed -i 's/^import .*_pb2 as/from . \0/' router_pb2_grpc.py
 sed -i 's/^import .*_pb2 as/from . \0/' lightning_pb2_grpc.py
 sed -i 's/^import .*_pb2 as/from . \0/' invoices_pb2_grpc.py
 sed -i 's/^import .*_pb2 as/from . \0/' verrpc_pb2_grpc.py
+
+sed -i 's/^import .*_pb2 as/from . \0/' node_pb2.py
+sed -i 's/^import .*_pb2 as/from . \0/' node_pb2_grpc.py