Merge branch 'master' into master

README.md

@@ -1,5 +1,17 @@
 **Linux**: [![](http://jenkins.tribler.org/job/ipv8/job/test_ipv8_linux/badge/icon)](http://jenkins.tribler.org/job/ipv8/job/test_ipv8_linux/) **Windows**: [![](http://jenkins.tribler.org/job/ipv8/job/test_ipv8_windows/badge/icon)](http://jenkins.tribler.org/job/ipv8/job/test_ipv8_windows/) **Mac**: [![](http://jenkins.tribler.org/job/ipv8/job/test_ipv8_mac/badge/icon)](http://jenkins.tribler.org/job/ipv8/job/test_ipv8_mac/)
 
+## What is IPv8 ?
+
+IPv8 aims to provide authenticated communication with privacy.
+
+### IPv8 Objectives
+
+- **Authentication**. We offer mutual authentication using strong cryptography. During an IPv8 communication session, both parties can be sure of the other party’s identity. IPv8 users are identified by their public key. The initial key exchange is designed so that secrets are never transmitted across the Internet, not even in encrypted form. We use a standard challenge/response protocol with protection against spoofing, man-in-the-middle, and replay attacks.
+- **Privacy**. IPv8 is specifically designed for strong privacy protection and end-to-end encryption with perfect forward secrecy. We enhanced the industry standard onion routing protocol, Tor, for usage in a trustless environment (e.g. no trusted central directory servers).
+- **No infrastructure dependency**. Everybody is equal in the world of IPv8. No central web server, discovery server, or support foundation is needed.
+- **NAT traversal**. IPv8 can establish direct communication in difficult network situations. This includes connecting people behind a NAT or firewall. IPv8 includes a single simple and effective NAT traversal technique: UDP hole-punching. This is essential when offering privacy without infrastructure and consumer-grade donated resources.
+- **Trust**. You can enhance your security if you tell IPv8 which people you know and trust. It tries to build a web-of-trust automatically.
+
 This Python 2 package contains IPv8: an amalgamation of peer-to-peer communication functionality from [Dispersy](https://github.com/Tribler/dispersy) and [Tribler](https://github.com/Tribler/tribler), developed over the last 13 years by students and employees of the Delft University of Technology.
 The IPv8 library allows you to interface with the existing Dispersy network to build your own applications.
 

ipv8/REST/attestation_endpoint.py

@@ -84,8 +84,11 @@ def render_GET(self, request):
  peers = self.session.network.get_peers_for_service(self.identity_overlay.master_peer.mid)
  return json.dumps([b64encode(p.mid) for p in peers])
  if request.args['type'][0] == 'attributes':
- mid_b64 = request.args['mid'][0]
- peer = self.get_peer_from_mid(mid_b64)
+ if 'mid' in request.args:
+ mid_b64 = request.args['mid'][0]
+ peer = self.get_peer_from_mid(mid_b64)
+ else:
+ peer = self.identity_overlay.my_peer
  if peer:
  blocks = self.identity_overlay.persistence.get_latest_blocks(peer.public_key.key_to_bin(), 200)
  return json.dumps([(b.transaction["name"], b64encode(b.transaction["hash"])) for b in blocks])

ipv8/attestation/trustchain/caches.py

@@ -7,6 +7,39 @@
 from ...requestcache import NumberCache
 
 
+class IntroCrawlTimeout(NumberCache):
+ """
+ A crawl request is sent with every introduction response. This can happen quite a lot of times per second.
+ We wish to slow down the amount of crawls we do to not overload any node with database IO.
+ """
+
+ def __init__(self, community, peer):
+ super(IntroCrawlTimeout, self).__init__(community.request_cache, u"introcrawltimeout",
+ self.get_number_for(peer))
+
+ @classmethod
+ def get_number_for(cls, peer):
+ """
+ Convert a Peer into an int. To do this we shift every byte of the mid into an integer.
+ """
+ return reduce(lambda a, b: ((a << 8) | b), [ord(c) for c in peer.mid], 0)
+
+ @property
+ def timeout_delay(self):
+ """
+ We crawl the same peer, at most once every 60 seconds.
+ :return:
+ """
+ return 60.0
+
+ def on_timeout(self):
+ """
+ This is expected, the super class will now remove itself from the request cache.
+ The node is then allowed to be crawled again.
+ """
+ pass
+
+
 class HalfBlockSignCache(NumberCache):
  """
  This request cache keeps track of outstanding half block signature requests.

ipv8/attestation/trustchain/community.py

@@ -11,7 +11,7 @@
 from twisted.internet.defer import Deferred, succeed, fail
 
 from .block import TrustChainBlock, ValidationResult, EMPTY_PK, GENESIS_SEQ, UNKNOWN_SEQ
-from .caches import CrawlRequestCache, HalfBlockSignCache
+from .caches import CrawlRequestCache, HalfBlockSignCache, IntroCrawlTimeout
 from .database import TrustChainDB
 from ...deprecated.community import Community
 from ...deprecated.payload import IntroductionResponsePayload
@@ -386,12 +386,18 @@ def get_peer_for_introduction(self, exclude=None):
 
  return eligible[-1]
 
+ @synchronized
  def on_introduction_response(self, source_address, data):
  super(TrustChainCommunity, self).on_introduction_response(source_address, data)
 
  auth, _, _ = self._ez_unpack_auth(IntroductionResponsePayload, data)
  peer = Peer(auth.public_key_bin, source_address)
- self.crawl_lowest_unknown(peer)
+ if self.request_cache.has(u"introcrawltimeout", IntroCrawlTimeout.get_number_for(peer)):
+ self.logger.debug("Not crawling %s, as we have already crawled it in the last %d seconds!",
+ peer.mid.encode('hex'), IntroCrawlTimeout.__new__(IntroCrawlTimeout).timeout_delay)
+ else:
+ self.request_cache.add(IntroCrawlTimeout(self, peer))
+ self.crawl_lowest_unknown(peer)
 
  def unload(self):
  self.logger.debug("Unloading the TrustChain Community.")

ipv8/attestation/wallet/community.py

@@ -48,12 +48,12 @@ def __init__(self, *args, **kwargs):
  super(AttestationCommunity, self).__init__(*args, **kwargs)
 
  self.database = AttestationsDB(working_directory, db_name)
- self.attestation_request_callbacks = [lambda x, y: None, lambda x, y, z: None]
+ self.attestation_request_callbacks = [lambda x, y: None, lambda x, y, z, w=None: None]
 
  # Map of attestation hash -> BonehPrivateKey
  self.attestation_keys = {}
  for hash, _, key in self.database.get_all():
- self.attestation_keys[hash] = BonehPrivateKey(key)
+ self.attestation_keys[str(hash)] = BonehPrivateKey.unserialize(str(key))
 
  self.request_cache = RequestCache()
 

ipv8/configuration.py

@@ -93,14 +93,26 @@
  {
  'class': 'AttestationCommunity',
  'key': "anonymous id",
- 'walkers': [],
+ 'walkers': [{
+ 'strategy': "RandomWalk",
+ 'peers': 4,
+ 'init': {
+ 'timeout': 60.0
+ }
+ }],
  'initialize': {},
  'on_start': []
  },
  {
  'class': 'IdentityCommunity',
  'key': "anonymous id",
- 'walkers': [],
+ 'walkers': [{
+ 'strategy': "RandomWalk",
+ 'peers': 4,
+ 'init': {
+ 'timeout': 60.0
+ }
+ }],
  'initialize': {},
  'on_start': []
  }

ipv8/messaging/anonymization/community.py

@@ -779,7 +779,9 @@ def on_cell(self, source_address, data):
 
  if self.is_relay(circuit_id):
  if not self.relay_packet(circuit_id, message_type, data):
- self.send_destroy(source_address, circuit_id, 0)
+ circuit = self.circuits.get(circuit_id, None)
+ if circuit:
+ self.send_destroy(circuit.sock_addr, circuit_id, 0)
 
  else:
  circuit = self.circuits.get(circuit_id, None)
@@ -792,7 +794,8 @@ def on_cell(self, source_address, data):
 
  except CryptoException, e:
  self.logger.warning(str(e))
- self.send_destroy(source_address, circuit_id, 0)
+ if circuit:
+ self.send_destroy(circuit.sock_addr, circuit_id, 0)
  return
 
  self.on_packet((source_address, convert_from_cell(data)), circuit_id=u"circuit_%d" % circuit_id)
@@ -974,7 +977,6 @@ def on_data(self, sock_addr, packet):
 
  except CryptoException, e:
  self.logger.warning(str(e))
- self.send_destroy(sock_addr, circuit_id, 0)
  return
 
  packet = plaintext + encrypted

ipv8/test/attestation/trustchain/test_community.py

@@ -331,3 +331,42 @@ def test_broadcast_half_block_pair(self):
  for node_nr in [0, 1]:
  self.assertIsNotNone(self.nodes[node_nr].overlay.persistence.get_latest(block1.public_key))
  self.assertIsNotNone(self.nodes[node_nr].overlay.persistence.get_latest(block2.public_key))
+
+ @twisted_wrapper
+ def test_intro_response_crawl(self):
+ """
+ Test if we crawl a node on introduction response and if we respect the crawl timeout.
+
+ This test does the following:
+
+ 1. Node 1 and 2 are introduced to each other and subsequently crawl each other.
+ 2. Node 1 and 3 perform a transaction.
+ 3. Node 1 and 2 are introduced to each other again.
+ 4. As we are still in the crawl timeout and node 2 is not involved in any transaction with node 1,
+ there should be no crawl request and therefore node 2 does not know about the transaction between
+ node 1 and 3.
+ """
+ node3 = self.create_node()
+ node3.my_peer.address = node3.endpoint.wan_address
+ self.nodes.append(node3)
+ my_pubkey = self.nodes[0].my_peer.public_key.key_to_bin()
+ his_pubkey = self.nodes[0].network.verified_peers[0].public_key.key_to_bin()
+ node3_pubkey = self.nodes[2].overlay.my_peer.public_key.key_to_bin()
+ yield self.nodes[0].overlay.sign_block(self.nodes[0].network.verified_peers[0], public_key=his_pubkey,
+ transaction={})
+
+ # Perform the first crawl with all nodes
+ yield self.introduce_nodes()
+
+ for node_nr in [0, 1]:
+ self.assertIsNotNone(self.nodes[node_nr].overlay.persistence.get(my_pubkey, 1))
+ self.assertEqual(self.nodes[node_nr].overlay.persistence.get(my_pubkey, 1).link_sequence_number,
+ UNKNOWN_SEQ)
+
+ # Perform a transaction between node 1 and 3
+ yield self.nodes[0].overlay.sign_block(node3.my_peer, public_key=node3_pubkey, transaction={})
+
+ # Perform the second crawl with all nodes
+ yield self.introduce_nodes()
+
+ self.assertIsNone(self.nodes[1].overlay.persistence.get(my_pubkey, 2))

ipv8/test/attestation/wallet/test_attestation_community.py

@@ -1,5 +1,6 @@
 import os
 
+from ....attestation.wallet.database import AttestationsDB
 from ....attestation.wallet.primitives.attestation import binary_relativity_sha512
 from ....attestation.wallet.community import Attestation, AttestationCommunity, BonehPrivateKey
 
@@ -50,7 +51,7 @@ def test_request_attestation(self):
  """
  Check if the request_attestation callback is correctly called.
  """
- def f(peer, attribute_name, _):
+ def f(peer, attribute_name, _, __=None):
  self.assertEqual(peer.address, self.nodes[1].endpoint.wan_address)
  self.assertEqual(attribute_name, "MyAttribute")
 
@@ -101,3 +102,26 @@ def callback(rhash, values):
 
  self.assertTrue(callback.called)
  self.nodes[1].overlay.request_cache.clear()
+
+ def test_load_key(self):
+ """
+ Check if we can load the community correctly after shut down.
+ """
+ # Write to a temporary folder.
+ overlay = self.nodes[0].overlay
+ temp_folder = self.temporary_directory()
+ overlay.database = AttestationsDB(temp_folder, "test")
+
+ # Create an attestation and write it to file.
+ # Then close the database.
+ attestation = Attestation(TestCommunity.private_key.public_key(), [])
+ overlay.on_attestation_complete(attestation, TestCommunity.private_key, None, "test", "a"*20)
+ overlay.database.close(True)
+
+ # Reload the community with the same database.
+ self.nodes[0].overlay.__init__(self.nodes[0].my_peer, self.nodes[0].endpoint, self.nodes[0].network,
+ working_directory=temp_folder, db_name="test")
+
+ # The attestation should persist
+ db_entries = self.nodes[0].overlay.database.get_all()
+ self.assertEqual(1, len(db_entries))

ipv8/test/base.py

@@ -1,3 +1,5 @@
+import os
+import shutil
 import sys
 import threading
 import time
@@ -49,10 +51,14 @@ def setUp(self):
  self.__lockup_timestamp__ = time.time()
 
  def tearDown(self):
- super(TestBase, self).tearDown()
- for node in self.nodes:
- node.unload()
- internet.clear()
+ try:
+ super(TestBase, self).tearDown()
+ for node in self.nodes:
+ node.unload()
+ internet.clear()
+ finally:
+ shutil.rmtree("temp", ignore_errors=True)
+
 
  @classmethod
  def setUpClass(cls):
@@ -128,3 +134,8 @@ def introduce_nodes(self):
  for node in self.nodes:
  node.discovery.take_step()
  yield self.sleep()
+
+ def temporary_directory(self):
+ d = os.path.join("temp", self.__class__.__name__ + str(int(time.time())))
+ os.makedirs(d)
+ return d