Skip to content

Commit

Permalink
Remove multihash from Swarm bzz:// for Feeds (#18175)
Browse files Browse the repository at this point in the history
  • Loading branch information
@jpeletier @nonsense
jpeletier authored and nonsense committed Nov 26, 2018
1 parent 0699287 commit 2714e8f
Show file tree
Hide file tree
Showing 7 changed files with 167 additions and 226 deletions.
6 changes: 2 additions & 4 deletions cmd/swarm/swarm-smoke/feed_upload_and_sync.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ import (
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/swarm/multihash"
"github.com/ethereum/go-ethereum/swarm/storage/feed"
colorable "github.com/mattn/go-colorable"
"github.com/pborman/uuid"
Expand All @@ -36,7 +35,7 @@ func cliFeedUploadAndSync(c *cli.Context) error {

generateEndpoints(scheme, cluster, from, to)

log.Info("generating and uploading MRUs to " + endpoints[0] + " and syncing")
log.Info("generating and uploading feeds to " + endpoints[0] + " and syncing")

// create a random private key to sign updates with and derive the address
pkFile, err := ioutil.TempFile("", "swarm-feed-smoke-test")
Expand Down Expand Up @@ -218,8 +217,7 @@ func cliFeedUploadAndSync(c *cli.Context) error {
if err != nil {
return err
}
multihashHex := hexutil.Encode(multihash.ToMultihash(hashBytes))

multihashHex := hexutil.Encode(hashBytes)
fileHash, err := digest(f)
if err != nil {
return err
Expand Down
1 change: 0 additions & 1 deletion swarm/OWNERS
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ swarm
├── fuse ────────────────── @jmozah, @holisticode
├── grafana_dashboards ──── @nonsense
├── metrics ─────────────── @nonsense, @holisticode
├── multihash ───────────── @nolash
├── network ─────────────── ethersphere
│ ├── bitvector ───────── @zelig, @janos, @gbalint
│ ├── priorityqueue ───── @zelig, @janos, @gbalint
Expand Down
25 changes: 12 additions & 13 deletions swarm/api/api.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@ import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/swarm/log"
"github.com/ethereum/go-ethereum/swarm/multihash"
"github.com/ethereum/go-ethereum/swarm/spancontext"
"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/ethereum/go-ethereum/swarm/storage/feed"
Expand Down Expand Up @@ -417,31 +416,31 @@ func (a *API) Get(ctx context.Context, decrypt DecryptFunc, manifestAddr storage
return reader, mimeType, status, nil, err
}
// get the data of the update
_, rsrcData, err := a.feed.GetContent(entry.Feed)
_, contentAddr, err := a.feed.GetContent(entry.Feed)
if err != nil {
apiGetNotFound.Inc(1)
status = http.StatusNotFound
log.Warn(fmt.Sprintf("get feed update content error: %v", err))
return reader, mimeType, status, nil, err
}

// extract multihash
decodedMultihash, err := multihash.FromMultihash(rsrcData)
if err != nil {
// extract content hash
if len(contentAddr) != storage.AddressLength {
apiGetInvalid.Inc(1)
status = http.StatusUnprocessableEntity
log.Warn("invalid multihash in feed update", "err", err)
return reader, mimeType, status, nil, err
errorMessage := fmt.Sprintf("invalid swarm hash in feed update. Expected %d bytes. Got %d", storage.AddressLength, len(contentAddr))
log.Warn(errorMessage)
return reader, mimeType, status, nil, errors.New(errorMessage)
}
manifestAddr = storage.Address(decodedMultihash)
log.Trace("feed update contains multihash", "key", manifestAddr)
manifestAddr = storage.Address(contentAddr)
log.Trace("feed update contains swarm hash", "key", manifestAddr)

// get the manifest the multihash digest points to
// get the manifest the swarm hash points to
trie, err := loadManifest(ctx, a.fileStore, manifestAddr, nil, NOOPDecrypt)
if err != nil {
apiGetNotFound.Inc(1)
status = http.StatusNotFound
log.Warn(fmt.Sprintf("loadManifestTrie (feed update multihash) error: %v", err))
log.Warn(fmt.Sprintf("loadManifestTrie (feed update) error: %v", err))
return reader, mimeType, status, nil, err
}

Expand All @@ -451,8 +450,8 @@ func (a *API) Get(ctx context.Context, decrypt DecryptFunc, manifestAddr storage
if entry == nil {
status = http.StatusNotFound
apiGetNotFound.Inc(1)
err = fmt.Errorf("manifest (feed update multihash) entry for '%s' not found", path)
log.Trace("manifest (feed update multihash) entry not found", "key", manifestAddr, "path", path)
err = fmt.Errorf("manifest (feed update) entry for '%s' not found", path)
log.Trace("manifest (feed update) entry not found", "key", manifestAddr, "path", path)
return reader, mimeType, status, nil, err
}
}
Expand Down
112 changes: 85 additions & 27 deletions swarm/api/client/client_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,13 @@ import (
"sort"
"testing"

"github.com/ethereum/go-ethereum/swarm/storage"
"github.com/ethereum/go-ethereum/swarm/storage/feed/lookup"

"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/swarm/api"
swarmhttp "github.com/ethereum/go-ethereum/swarm/api/http"
"github.com/ethereum/go-ethereum/swarm/multihash"
"github.com/ethereum/go-ethereum/swarm/storage/feed"
)

Expand Down Expand Up @@ -368,58 +368,99 @@ func newTestSigner() (*feed.GenericSigner, error) {
return feed.NewGenericSigner(privKey), nil
}

// test the transparent resolving of multihash feed updates with bzz:// scheme
// Test the transparent resolving of feed updates with bzz:// scheme
//
// first upload data, and store the multihash to the resulting manifest in a feed update
// retrieving the update with the multihash should return the manifest pointing directly to the data
// First upload data to bzz:, and store the Swarm hash to the resulting manifest in a feed update.
// This effectively uses a feed to store a pointer to content rather than the content itself
// Retrieving the update with the Swarm hash should return the manifest pointing directly to the data
// and raw retrieve of that hash should return the data
func TestClientCreateFeedMultihash(t *testing.T) {
func TestClientBzzWithFeed(t *testing.T) {

signer, _ := newTestSigner()

// Initialize a Swarm test server
srv := swarmhttp.NewTestSwarmServer(t, serverFunc, nil)
client := NewClient(srv.URL)
swarmClient := NewClient(srv.URL)
defer srv.Close()

// add the data our multihash aliased manifest will point to
databytes := []byte("bar")

swarmHash, err := client.UploadRaw(bytes.NewReader(databytes), int64(len(databytes)), false)
// put together some data for our test:
dataBytes := []byte(`
//
// Create some data our manifest will point to. Data that could be very big and wouldn't fit in a feed update.
// So what we are going to do is upload it to Swarm bzz:// and obtain a **manifest hash** pointing to it:
//
// MANIFEST HASH --> DATA
//
// Then, we store that **manifest hash** into a Swarm Feed update. Once we have done this,
// we can use the **feed manifest hash** in bzz:// instead, this way: bzz://feed-manifest-hash.
//
// FEED MANIFEST HASH --> MANIFEST HASH --> DATA
//
// Given that we can update the feed at any time with a new **manifest hash** but the **feed manifest hash**
// stays constant, we have effectively created a fixed address to changing content. (Applause)
//
// FEED MANIFEST HASH (the same) --> MANIFEST HASH(2) --> DATA(2)
//
`)

// Create a virtual File out of memory containing the above data
f := &File{
ReadCloser: ioutil.NopCloser(bytes.NewReader(dataBytes)),
ManifestEntry: api.ManifestEntry{
ContentType: "text/plain",
Mode: 0660,
Size: int64(len(dataBytes)),
},
}

// upload data to bzz:// and retrieve the content-addressed manifest hash, hex-encoded.
manifestAddressHex, err := swarmClient.Upload(f, "", false)
if err != nil {
t.Fatalf("Error uploading raw test data: %s", err)
t.Fatalf("Error creating manifest: %s", err)
}

s := common.FromHex(swarmHash)
mh := multihash.ToMultihash(s)
// convert the hex-encoded manifest hash to a 32-byte slice
manifestAddress := common.FromHex(manifestAddressHex)

if len(manifestAddress) != storage.AddressLength {
t.Fatalf("Something went wrong. Got a hash of an unexpected length. Expected %d bytes. Got %d", storage.AddressLength, len(manifestAddress))
}

// our feed topic
topic, _ := feed.NewTopic("foo.eth", nil)
// Now create a **feed manifest**. For that, we need a topic:
topic, _ := feed.NewTopic("interesting topic indeed", nil)

createRequest := feed.NewFirstRequest(topic)
// Build a feed request to update data
request := feed.NewFirstRequest(topic)

createRequest.SetData(mh)
if err := createRequest.Sign(signer); err != nil {
// Put the 32-byte address of the manifest into the feed update
request.SetData(manifestAddress)

// Sign the update
if err := request.Sign(signer); err != nil {
t.Fatalf("Error signing update: %s", err)
}

feedManifestHash, err := client.CreateFeedWithManifest(createRequest)

// Publish the update and at the same time request a **feed manifest** to be created
feedManifestAddressHex, err := swarmClient.CreateFeedWithManifest(request)
if err != nil {
t.Fatalf("Error creating feed manifest: %s", err)
}

correctManifestAddrHex := "bb056a5264c295c2b0f613c8409b9c87ce9d71576ace02458160df4cc894210b"
if feedManifestHash != correctManifestAddrHex {
t.Fatalf("Response feed manifest mismatch, expected '%s', got '%s'", correctManifestAddrHex, feedManifestHash)
// Check we have received the exact **feed manifest** to be expected
// given the topic and user signing the updates:
correctFeedManifestAddrHex := "747c402e5b9dc715a25a4393147512167bab018a007fad7cdcd9adc7fce1ced2"
if feedManifestAddressHex != correctFeedManifestAddrHex {
t.Fatalf("Response feed manifest mismatch, expected '%s', got '%s'", correctFeedManifestAddrHex, feedManifestAddressHex)
}

// Check we get a not found error when trying to get feed updates with a made-up manifest
_, err = client.QueryFeed(nil, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")
_, err = swarmClient.QueryFeed(nil, "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")
if err != ErrNoFeedUpdatesFound {
t.Fatalf("Expected to receive ErrNoFeedUpdatesFound error. Got: %s", err)
}

reader, err := client.QueryFeed(nil, correctManifestAddrHex)
// If we query the feed directly we should get **manifest hash** back:
reader, err := swarmClient.QueryFeed(nil, correctFeedManifestAddrHex)
if err != nil {
t.Fatalf("Error retrieving feed updates: %s", err)
}
Expand All @@ -428,10 +469,27 @@ func TestClientCreateFeedMultihash(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(mh, gotData) {
t.Fatalf("Expected: %v, got %v", mh, gotData)

//Check that indeed the **manifest hash** is retrieved
if !bytes.Equal(manifestAddress, gotData) {
t.Fatalf("Expected: %v, got %v", manifestAddress, gotData)
}

// Now the final test we were looking for: Use bzz://<feed-manifest> and that should resolve all manifests
// and return the original data directly:
f, err = swarmClient.Download(feedManifestAddressHex, "")
if err != nil {
t.Fatal(err)
}
gotData, err = ioutil.ReadAll(f)
if err != nil {
t.Fatal(err)
}

// Check that we get back the original data:
if !bytes.Equal(dataBytes, gotData) {
t.Fatalf("Expected: %v, got %v", manifestAddress, gotData)
}
}

// TestClientCreateUpdateFeed will check that feeds can be created and updated via the HTTP client.
Expand Down
Loading

0 comments on commit 2714e8f

Please sign in to comment.