From a6ffb28b9e2a5f16966357d8664b6f36ee7f9060 Mon Sep 17 00:00:00 2001
From: Cory Schwartz <cory@protocol.ai>
Date: Wed, 9 Jun 2021 00:03:01 -0700
Subject: [PATCH 01/16] multibase encoding on pubsub

---
 core/commands/pubsub.go | 36 ++++++++++++++++++++++++++++++------
 1 file changed, 30 insertions(+), 6 deletions(-)

diff --git a/core/commands/pubsub.go b/core/commands/pubsub.go
index 1ec5016d6fd..ab5430da5e7 100644
--- a/core/commands/pubsub.go
+++ b/core/commands/pubsub.go
@@ -5,10 +5,14 @@ import (
 	"encoding/binary"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net/http"
+	"os"
 	"sort"
 
 	cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv"
+	mbase "github.com/multiformats/go-multibase"
+	"github.com/pkg/errors"
 
 	cmds "github.com/ipfs/go-ipfs-cmds"
 	options "github.com/ipfs/interface-go-ipfs-core/options"
@@ -147,7 +151,25 @@ To use, the daemon must be run with '--enable-pubsub-experiment'.
 	},
 	Arguments: []cmds.Argument{
 		cmds.StringArg("topic", true, false, "Topic to publish to."),
-		cmds.StringArg("data", true, true, "Payload of message to publish.").EnableStdin(),
+		cmds.StringArg("data", false, true, "Payload of message to publish."),
+	},
+	PreRun: func(req *cmds.Request, env cmds.Environment) error {
+		// encode all arguments
+
+		encoder, _ := mbase.EncoderByName("base64")
+		for n, arg := range req.Arguments {
+			req.Arguments[n] = encoder.Encode([]byte(arg))
+		}
+
+		// when there are no string args, read from stdin.
+		if len(req.Arguments) == 1 {
+			buf, err := ioutil.ReadAll(os.Stdin)
+			if err != nil {
+				return err
+			}
+			req.Arguments = append(req.Arguments, encoder.Encode(buf))
+		}
+		return nil
 	},
 	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
 		api, err := cmdenv.GetApi(env, req)
@@ -155,15 +177,17 @@ To use, the daemon must be run with '--enable-pubsub-experiment'.
 			return err
 		}
 
-		topic := req.Arguments[0]
-
-		err = req.ParseBodyArgs()
+		_, topic, err := mbase.Decode(req.Arguments[0])
 		if err != nil {
-			return err
+			return errors.Wrap(err, "pubsub topic must be multibase encoded")
 		}
 
 		for _, data := range req.Arguments[1:] {
-			if err := api.PubSub().Publish(req.Context, topic, []byte(data)); err != nil {
+			_, datab, err := mbase.Decode(data)
+			if err != nil {
+				return errors.Wrap(err, "pubsub data must be multibase encoded")
+			}
+			if err := api.PubSub().Publish(req.Context, string(topic), datab); err != nil {
 				return err
 			}
 		}

From fce641301569a5c163eefa714fb97c4e650141b0 Mon Sep 17 00:00:00 2001
From: Cory Schwartz <cory@protocol.ai>
Date: Wed, 9 Jun 2021 16:44:06 -0700
Subject: [PATCH 02/16] emit multibase for json clients

---
 core/commands/pubsub.go | 38 ++++++++++++++++++++++++--------------
 1 file changed, 24 insertions(+), 14 deletions(-)

diff --git a/core/commands/pubsub.go b/core/commands/pubsub.go
index ab5430da5e7..796d2d09ad8 100644
--- a/core/commands/pubsub.go
+++ b/core/commands/pubsub.go
@@ -44,9 +44,9 @@ const (
 )
 
 type pubsubMessage struct {
-	From     []byte   `json:"from,omitempty"`
-	Data     []byte   `json:"data,omitempty"`
-	Seqno    []byte   `json:"seqno,omitempty"`
+	From     string   `json:"from,omitempty"`
+	Data     string   `json:"data,omitempty"`
+	Seqno    string   `json:"seqno,omitempty"`
 	TopicIDs []string `json:"topicIDs,omitempty"`
 }
 
@@ -105,24 +105,36 @@ This command outputs data in the following encodings:
 				return err
 			}
 
-			if err := res.Emit(&pubsubMessage{
-				Data:     msg.Data(),
-				From:     []byte(msg.From()),
-				Seqno:    msg.Seq(),
-				TopicIDs: msg.Topics(),
-			}); err != nil {
+			encoder, _ := mbase.EncoderByName("base64")
+			psm := pubsubMessage{
+				Data:  encoder.Encode(msg.Data()),
+				From:  encoder.Encode([]byte(msg.From())),
+				Seqno: encoder.Encode(msg.Seq()),
+			}
+			for _, topic := range msg.Topics() {
+				psm.TopicIDs = append(psm.TopicIDs, encoder.Encode([]byte(topic)))
+			}
+			if err := res.Emit(&psm); err != nil {
 				return err
 			}
 		}
 	},
 	Encoders: cmds.EncoderMap{
 		cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, psm *pubsubMessage) error {
-			_, err := w.Write(psm.Data)
+			_, dec, err := mbase.Decode(psm.Data)
+			if err != nil {
+				return err
+			}
+			_, err = w.Write(dec)
 			return err
 		}),
 		"ndpayload": cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, psm *pubsubMessage) error {
-			psm.Data = append(psm.Data, '\n')
-			_, err := w.Write(psm.Data)
+			_, dec, err := mbase.Decode(psm.Data)
+			if err != nil {
+				return err
+			}
+			data := append(dec, '\n')
+			_, err = w.Write(data)
 			return err
 		}),
 		"lenpayload": cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, psm *pubsubMessage) error {
@@ -154,8 +166,6 @@ To use, the daemon must be run with '--enable-pubsub-experiment'.
 		cmds.StringArg("data", false, true, "Payload of message to publish."),
 	},
 	PreRun: func(req *cmds.Request, env cmds.Environment) error {
-		// encode all arguments
-
 		encoder, _ := mbase.EncoderByName("base64")
 		for n, arg := range req.Arguments {
 			req.Arguments[n] = encoder.Encode([]byte(arg))

From a663e78ab572ff2e4b63ccc3694d4e4849c27df7 Mon Sep 17 00:00:00 2001
From: Cory Schwartz <cory@protocol.ai>
Date: Thu, 12 Aug 2021 14:04:50 -0700
Subject: [PATCH 03/16] Update core/commands/pubsub.go

Co-authored-by: Marcin Rataj <lidel@lidel.org>
---
 core/commands/pubsub.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/core/commands/pubsub.go b/core/commands/pubsub.go
index 796d2d09ad8..0fede4a1b44 100644
--- a/core/commands/pubsub.go
+++ b/core/commands/pubsub.go
@@ -166,7 +166,7 @@ To use, the daemon must be run with '--enable-pubsub-experiment'.
 		cmds.StringArg("data", false, true, "Payload of message to publish."),
 	},
 	PreRun: func(req *cmds.Request, env cmds.Environment) error {
-		encoder, _ := mbase.EncoderByName("base64")
+		encoder, _ := mbase.EncoderByName("base64url")
 		for n, arg := range req.Arguments {
 			req.Arguments[n] = encoder.Encode([]byte(arg))
 		}

From d075dd646152722c3b094e0a97759ac2d6704a16 Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Wed, 6 Oct 2021 16:32:48 +0200
Subject: [PATCH 04/16] refactor(pubsub): base64url for all URL args

This makes it easier to reason about.
Also added better helptext to each command explaining how the binary
data is encoded on the wire, and how to process it in userland.
---
 core/commands/pubsub.go | 187 ++++++++++++++++++++++++++++------------
 1 file changed, 133 insertions(+), 54 deletions(-)

diff --git a/core/commands/pubsub.go b/core/commands/pubsub.go
index 0fede4a1b44..4632e6fdf8c 100644
--- a/core/commands/pubsub.go
+++ b/core/commands/pubsub.go
@@ -25,10 +25,11 @@ var PubsubCmd = &cmds.Command{
 ipfs pubsub allows you to publish messages to a given topic, and also to
 subscribe to new messages on a given topic.
 
-This is an experimental feature. It is not intended in its current state
-to be used in a production environment.
+EXPERIMENTAL FEATURE
 
-To use, the daemon must be run with '--enable-pubsub-experiment'.
+  It is not intended in its current state to be used in a production
+  environment.  To use, the daemon must be run with
+  '--enable-pubsub-experiment'.
 `,
 	},
 	Subcommands: map[string]*cmds.Command{
@@ -39,10 +40,6 @@ To use, the daemon must be run with '--enable-pubsub-experiment'.
 	},
 }
 
-const (
-	pubsubDiscoverOptionName = "discover"
-)
-
 type pubsubMessage struct {
 	From     string   `json:"from,omitempty"`
 	Data     string   `json:"data,omitempty"`
@@ -56,37 +53,37 @@ var PubsubSubCmd = &cmds.Command{
 		ShortDescription: `
 ipfs pubsub sub subscribes to messages on a given topic.
 
-This is an experimental feature. It is not intended in its current state
-to be used in a production environment.
+EXPERIMENTAL FEATURE
 
-To use, the daemon must be run with '--enable-pubsub-experiment'.
-`,
-		LongDescription: `
-ipfs pubsub sub subscribes to messages on a given topic.
+  It is not intended in its current state to be used in a production
+  environment.  To use, the daemon must be run with
+  '--enable-pubsub-experiment'.
 
-This is an experimental feature. It is not intended in its current state
-to be used in a production environment.
+TOPIC ENCODING
 
-To use, the daemon must be run with '--enable-pubsub-experiment'.
+  Topic names are a binary data. To ensure all bytes are transferred
+  correctly RPC client and server will use multibase encoding behind
+  the scenes.
 
-This command outputs data in the following encodings:
-  * "json"
-(Specified by the "--encoding" or "--enc" flag)
+  You can inspect the format by passing --enc=json. ipfs multibase commands
+  can be used for encoding/decoding multibase strings in the userland.
 `,
 	},
 	Arguments: []cmds.Argument{
-		cmds.StringArg("topic", true, false, "String name of topic to subscribe to."),
-	},
-	Options: []cmds.Option{
-		cmds.BoolOption(pubsubDiscoverOptionName, "Deprecated option to instruct pubsub to discovery peers for the topic. Discovery is now built into pubsub."),
+		cmds.StringArg("topic", true, false, "Name of topic to subscribe to."),
 	},
+	PreRun: urlArgsEncoder,
 	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
 		api, err := cmdenv.GetApi(env, req)
 		if err != nil {
 			return err
 		}
+		if err := urlArgsDecoder(req, env); err != nil {
+			return err
+		}
 
 		topic := req.Arguments[0]
+
 		sub, err := api.PubSub().Subscribe(req.Context, topic)
 		if err != nil {
 			return err
@@ -105,7 +102,9 @@ This command outputs data in the following encodings:
 				return err
 			}
 
-			encoder, _ := mbase.EncoderByName("base64")
+			// encode as base64url so the same string is present in body and URL args
+			// when sent over HTTP RPC API
+			encoder, _ := mbase.EncoderByName("base64url")
 			psm := pubsubMessage{
 				Data:  encoder.Encode(msg.Data()),
 				From:  encoder.Encode([]byte(msg.From())),
@@ -128,6 +127,7 @@ This command outputs data in the following encodings:
 			_, err = w.Write(dec)
 			return err
 		}),
+		// --enc=ndpayload is not documented, byt used internally by sharness tests
 		"ndpayload": cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, psm *pubsubMessage) error {
 			_, dec, err := mbase.Decode(psm.Data)
 			if err != nil {
@@ -137,6 +137,7 @@ This command outputs data in the following encodings:
 			_, err = w.Write(data)
 			return err
 		}),
+		// ¯\_(ツ)_/¯ – seems unused, can we remove this?
 		"lenpayload": cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, psm *pubsubMessage) error {
 			buf := make([]byte, 8, len(psm.Data)+8)
 
@@ -155,10 +156,20 @@ var PubsubPubCmd = &cmds.Command{
 		ShortDescription: `
 ipfs pubsub pub publishes a message to a specified topic.
 
-This is an experimental feature. It is not intended in its current state
-to be used in a production environment.
+EXPERIMENTAL FEATURE
+
+  It is not intended in its current state to be used in a production
+  environment.  To use, the daemon must be run with
+  '--enable-pubsub-experiment'.
+
+TOPIC AND DATA ENCODING
+
+  Topic names are a binary data too. To ensure all bytes are transferred
+  correctly RPC client and server will use multibase encoding behind
+  the scenes.
 
-To use, the daemon must be run with '--enable-pubsub-experiment'.
+  You can inspect the format by passing --enc=json. ipfs multibase commands
+  can be used for encoding/decoding multibase strings in the userland.
 `,
 	},
 	Arguments: []cmds.Argument{
@@ -166,38 +177,28 @@ To use, the daemon must be run with '--enable-pubsub-experiment'.
 		cmds.StringArg("data", false, true, "Payload of message to publish."),
 	},
 	PreRun: func(req *cmds.Request, env cmds.Environment) error {
-		encoder, _ := mbase.EncoderByName("base64url")
-		for n, arg := range req.Arguments {
-			req.Arguments[n] = encoder.Encode([]byte(arg))
-		}
-
-		// when there are no string args, read from stdin.
+		// when there are no string args with data, read from stdin.
 		if len(req.Arguments) == 1 {
 			buf, err := ioutil.ReadAll(os.Stdin)
 			if err != nil {
 				return err
 			}
-			req.Arguments = append(req.Arguments, encoder.Encode(buf))
+			req.Arguments = append(req.Arguments, string(buf))
 		}
-		return nil
+		return urlArgsEncoder(req, env)
 	},
 	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
 		api, err := cmdenv.GetApi(env, req)
 		if err != nil {
 			return err
 		}
-
-		_, topic, err := mbase.Decode(req.Arguments[0])
-		if err != nil {
-			return errors.Wrap(err, "pubsub topic must be multibase encoded")
+		if err := urlArgsDecoder(req, env); err != nil {
+			return err
 		}
 
+		topic := req.Arguments[0]
 		for _, data := range req.Arguments[1:] {
-			_, datab, err := mbase.Decode(data)
-			if err != nil {
-				return errors.Wrap(err, "pubsub data must be multibase encoded")
-			}
-			if err := api.PubSub().Publish(req.Context, string(topic), datab); err != nil {
+			if err := api.PubSub().Publish(req.Context, topic, []byte(data)); err != nil {
 				return err
 			}
 		}
@@ -212,10 +213,20 @@ var PubsubLsCmd = &cmds.Command{
 		ShortDescription: `
 ipfs pubsub ls lists out the names of topics you are currently subscribed to.
 
-This is an experimental feature. It is not intended in its current state
-to be used in a production environment.
+EXPERIMENTAL FEATURE
 
-To use, the daemon must be run with '--enable-pubsub-experiment'.
+  It is not intended in its current state to be used in a production
+  environment.  To use, the daemon must be run with
+  '--enable-pubsub-experiment'.
+
+TOPIC ENCODING
+
+  Topic names are a binary data. To ensure all bytes are transferred
+  correctly RPC client and server will use multibase encoding behind
+  the scenes.
+
+  You can inspect the format by passing --enc=json. ipfs multibase commands
+  can be used for encoding/decoding multibase strings in the userland.
 `,
 	},
 	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
@@ -229,14 +240,31 @@ To use, the daemon must be run with '--enable-pubsub-experiment'.
 			return err
 		}
 
+		// emit topics encoded in multibase
+		encoder, _ := mbase.EncoderByName("base64url")
+		for n, topic := range l {
+			l[n] = encoder.Encode([]byte(topic))
+		}
+
 		return cmds.EmitOnce(res, stringList{l})
 	},
 	Type: stringList{},
 	Encoders: cmds.EncoderMap{
-		cmds.Text: cmds.MakeTypedEncoder(stringListEncoder),
+		cmds.Text: cmds.MakeTypedEncoder(multibaseDecodedStringListEncoder),
 	},
 }
 
+func multibaseDecodedStringListEncoder(req *cmds.Request, w io.Writer, list *stringList) error {
+	for n, mb := range list.Strings {
+		_, data, err := mbase.Decode(mb)
+		if err != nil {
+			return err
+		}
+		list.Strings[n] = string(data)
+	}
+	return stringListEncoder(req, w, list)
+}
+
 func stringListEncoder(req *cmds.Request, w io.Writer, list *stringList) error {
 	for _, str := range list.Strings {
 		_, err := fmt.Fprintf(w, "%s\n", cmdenv.EscNonPrint(str))
@@ -252,23 +280,37 @@ var PubsubPeersCmd = &cmds.Command{
 		Tagline: "List peers we are currently pubsubbing with.",
 		ShortDescription: `
 ipfs pubsub peers with no arguments lists out the pubsub peers you are
-currently connected to. If given a topic, it will list connected
-peers who are subscribed to the named topic.
+currently connected to. If given a topic, it will list connected peers who are
+subscribed to the named topic.
 
-This is an experimental feature. It is not intended in its current state
-to be used in a production environment.
+EXPERIMENTAL FEATURE
 
-To use, the daemon must be run with '--enable-pubsub-experiment'.
+  It is not intended in its current state to be used in a production
+  environment.  To use, the daemon must be run with
+  '--enable-pubsub-experiment'.
+
+TOPIC AND DATA ENCODING
+
+  Topic names are a binary data. To ensure all bytes are transferred
+  correctly RPC client and server will use multibase encoding behind
+  the scenes.
+
+  You can inspect the format by passing --enc=json. ipfs multibase commands
+  can be used for encoding/decoding multibase strings in the userland.
 `,
 	},
 	Arguments: []cmds.Argument{
-		cmds.StringArg("topic", false, false, "topic to list connected peers of"),
+		cmds.StringArg("topic", false, false, "Topic to list connected peers of."),
 	},
+	PreRun: urlArgsEncoder,
 	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
 		api, err := cmdenv.GetApi(env, req)
 		if err != nil {
 			return err
 		}
+		if err := urlArgsDecoder(req, env); err != nil {
+			return err
+		}
 
 		var topic string
 		if len(req.Arguments) == 1 {
@@ -293,3 +335,40 @@ To use, the daemon must be run with '--enable-pubsub-experiment'.
 		cmds.Text: cmds.MakeTypedEncoder(stringListEncoder),
 	},
 }
+
+// Encode binary data to be passed as multibase string in URL arguments.
+// (avoiding issues described in https://github.com/ipfs/go-ipfs/issues/7939)
+func urlArgsEncoder(req *cmds.Request, env cmds.Environment) error {
+	encoder, _ := mbase.EncoderByName("base64url")
+	for n, arg := range req.Arguments {
+		req.Arguments[n] = encoder.Encode([]byte(arg))
+	}
+	return nil
+}
+
+// Decode binary data passed as multibase string in URL arguments.
+// (avoiding issues described in https://github.com/ipfs/go-ipfs/issues/7939)
+func urlArgsDecoder(req *cmds.Request, env cmds.Environment) error {
+	err := req.ParseBodyArgs()
+	if err != nil {
+		return err
+	}
+	for n, arg := range req.Arguments {
+		encoding, data, err := mbase.Decode(arg)
+		if err != nil {
+			return errors.Wrap(err, "URL arg must be multibase encoded")
+		}
+
+		// Enforce URL-safe encoding is used for data passed via URL arguments
+		// - without this we get data corruption similar to https://github.com/ipfs/go-ipfs/issues/7939
+		// - we can't just deny base64, because there may be other bases that
+		//   are not URL-safe – better to force base64url which is known to be
+		//   safe in URL context
+		if encoding != mbase.Base64url {
+			return errors.New("URL arg must be base64url encoded")
+		}
+
+		req.Arguments[n] = string(data)
+	}
+	return nil
+}

From 82a0a95bf61385cf5a15b2679791a658ceb8ea45 Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Wed, 6 Oct 2021 21:29:04 +0200
Subject: [PATCH 05/16] refactor: remove ndpayload and lenpayload

Those output formats are undocumented and seem to be only used in tests.
This change removes their implementation and replaces it with error
message to use JSON instead.

I also refactored tests to test the --enc=json response format instead
of imaginary one, making tests more useful as they also act as
regression tests for HTTP RPC.
---
 core/commands/pubsub.go                 | 22 ++++++----------------
 test/sharness/t0180-pubsub-gossipsub.sh | 15 +++++++--------
 test/sharness/t0180-pubsub.sh           | 17 ++++++++---------
 3 files changed, 21 insertions(+), 33 deletions(-)

diff --git a/core/commands/pubsub.go b/core/commands/pubsub.go
index 4632e6fdf8c..b45d657dca2 100644
--- a/core/commands/pubsub.go
+++ b/core/commands/pubsub.go
@@ -2,7 +2,6 @@ package commands
 
 import (
 	"context"
-	"encoding/binary"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -127,24 +126,15 @@ TOPIC ENCODING
 			_, err = w.Write(dec)
 			return err
 		}),
-		// --enc=ndpayload is not documented, byt used internally by sharness tests
+		// DEPRECATED, undocumented format we used in tests, but not anymore
+		// <message.payload>\n<message.payload>\n
 		"ndpayload": cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, psm *pubsubMessage) error {
-			_, dec, err := mbase.Decode(psm.Data)
-			if err != nil {
-				return err
-			}
-			data := append(dec, '\n')
-			_, err = w.Write(data)
-			return err
+			return errors.New("--enc=ndpayload was removed, use --enc=json instead")
 		}),
-		// ¯\_(ツ)_/¯ – seems unused, can we remove this?
+		// DEPRECATED, uncodumented format we used in tests, but not anymore
+		// <varint-len><message.payload><varint-len><message.payload>
 		"lenpayload": cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, psm *pubsubMessage) error {
-			buf := make([]byte, 8, len(psm.Data)+8)
-
-			n := binary.PutUvarint(buf, uint64(len(psm.Data)))
-			buf = append(buf[:n], psm.Data...)
-			_, err := w.Write(buf)
-			return err
+			return errors.New("--enc=lenpayload was removed, use --enc=json instead")
 		}),
 	},
 	Type: pubsubMessage{},
diff --git a/test/sharness/t0180-pubsub-gossipsub.sh b/test/sharness/t0180-pubsub-gossipsub.sh
index c5ff8daf39e..8c0686efd1e 100755
--- a/test/sharness/t0180-pubsub-gossipsub.sh
+++ b/test/sharness/t0180-pubsub-gossipsub.sh
@@ -25,7 +25,7 @@ test_expect_success 'peer ids' '
 '
 
 test_expect_success 'pubsub' '
-  echo "testOK" > expected &&
+  echo -n "testOK" | ipfs multibase encode -b base64url > expected &&
   touch empty &&
   mkfifo wait ||
   test_fsh echo init fail
@@ -33,8 +33,8 @@ test_expect_success 'pubsub' '
   # ipfs pubsub sub is long-running so we need to start it in the background and
   # wait put its output somewhere where we can access it
   (
-    ipfsi 0 pubsub sub --enc=ndpayload testTopic | if read line; then
-        echo $line > actual &&
+    ipfsi 0 pubsub sub --enc=json testTopic | if read line; then
+        echo $line | jq -j .data > actual &&
         echo > wait
       fi
   ) &
@@ -64,15 +64,15 @@ test_expect_success "wait until echo > wait executed" '
 '
 
 test_expect_success "wait for another pubsub message" '
-  echo "testOK2" > expected &&
+  echo -n "testOK2" | ipfs multibase encode -b base64url > expected &&
   mkfifo wait2 ||
   test_fsh echo init fail
 
   # ipfs pubsub sub is long-running so we need to start it in the background and
   # wait put its output somewhere where we can access it
   (
-    ipfsi 2 pubsub sub --enc=ndpayload testTopic | if read line; then
-        echo $line > actual &&
+    ipfsi 2 pubsub sub --enc=json testTopic | if read line; then
+        echo $line | jq -j .data > actual &&
         echo > wait2
       fi
   ) &
@@ -83,11 +83,10 @@ test_expect_success "wait until ipfs pubsub sub is ready to do work" '
 '
 
 test_expect_success "publish something" '
-  echo "testOK2" | ipfsi 1 pubsub pub testTopic &> pubErr
+  echo -n "testOK2" | ipfsi 1 pubsub pub testTopic &> pubErr
 '
 
 test_expect_success "wait until echo > wait executed" '
-  echo "testOK2" > expected &&
   cat wait2 &&
   test_cmp pubErr empty &&
   test_cmp expected actual
diff --git a/test/sharness/t0180-pubsub.sh b/test/sharness/t0180-pubsub.sh
index 349e98d54c3..20d5f4c1c74 100755
--- a/test/sharness/t0180-pubsub.sh
+++ b/test/sharness/t0180-pubsub.sh
@@ -22,7 +22,7 @@ run_pubsub_tests() {
   
   # ipfs pubsub sub
   test_expect_success 'pubsub' '
-    echo "testOK" > expected &&
+    echo -n "testOK" | ipfs multibase encode -b base64url > expected &&
     touch empty &&
     mkfifo wait ||
     test_fsh echo init fail
@@ -30,8 +30,8 @@ run_pubsub_tests() {
     # ipfs pubsub sub is long-running so we need to start it in the background and
     # wait put its output somewhere where we can access it
     (
-      ipfsi 0 pubsub sub --enc=ndpayload testTopic | if read line; then
-          echo $line > actual &&
+      ipfsi 0 pubsub sub --enc=json testTopic | if read line; then
+          echo $line | jq -j .data > actual &&
           echo > wait
         fi
     ) &
@@ -61,15 +61,15 @@ run_pubsub_tests() {
   '
   
   test_expect_success "wait for another pubsub message" '
-    echo "testOK2" > expected &&
+    echo -n "testOK2" | ipfs multibase encode -b base64url > expected &&
     mkfifo wait2 ||
     test_fsh echo init fail
   
     # ipfs pubsub sub is long-running so we need to start it in the background and
     # wait put its output somewhere where we can access it
     (
-      ipfsi 2 pubsub sub --enc=ndpayload testTopic | if read line; then
-          echo $line > actual &&
+      ipfsi 2 pubsub sub --enc=json testTopic | if read line; then
+          echo $line | jq -j .data > actual &&
           echo > wait2
         fi
     ) &
@@ -80,11 +80,10 @@ run_pubsub_tests() {
   '
   
   test_expect_success "publish something" '
-    echo "testOK2" | ipfsi 3 pubsub pub testTopic &> pubErr
+    echo -n "testOK2" | ipfsi 3 pubsub pub testTopic &> pubErr
   '
   
   test_expect_success "wait until echo > wait executed" '
-    echo "testOK2" > expected &&
     cat wait2 &&
     test_cmp pubErr empty &&
     test_cmp expected actual
@@ -114,7 +113,7 @@ startup_cluster $NUM_NODES --enable-pubsub-experiment
 
 test_expect_success 'set node 4 to listen on testTopic' '
   rm -f node4_actual &&
-  ipfsi 4 pubsub sub --enc=ndpayload testTopic > node4_actual &
+  ipfsi 4 pubsub sub --enc=json testTopic > node4_actual &
 '
 
 run_pubsub_tests

From d4f80fbda916100eda8d7304af3ff074d8bdc229 Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Tue, 12 Oct 2021 12:43:21 +0200
Subject: [PATCH 06/16] test(pubsub): go-ipfs-api

Testing against compatible version from
https://github.com/ipfs/go-ipfs-api/pull/255
---
 .circleci/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/main.yml b/.circleci/main.yml
index f11c9519996..698a30a0223 100644
--- a/.circleci/main.yml
+++ b/.circleci/main.yml
@@ -237,7 +237,7 @@ jobs:
       - run:
           name: Cloning
           command: |
-            git clone https://github.com/ipfs/go-ipfs-api.git
+            git clone --branch feat/pubsub-require-multibase https://github.com/ipfs/go-ipfs-api.git
             git -C go-ipfs-api log -1
       - run:
           name: Starting the daemon

From 7afd1251870c05c24cf35871e4d97b0a1c9eab1e Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Tue, 12 Oct 2021 13:06:48 +0200
Subject: [PATCH 07/16] refactor: safeTextListEncoder

Making it clear what it does and why
---
 core/commands/pubsub.go |  9 ++++++---
 core/commands/swarm.go  | 14 +++++++-------
 2 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/core/commands/pubsub.go b/core/commands/pubsub.go
index b45d657dca2..cde8bb70d7e 100644
--- a/core/commands/pubsub.go
+++ b/core/commands/pubsub.go
@@ -252,10 +252,13 @@ func multibaseDecodedStringListEncoder(req *cmds.Request, w io.Writer, list *str
 		}
 		list.Strings[n] = string(data)
 	}
-	return stringListEncoder(req, w, list)
+	return safeTextListEncoder(req, w, list)
 }
 
-func stringListEncoder(req *cmds.Request, w io.Writer, list *stringList) error {
+// converts list of strings to text representation where each string is placed
+// in separate line with non-printable/unsafe characters escaped
+// (this protects terminal output from being mangled by non-ascii topic names)
+func safeTextListEncoder(req *cmds.Request, w io.Writer, list *stringList) error {
 	for _, str := range list.Strings {
 		_, err := fmt.Fprintf(w, "%s\n", cmdenv.EscNonPrint(str))
 		if err != nil {
@@ -322,7 +325,7 @@ TOPIC AND DATA ENCODING
 	},
 	Type: stringList{},
 	Encoders: cmds.EncoderMap{
-		cmds.Text: cmds.MakeTypedEncoder(stringListEncoder),
+		cmds.Text: cmds.MakeTypedEncoder(safeTextListEncoder),
 	},
 }
 
diff --git a/core/commands/swarm.go b/core/commands/swarm.go
index 7cde3f38d43..7c7ee3e814f 100644
--- a/core/commands/swarm.go
+++ b/core/commands/swarm.go
@@ -453,7 +453,7 @@ var swarmAddrsLocalCmd = &cmds.Command{
 	},
 	Type: stringList{},
 	Encoders: cmds.EncoderMap{
-		cmds.Text: cmds.MakeTypedEncoder(stringListEncoder),
+		cmds.Text: cmds.MakeTypedEncoder(safeTextListEncoder),
 	},
 }
 
@@ -485,7 +485,7 @@ var swarmAddrsListenCmd = &cmds.Command{
 	},
 	Type: stringList{},
 	Encoders: cmds.EncoderMap{
-		cmds.Text: cmds.MakeTypedEncoder(stringListEncoder),
+		cmds.Text: cmds.MakeTypedEncoder(safeTextListEncoder),
 	},
 }
 
@@ -535,7 +535,7 @@ ipfs swarm connect /ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N
 		return cmds.EmitOnce(res, &stringList{output})
 	},
 	Encoders: cmds.EncoderMap{
-		cmds.Text: cmds.MakeTypedEncoder(stringListEncoder),
+		cmds.Text: cmds.MakeTypedEncoder(safeTextListEncoder),
 	},
 	Type: stringList{},
 }
@@ -600,7 +600,7 @@ it will reconnect.
 		return cmds.EmitOnce(res, &stringList{output})
 	},
 	Encoders: cmds.EncoderMap{
-		cmds.Text: cmds.MakeTypedEncoder(stringListEncoder),
+		cmds.Text: cmds.MakeTypedEncoder(safeTextListEncoder),
 	},
 	Type: stringList{},
 }
@@ -722,7 +722,7 @@ Filters default to those specified under the "Swarm.AddrFilters" config key.
 		return cmds.EmitOnce(res, &stringList{output})
 	},
 	Encoders: cmds.EncoderMap{
-		cmds.Text: cmds.MakeTypedEncoder(stringListEncoder),
+		cmds.Text: cmds.MakeTypedEncoder(safeTextListEncoder),
 	},
 	Type: stringList{},
 }
@@ -778,7 +778,7 @@ var swarmFiltersAddCmd = &cmds.Command{
 		return cmds.EmitOnce(res, &stringList{added})
 	},
 	Encoders: cmds.EncoderMap{
-		cmds.Text: cmds.MakeTypedEncoder(stringListEncoder),
+		cmds.Text: cmds.MakeTypedEncoder(safeTextListEncoder),
 	},
 	Type: stringList{},
 }
@@ -844,7 +844,7 @@ var swarmFiltersRmCmd = &cmds.Command{
 		return cmds.EmitOnce(res, &stringList{removed})
 	},
 	Encoders: cmds.EncoderMap{
-		cmds.Text: cmds.MakeTypedEncoder(stringListEncoder),
+		cmds.Text: cmds.MakeTypedEncoder(safeTextListEncoder),
 	},
 	Type: stringList{},
 }

From 31b354186bc8cc69bb1e0434a1763524003ee8d6 Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Tue, 12 Oct 2021 14:31:22 +0200
Subject: [PATCH 08/16] refactor(pubsub): unify peerids

This ensures `ipfs pubsub sub` returns the same peerids in the `From`
field as `ipfs pubsub peers`.

libp2p already uses base encoding, no need to double wrap or use custom
multibase.
---
 core/commands/pubsub.go | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/core/commands/pubsub.go b/core/commands/pubsub.go
index cde8bb70d7e..d8b13e061e7 100644
--- a/core/commands/pubsub.go
+++ b/core/commands/pubsub.go
@@ -58,9 +58,14 @@ EXPERIMENTAL FEATURE
   environment.  To use, the daemon must be run with
   '--enable-pubsub-experiment'.
 
-TOPIC ENCODING
+PEER ENCODING
 
-  Topic names are a binary data. To ensure all bytes are transferred
+  Peer IDs in From fields are encoded using the default text representation
+  from go-libp2p. This ensures the same string values as in 'ipfs pubsub peers'.
+
+TOPIC AND DATA ENCODING
+
+  Topics, Data and Seqno are binary data. To ensure all bytes are transferred
   correctly RPC client and server will use multibase encoding behind
   the scenes.
 
@@ -101,12 +106,11 @@ TOPIC ENCODING
 				return err
 			}
 
-			// encode as base64url so the same string is present in body and URL args
-			// when sent over HTTP RPC API
+			// turn bytes into strings
 			encoder, _ := mbase.EncoderByName("base64url")
 			psm := pubsubMessage{
 				Data:  encoder.Encode(msg.Data()),
-				From:  encoder.Encode([]byte(msg.From())),
+				From:  msg.From().Pretty(),
 				Seqno: encoder.Encode(msg.Seq()),
 			}
 			for _, topic := range msg.Topics() {

From 5bd282d819056ad19bd849a17d062fa27866662b Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Tue, 12 Oct 2021 17:26:44 +0200
Subject: [PATCH 09/16] test(pubsub): go-ipfs-http-client

---
 .circleci/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/main.yml b/.circleci/main.yml
index 698a30a0223..2e135a87cc8 100644
--- a/.circleci/main.yml
+++ b/.circleci/main.yml
@@ -274,7 +274,7 @@ jobs:
       - run:
           name: Cloning
           command: |
-            git clone https://github.com/ipfs/go-ipfs-http-client.git
+            git clone --branch feat/pubsub-require-multibase https://github.com/ipfs/go-ipfs-http-client.git
             git -C go-ipfs-http-client log -1
       - restore_cache:
           keys:

From e426088664d7b8af2734556c8b6cf08b3f17d76c Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Fri, 15 Oct 2021 13:38:47 +0200
Subject: [PATCH 10/16] refactor(pubsub): make pub command read from a file

We want to send payload in the body as multipart so users can use
existing tools like curl for publishing arbitrary bytes to a topic.

StringArg was created for "one message per line" use case, and if data
has `\n` or `\r\n` byte sequences, it will cause payload to be split. It
is not possible to undo this, because mentioned sequences are lost, so
we are not able to tell if it was `\n` or `\r\n`

We already avoid this problem in `block put` and `dht put` by reading
payload via FileArg which does not mangle binary data and send it as-is.
It feel like `pubsub pub` should be using it in the first place anyway,
so this commit replaces StringArg with FileArg.

This also closes https://github.com/ipfs/go-ipfs/issues/8454
and makes rpc in go-ipfs easier to code against.
---
 core/commands/pubsub.go       | 50 +++++++++++++++--------------------
 test/sharness/t0180-pubsub.sh |  7 ++---
 2 files changed, 26 insertions(+), 31 deletions(-)

diff --git a/core/commands/pubsub.go b/core/commands/pubsub.go
index d8b13e061e7..6b6cb3e7b39 100644
--- a/core/commands/pubsub.go
+++ b/core/commands/pubsub.go
@@ -6,7 +6,6 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
-	"os"
 	"sort"
 
 	cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv"
@@ -146,9 +145,10 @@ TOPIC AND DATA ENCODING
 
 var PubsubPubCmd = &cmds.Command{
 	Helptext: cmds.HelpText{
-		Tagline: "Publish a message to a given pubsub topic.",
+		Tagline: "Publish data to a given pubsub topic.",
 		ShortDescription: `
 ipfs pubsub pub publishes a message to a specified topic.
+It reads binary data from stdin or a file.
 
 EXPERIMENTAL FEATURE
 
@@ -156,31 +156,21 @@ EXPERIMENTAL FEATURE
   environment.  To use, the daemon must be run with
   '--enable-pubsub-experiment'.
 
-TOPIC AND DATA ENCODING
+HTTP RPC ENCODING
+
+  The data to be published is sent in HTTP request body as multipart/form-data.
 
   Topic names are a binary data too. To ensure all bytes are transferred
-  correctly RPC client and server will use multibase encoding behind
-  the scenes.
+  correctly via URL params, the RPC client and server will use multibase
+  encoding behind the scenes.
 
-  You can inspect the format by passing --enc=json. ipfs multibase commands
-  can be used for encoding/decoding multibase strings in the userland.
 `,
 	},
 	Arguments: []cmds.Argument{
 		cmds.StringArg("topic", true, false, "Topic to publish to."),
-		cmds.StringArg("data", false, true, "Payload of message to publish."),
-	},
-	PreRun: func(req *cmds.Request, env cmds.Environment) error {
-		// when there are no string args with data, read from stdin.
-		if len(req.Arguments) == 1 {
-			buf, err := ioutil.ReadAll(os.Stdin)
-			if err != nil {
-				return err
-			}
-			req.Arguments = append(req.Arguments, string(buf))
-		}
-		return urlArgsEncoder(req, env)
+		cmds.FileArg("data", true, false, "The data to be published.").EnableStdin(),
 	},
+	PreRun: urlArgsEncoder,
 	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
 		api, err := cmdenv.GetApi(env, req)
 		if err != nil {
@@ -191,13 +181,20 @@ TOPIC AND DATA ENCODING
 		}
 
 		topic := req.Arguments[0]
-		for _, data := range req.Arguments[1:] {
-			if err := api.PubSub().Publish(req.Context, topic, []byte(data)); err != nil {
-				return err
-			}
+
+		// read data passed as a file
+		file, err := cmdenv.GetFileArg(req.Files.Entries())
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+		data, err := ioutil.ReadAll(file)
+		if err != nil {
+			return err
 		}
 
-		return nil
+		// publish
+		return api.PubSub().Publish(req.Context, topic, data)
 	},
 }
 
@@ -333,6 +330,7 @@ TOPIC AND DATA ENCODING
 	},
 }
 
+// TODO: move to cmdenv?
 // Encode binary data to be passed as multibase string in URL arguments.
 // (avoiding issues described in https://github.com/ipfs/go-ipfs/issues/7939)
 func urlArgsEncoder(req *cmds.Request, env cmds.Environment) error {
@@ -346,10 +344,6 @@ func urlArgsEncoder(req *cmds.Request, env cmds.Environment) error {
 // Decode binary data passed as multibase string in URL arguments.
 // (avoiding issues described in https://github.com/ipfs/go-ipfs/issues/7939)
 func urlArgsDecoder(req *cmds.Request, env cmds.Environment) error {
-	err := req.ParseBodyArgs()
-	if err != nil {
-		return err
-	}
 	for n, arg := range req.Arguments {
 		encoding, data, err := mbase.Decode(arg)
 		if err != nil {
diff --git a/test/sharness/t0180-pubsub.sh b/test/sharness/t0180-pubsub.sh
index 20d5f4c1c74..08e05ec9c86 100755
--- a/test/sharness/t0180-pubsub.sh
+++ b/test/sharness/t0180-pubsub.sh
@@ -50,8 +50,9 @@ run_pubsub_tests() {
     test_cmp peers_exp peers_out
   '
   
-  test_expect_success "publish something" '
-    ipfsi 1 pubsub pub testTopic "testOK" &> pubErr
+  test_expect_success "publish something from file" '
+    echo -n "testOK" > payload-file &&
+    ipfsi 1 pubsub pub testTopic payload-file &> pubErr
   '
   
   test_expect_success "wait until echo > wait executed" '
@@ -79,7 +80,7 @@ run_pubsub_tests() {
     go-sleep 500ms
   '
   
-  test_expect_success "publish something" '
+  test_expect_success "publish something from stdin" '
     echo -n "testOK2" | ipfsi 3 pubsub pub testTopic &> pubErr
   '
   

From 84e0eb18ce21b5b7c0bd0a8e1586347765fd0115 Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Fri, 15 Oct 2021 14:46:42 +0200
Subject: [PATCH 11/16] test(pubsub): publishing with line breaks

Making sure we don't see regressions in the future.
Ref. https://github.com/ipfs/go-ipfs/issues/7939
---
 test/sharness/t0180-pubsub-gossipsub.sh | 11 ++++++-----
 test/sharness/t0180-pubsub.sh           |  8 ++++----
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/test/sharness/t0180-pubsub-gossipsub.sh b/test/sharness/t0180-pubsub-gossipsub.sh
index 8c0686efd1e..c89c8d1ae92 100755
--- a/test/sharness/t0180-pubsub-gossipsub.sh
+++ b/test/sharness/t0180-pubsub-gossipsub.sh
@@ -25,7 +25,7 @@ test_expect_success 'peer ids' '
 '
 
 test_expect_success 'pubsub' '
-  echo -n "testOK" | ipfs multibase encode -b base64url > expected &&
+  echo -n -e "test\nOK" | ipfs multibase encode -b base64url > expected &&
   touch empty &&
   mkfifo wait ||
   test_fsh echo init fail
@@ -53,8 +53,9 @@ test_expect_success "output looks good" '
   test_cmp peers_exp peers_out
 '
 
-test_expect_success "publish something" '
-  ipfsi 1 pubsub pub testTopic "testOK" &> pubErr
+test_expect_success "publish something from a file" '
+  echo -n -e "test\nOK" > payload-file &&
+  ipfsi 1 pubsub pub testTopic payload-file &> pubErr
 '
 
 test_expect_success "wait until echo > wait executed" '
@@ -64,7 +65,7 @@ test_expect_success "wait until echo > wait executed" '
 '
 
 test_expect_success "wait for another pubsub message" '
-  echo -n "testOK2" | ipfs multibase encode -b base64url > expected &&
+  echo -n -e "test\nOK2" | ipfs multibase encode -b base64url > expected &&
   mkfifo wait2 ||
   test_fsh echo init fail
 
@@ -83,7 +84,7 @@ test_expect_success "wait until ipfs pubsub sub is ready to do work" '
 '
 
 test_expect_success "publish something" '
-  echo -n "testOK2" | ipfsi 1 pubsub pub testTopic &> pubErr
+  echo -n -e "test\nOK2" | ipfsi 1 pubsub pub testTopic &> pubErr
 '
 
 test_expect_success "wait until echo > wait executed" '
diff --git a/test/sharness/t0180-pubsub.sh b/test/sharness/t0180-pubsub.sh
index 08e05ec9c86..04b120c3e21 100755
--- a/test/sharness/t0180-pubsub.sh
+++ b/test/sharness/t0180-pubsub.sh
@@ -22,7 +22,7 @@ run_pubsub_tests() {
   
   # ipfs pubsub sub
   test_expect_success 'pubsub' '
-    echo -n "testOK" | ipfs multibase encode -b base64url > expected &&
+    echo -n -e "test\nOK" | ipfs multibase encode -b base64url > expected &&
     touch empty &&
     mkfifo wait ||
     test_fsh echo init fail
@@ -51,7 +51,7 @@ run_pubsub_tests() {
   '
   
   test_expect_success "publish something from file" '
-    echo -n "testOK" > payload-file &&
+    echo -n -e "test\nOK" > payload-file &&
     ipfsi 1 pubsub pub testTopic payload-file &> pubErr
   '
   
@@ -62,7 +62,7 @@ run_pubsub_tests() {
   '
   
   test_expect_success "wait for another pubsub message" '
-    echo -n "testOK2" | ipfs multibase encode -b base64url > expected &&
+    echo -n -e "test\nOK\r\n2" | ipfs multibase encode -b base64url > expected &&
     mkfifo wait2 ||
     test_fsh echo init fail
   
@@ -81,7 +81,7 @@ run_pubsub_tests() {
   '
   
   test_expect_success "publish something from stdin" '
-    echo -n "testOK2" | ipfsi 3 pubsub pub testTopic &> pubErr
+    echo -n -e "test\nOK\r\n2" | ipfsi 3 pubsub pub testTopic &> pubErr
   '
   
   test_expect_success "wait until echo > wait executed" '

From 153697d524f449ee9bec97245b0fcd7ebc2e8170 Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Mon, 25 Oct 2021 20:32:52 +0200
Subject: [PATCH 12/16] chore: disable pubsub interop for now

See
https://github.com/ipfs/interop/commit/344f692d8cdc68fabe424814214dfb43c716edac
---
 .circleci/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/main.yml b/.circleci/main.yml
index 2e135a87cc8..260b3f2cd3b 100644
--- a/.circleci/main.yml
+++ b/.circleci/main.yml
@@ -212,7 +212,7 @@ jobs:
           command: |
             npm init -y
             npm install ipfs@^0.59.1
-            npm install ipfs-interop@^7.0.3
+            npm install ipfs-interop@^7.0.5
             npm install mocha-circleci-reporter@0.0.3
           working_directory: ~/ipfs/go-ipfs/interop
       - run:

From 8c773776ce78922ab73e4391f819cb8c38a23ad1 Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Tue, 23 Nov 2021 16:36:55 +0100
Subject: [PATCH 13/16] test: t0322-pubsub-http-rpc.sh

- Adds HTTP RPC regression test that ensures topic is encoded as URL-safe
  multibase.
- Moves pubsub tests to live in unique range ./t032x
---
 .../{t0180-pubsub.sh => t0320-pubsub.sh}      | 26 ++++++++---------
 ...gossipsub.sh => t0321-pubsub-gossipsub.sh} |  0
 test/sharness/t0322-pubsub-http-rpc.sh        | 29 +++++++++++++++++++
 3 files changed, 42 insertions(+), 13 deletions(-)
 rename test/sharness/{t0180-pubsub.sh => t0320-pubsub.sh} (98%)
 rename test/sharness/{t0180-pubsub-gossipsub.sh => t0321-pubsub-gossipsub.sh} (100%)
 create mode 100755 test/sharness/t0322-pubsub-http-rpc.sh

diff --git a/test/sharness/t0180-pubsub.sh b/test/sharness/t0320-pubsub.sh
similarity index 98%
rename from test/sharness/t0180-pubsub.sh
rename to test/sharness/t0320-pubsub.sh
index 04b120c3e21..44ee8b58285 100755
--- a/test/sharness/t0180-pubsub.sh
+++ b/test/sharness/t0320-pubsub.sh
@@ -19,14 +19,14 @@ run_pubsub_tests() {
     PEERID_0=$(iptb attr get 0 id) &&
     PEERID_2=$(iptb attr get 2 id)
   '
-  
+
   # ipfs pubsub sub
   test_expect_success 'pubsub' '
     echo -n -e "test\nOK" | ipfs multibase encode -b base64url > expected &&
     touch empty &&
     mkfifo wait ||
     test_fsh echo init fail
-  
+
     # ipfs pubsub sub is long-running so we need to start it in the background and
     # wait put its output somewhere where we can access it
     (
@@ -36,36 +36,36 @@ run_pubsub_tests() {
         fi
     ) &
   '
-  
+
   test_expect_success "wait until ipfs pubsub sub is ready to do work" '
     go-sleep 500ms
   '
-  
+
   test_expect_success "can see peer subscribed to testTopic" '
     ipfsi 1 pubsub peers testTopic > peers_out
   '
-  
+
   test_expect_success "output looks good" '
     echo $PEERID_0 > peers_exp &&
     test_cmp peers_exp peers_out
   '
-  
+
   test_expect_success "publish something from file" '
     echo -n -e "test\nOK" > payload-file &&
     ipfsi 1 pubsub pub testTopic payload-file &> pubErr
   '
-  
+
   test_expect_success "wait until echo > wait executed" '
     cat wait &&
     test_cmp pubErr empty &&
     test_cmp expected actual
   '
-  
+
   test_expect_success "wait for another pubsub message" '
     echo -n -e "test\nOK\r\n2" | ipfs multibase encode -b base64url > expected &&
     mkfifo wait2 ||
     test_fsh echo init fail
-  
+
     # ipfs pubsub sub is long-running so we need to start it in the background and
     # wait put its output somewhere where we can access it
     (
@@ -75,15 +75,15 @@ run_pubsub_tests() {
         fi
     ) &
   '
-  
+
   test_expect_success "wait until ipfs pubsub sub is ready to do work" '
     go-sleep 500ms
   '
-  
+
   test_expect_success "publish something from stdin" '
     echo -n -e "test\nOK\r\n2" | ipfsi 3 pubsub pub testTopic &> pubErr
   '
-  
+
   test_expect_success "wait until echo > wait executed" '
     cat wait2 &&
     test_cmp pubErr empty &&
@@ -93,7 +93,7 @@ run_pubsub_tests() {
   test_expect_success 'cleanup fifos' '
     rm -f wait wait2
   '
-  
+
 }
 
 # Normal tests
diff --git a/test/sharness/t0180-pubsub-gossipsub.sh b/test/sharness/t0321-pubsub-gossipsub.sh
similarity index 100%
rename from test/sharness/t0180-pubsub-gossipsub.sh
rename to test/sharness/t0321-pubsub-gossipsub.sh
diff --git a/test/sharness/t0322-pubsub-http-rpc.sh b/test/sharness/t0322-pubsub-http-rpc.sh
new file mode 100755
index 00000000000..4ecfefb6977
--- /dev/null
+++ b/test/sharness/t0322-pubsub-http-rpc.sh
@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+
+test_description="Test pubsub command behavior over HTTP RPC API"
+
+. lib/test-lib.sh
+
+test_init_ipfs
+test_launch_ipfs_daemon --enable-pubsub-experiment
+
+# Require topic as multibase
+# https://github.com/ipfs/go-ipfs/pull/8183
+test_expect_success "/api/v0/pubsub/pub URL arg must be multibase encoded" '
+  echo test > data.txt &&
+  curl -s -X POST -F "data=@data.txt" "$API_ADDR/api/v0/pubsub/pub?arg=foobar" > result &&
+  test_should_contain "error" result &&
+  test_should_contain "URL arg must be multibase encoded" result
+'
+
+# Use URL-safe multibase
+# base64 should produce error when used in URL args, base64url should be used
+test_expect_success "/api/v0/pubsub/pub URL arg must be in URL-safe multibase" '
+  echo test > data.txt &&
+  curl -s -X POST -F "data=@data.txt" "$API_ADDR/api/v0/pubsub/pub?arg=mZm9vYmFyCg" > result &&
+  test_should_contain "error" result &&
+  test_should_contain "URL arg must be base64url encoded" result
+'
+
+test_kill_ipfs_daemon
+test_done

From 99550715a4b4be056310f0c324a576de3746e1b8 Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Mon, 29 Nov 2021 22:29:00 +0100
Subject: [PATCH 14/16] chore: switch to master versions of http clients

---
 .circleci/main.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.circleci/main.yml b/.circleci/main.yml
index 844f9b6c4fc..061bffac86e 100644
--- a/.circleci/main.yml
+++ b/.circleci/main.yml
@@ -253,7 +253,7 @@ jobs:
       - run:
           name: Cloning
           command: |
-            git clone --branch feat/pubsub-require-multibase https://github.com/ipfs/go-ipfs-api.git
+            git clone https://github.com/ipfs/go-ipfs-api.git
             git -C go-ipfs-api log -1
       - run:
           name: Starting the daemon
@@ -290,7 +290,7 @@ jobs:
       - run:
           name: Cloning
           command: |
-            git clone --branch feat/pubsub-require-multibase https://github.com/ipfs/go-ipfs-http-client.git
+            git clone https://github.com/ipfs/go-ipfs-http-client.git
             git -C go-ipfs-http-client log -1
       - restore_cache:
           keys:

From 582eaf1bc6dd753498db13cb1c7d334b87a6d1e7 Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Mon, 29 Nov 2021 22:46:14 +0100
Subject: [PATCH 15/16] fix(ci):  js-ipfs with  fixed pubsub wire format

uses js-ipfs from
https://github.com/ipfs/js-ipfs/pull/3922
---
 .circleci/main.yml | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/.circleci/main.yml b/.circleci/main.yml
index 061bffac86e..8139b97d7dd 100644
--- a/.circleci/main.yml
+++ b/.circleci/main.yml
@@ -234,6 +234,15 @@ jobs:
       - run:
           name: Running tests
           command: |
+            WORKDIR=$(pwd)
+            cd /tmp
+            git clone https://github.com/ipfs/js-ipfs.git
+            cd js-ipfs
+            git checkout 1dcac76f56972fc3519526e93567e39d685033dd
+            npm install
+            npm run build
+            npm run link
+            cd $WORKDIR
             mkdir -p /tmp/test-results/interop/
             export MOCHA_FILE="$(mktemp /tmp/test-results/interop/unit.XXXXXX.xml)"
             npx ipfs-interop -- -t node -f $(sed -n -e "s|^require('\(.*\)')$|test/\1|p" node_modules/ipfs-interop/test/node.js | circleci tests split) -- --reporter mocha-circleci-reporter
@@ -242,6 +251,9 @@ jobs:
             LIBP2P_TCP_REUSEPORT: false
             LIBP2P_ALLOW_WEAK_RSA_KEYS: 1
             IPFS_GO_EXEC: /tmp/circleci-workspace/bin/ipfs
+            IPFS_JS_EXEC: /tmp/js-ipfs/packages/ipfs/src/cli.js
+            IPFS_JS_MODULE: /tmp/js-ipfs/packages/ipfs/dist/cjs/src/index.js
+            IPFS_JS_HTTP_MODULE: /tmp/js-ipfs/packages/ipfs-http-client/dist/cjs/src/index.js
       - store_test_results:
           path: /tmp/test-results
   go-ipfs-api:

From 1a8d7715e19c853812bc3e1b81838dcfd2ecd2d3 Mon Sep 17 00:00:00 2001
From: Adin Schmahmann <adin.schmahmann@gmail.com>
Date: Mon, 29 Nov 2021 17:02:02 -0500
Subject: [PATCH 16/16] docs: grammar fixes

---
 core/commands/pubsub.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/core/commands/pubsub.go b/core/commands/pubsub.go
index 6b6cb3e7b39..47d893852af 100644
--- a/core/commands/pubsub.go
+++ b/core/commands/pubsub.go
@@ -65,10 +65,10 @@ PEER ENCODING
 TOPIC AND DATA ENCODING
 
   Topics, Data and Seqno are binary data. To ensure all bytes are transferred
-  correctly RPC client and server will use multibase encoding behind
+  correctly the RPC client and server will use multibase encoding behind
   the scenes.
 
-  You can inspect the format by passing --enc=json. ipfs multibase commands
+  You can inspect the format by passing --enc=json. The ipfs multibase commands
   can be used for encoding/decoding multibase strings in the userland.
 `,
 	},
@@ -160,7 +160,7 @@ HTTP RPC ENCODING
 
   The data to be published is sent in HTTP request body as multipart/form-data.
 
-  Topic names are a binary data too. To ensure all bytes are transferred
+  Topic names are binary data too. To ensure all bytes are transferred
   correctly via URL params, the RPC client and server will use multibase
   encoding behind the scenes.