Merge branch 'master' into swarmy

.travis.yml

@@ -9,6 +9,7 @@ env:
  - SSL_MODE_DISABLE=true
  global:
  - GO111MODULE=on
+ - RECAPTCHA_KEY=cantbeblankortestsfail
  # decodes into SENGRID_API_KEY environment variable
  - secure: GXKJ+1Wa+pqUm1PKHJMbDrejSfIsqhsZ6l88i0paTvVou9T/mv03mx9hzrSLsYXFFVjinfbKysxMtEorTFDl0YBSNxuPc7eaiJinnoptubBh++bvEzF/A8wmWHgIBxroFIrK4SNsnreCXPqT1YBQZWi3JenMkzS68j1oa/uPa5ODZ3rjx1Wmu6hSANSZPqzOoGO6lnRP8G30oiFzbawEnB/52iBcHhPtffUihLFnn3k1wXyL8fpi4JYuyNQe84Br+w91KSS2nMmG+RVvJWumHNGXFyjEVv0n7HbirjCBI1iZo50bTdFtujkGSOCoHqM1hwa2yLWISipkb19Ls4eeWcWZBFpcdBhIIeHEmz9iCXxn0ksIGzuw00Xof/HTcWUpnzzQFq4E8iVE4tmMrFbYEcD5vcGB3S7bURjTD38uZ3/7I4Oyuo33/WQGdDNIzugU/dycGkSNzwTeIB3yJfpr/wbC6lU5RnADh5Ej5hGreAfkng/iuC9T36Fnn9u4fTocj++smOxZSjbbKsyekNNdadYmU0sfd6Ka9mFcv8H6sDs7zwqyZLTDu6P57CN5VFz4Yjc1emupFX5NFWor5jull9cr+ilOmfnBSCQPV0ArKrpKjrL9K9NJAhRKOqxmpLKx3pozGnrQMFKM0xsSuTcOCdyTdZF8JEQESeENHzwyoB4=
 install:

api/middleware/jwt.go

@@ -48,16 +48,21 @@ func JwtConfigGenerate(jwtKey, realmName string, db *gorm.DB, l *zap.SugaredLogg
  return "", false
  }
  }
+ // email enabled implies they have verified their email
+ if !usr.EmailEnabled {
+ return "", false
+ }
  lAuth.Info("successful login", "username", usr.UserName)
  return usr.UserName, true
  },
  Authorizator: func(userId string, c *gin.Context) bool {
  // as a final security step, ensure that we can find the user in our database
  userManager := models.NewUserManager(db)
- if _, err := userManager.FindByUserName(userId); err != nil {
+ usr, err := userManager.FindByUserName(userId)
+ if err != nil {
  return false
  }
- return true
+ return usr.EmailEnabled && usr.AccountEnabled
  },
  Unauthorized: func(c *gin.Context, code int, message string) {
  l.Error("invalid login detected")

api/v2/api.go

@@ -9,22 +9,21 @@ import (
  "strconv"
  "time"
 
- "github.com/streadway/amqp"
-
+ "github.com/RTradeLtd/ChainRider-Go/dash"
+ "github.com/RTradeLtd/Temporal/queue"
  "github.com/RTradeLtd/Temporal/rtfscluster"
  pbLens "github.com/RTradeLtd/grpc/lensv2"
  pbOrch "github.com/RTradeLtd/grpc/nexus"
  pbSigner "github.com/RTradeLtd/grpc/pay"
- pbBchWallet "github.com/gcash/bchwallet/rpc/walletrpc"
-
  "github.com/RTradeLtd/kaas/v2"
- "go.uber.org/zap"
-
- "github.com/RTradeLtd/ChainRider-Go/dash"
- "github.com/RTradeLtd/Temporal/queue"
  "github.com/RTradeLtd/rtfs/v2"
-
- limit "github.com/aviddiviner/gin-limit"
+ recaptcha "github.com/ezzarghili/recaptcha-go"
+ pbBchWallet "github.com/gcash/bchwallet/rpc/walletrpc"
+ "github.com/streadway/amqp"
+ "github.com/ulule/limiter/v3"
+ mgin "github.com/ulule/limiter/v3/drivers/middleware/gin"
+ "github.com/ulule/limiter/v3/drivers/store/memory"
+ "go.uber.org/zap"
 
  "github.com/RTradeLtd/config/v2"
  stats "github.com/semihalev/gin-stats"
@@ -62,8 +61,10 @@ type API struct {
  dc *dash.Client
  queues queues
  service string
- cmcAPIKey string
  version string
+
+ captcha recaptcha.ReCAPTCHA
+ captchaEnabled bool
 }
 
 // Initialize is used ot initialize our API service. debug = true is useful
@@ -83,6 +84,8 @@ func Initialize(
  err error
  router = gin.Default()
  )
+ // if we dont set this, rate limiting wont work properly
+ router.ForwardedByClientIP = true
  // update dev mode
  dev = opts.DevMode
  l = l.Named("api")
@@ -108,7 +111,14 @@ func Initialize(
  return nil, err
  }
  api.version = version
-
+ if api.getCaptchaKey() != "" {
+ captcha, err := recaptcha.NewReCAPTCHA(api.getCaptchaKey(), recaptcha.V3, time.Second*20)
+ if err != nil {
+ return nil, err
+ }
+ api.captcha = captcha
+ api.captchaEnabled = true
+ }
  // init routes
  if err = api.setupRoutes(opts.DebugLogging); err != nil {
  return nil, err
@@ -124,7 +134,6 @@ func new(cfg *config.TemporalConfig, router *gin.Engine, l *zap.SugaredLogger, c
  dbm *database.Manager
  err error
  )
-
  // set up database manager
  dbm, err = database.New(cfg, database.Options{LogMode: debug})
  if err != nil {
@@ -379,6 +388,18 @@ func (api *API) setupRoutes(debug bool) error {
  return err
  }
  }
+ // make sure we dont throttle dev too much
+ var rateLimit string
+ if dev {
+ rateLimit = "100000-H"
+ } else {
+ rateLimit = fmt.Sprintf("%v-H", connLimit)
+ }
+ rate, err := limiter.NewRateFromFormatted(rateLimit)
+ if err != nil {
+ return err
+ }
+
  // ensure we have valid cors configuration, otherwise default to allow all
  var allowedOrigins []string
  if len(api.cfg.API.Connection.CORS.AllowedOrigins) > 0 {
@@ -392,7 +413,7 @@ func (api *API) setupRoutes(debug bool) error {
  // greater than what can be configured with HTTP Headers
  xssMdlwr.RemoveXss(),
  // rate limiting
- limit.MaxAllowed(connLimit),
+ mgin.NewMiddleware(limiter.New(memory.NewStore(), rate)),
  // security middleware
  middleware.NewSecWare(dev),
  // request id middleware
@@ -650,6 +671,13 @@ func (api *API) setupRoutes(debug bool) error {
  ens.POST("/claim", api.ClaimENSName)
  ens.POST("/update", api.UpdateContentHash)
  }
+ if api.captchaEnabled {
+ recap := v2.Group("/captcha")
+ {
+ recap.POST("/verify", api.verifyCaptcha)
+ }
+
+ }
 
  // swarm routes
  swarm := v2.Group("/swarm")

api/v2/api_test.go

@@ -256,7 +256,8 @@ func Test_API_Setup(t *testing.T) {
  args args
  wantCode int
  }{
- {"Login-testuser2", args{"POST", "/v2/auth/login", "testuser2", "password123!@#$%^&&**(!@#!", ""}, 200},
+ // this test should fail due to invalid email
+ {"Login-testuser2", args{"POST", "/v2/auth/login", "testuser2", "password123!@#$%^&&**(!@#!", ""}, 401},
  {"Login-testuser", args{"POST", "/v2/auth/login", "testuser", "admin", ""}, 200},
  // tests login via the email instead of using
  {"Login-TestUser-Email", args{"POST", "/v2/auth/login", "test@email.com", "admin", ""}, 200},
@@ -272,7 +273,7 @@ func Test_API_Setup(t *testing.T) {
  )
  api.r.ServeHTTP(testRecorder, req)
  if testRecorder.Code != tt.wantCode {
- t.Fatalf("bad http status code from %s", tt.args.call)
+ t.Fatalf("bad http status code from %s. got %v, want %v", tt.args.call, testRecorder.Code, tt.wantCode)
  }
  bodBytes, err := ioutil.ReadAll(testRecorder.Result().Body)
  if err != nil {

api/v2/routes_account.go

@@ -328,6 +328,9 @@ func (api *API) upgradeAccount(c *gin.Context) {
  api.LogError(c, err, eh.UserSearchError)(http.StatusBadRequest)
  return
  }
+ if usages.Tier == models.Unverified {
+ Fail(c, errors.New("unverified account upgrade process must be done via email verification"))
+ }
  // prevent people from repeatedly calling this granting perpetual credits
  if usages.Tier != models.Free {
  Fail(c, errors.New("user account is already upgrade"))

api/v2/routes_ens.go

@@ -24,7 +24,7 @@ func (api *API) ClaimENSName(c *gin.Context) {
  return
  }
  // prevent processing if account is free tier
- if usage.Tier == models.Free {
+ if usage.Tier == models.Free || usage.Tier == models.Unverified {
  Fail(c, errors.New("free accounts not eligible for ens claim"), http.StatusBadRequest)
  return
  }
@@ -74,7 +74,7 @@ func (api *API) UpdateContentHash(c *gin.Context) {
  return
  }
  // prevent processing if account is free tier
- if usage.Tier == models.Free {
+ if usage.Tier == models.Free || usage.Tier == models.Unverified {
  Fail(c, errors.New("free accounts not eligible for ens claim"), http.StatusBadRequest)
  return
  }

api/v2/routes_frontend.go

@@ -35,7 +35,7 @@ func (api *API) calculatePinCost(c *gin.Context) {
  return
  }
  // calculate pin cost
- totalCost, err := utils.CalculatePinCost(username, hash, holdTimeInt, api.ipfs, api.usage)
+ totalCost, _, err := utils.CalculatePinCost(username, hash, holdTimeInt, api.ipfs, api.usage)
  if err != nil {
  api.LogError(c, err, eh.CostCalculationError)(http.StatusBadRequest)
  Fail(c, err)

api/v2/routes_ipns.go

@@ -167,14 +167,8 @@ func (api *API) pinIPNSHash(c *gin.Context) {
  Respond(c, http.StatusBadRequest, gin.H{"response": alreadyUploadedMessage})
  return
  }
- // get size of object
- stats, err := api.ipfs.Stat(hash)
- if err != nil {
- api.LogError(c, err, eh.IPFSObjectStatError)(http.StatusBadRequest)
- return
- }
  // get the cost of this object
- cost, err := utils.CalculatePinCost(username, hash, holdTimeInt, api.ipfs, api.usage)
+ cost, size, err := utils.CalculatePinCost(username, hash, holdTimeInt, api.ipfs, api.usage)
  if err != nil {
  api.LogError(c, err, eh.CostCalculationError)(http.StatusBadRequest)
  return
@@ -184,7 +178,7 @@ func (api *API) pinIPNSHash(c *gin.Context) {
  api.LogError(c, err, eh.InvalidBalanceError)(http.StatusPaymentRequired)
  return
  }
- if err := api.usage.UpdateDataUsage(username, uint64(stats.CumulativeSize)); err != nil {
+ if err := api.usage.UpdateDataUsage(username, uint64(size)); err != nil {
  api.LogError(c, err, eh.CantUploadError)(http.StatusBadRequest)
  api.refundUserCredits(username, "pin", cost)
  return
@@ -196,13 +190,13 @@ func (api *API) pinIPNSHash(c *gin.Context) {
  UserName: username,
  HoldTimeInMonths: holdTimeInt,
  CreditCost: cost,
- Size: int64(stats.CumulativeSize),
+ Size: int64(size),
  }
  // send message for processing
  if err = api.queues.cluster.PublishMessage(qp); err != nil {
  api.LogError(c, err, eh.QueuePublishError)(http.StatusBadRequest)
  api.refundUserCredits(username, "pin", cost)
- api.usage.ReduceDataUsage(username, uint64(stats.CumulativeSize))
+ api.usage.ReduceDataUsage(username, uint64(size))
  return
  }
  // log and return

api/v2/routes_ipns_test.go

@@ -188,6 +188,12 @@ func Test_API_Routes_IPNS_Pin(t *testing.T) {
  fakeManager.ResolveReturnsOnCall(0, validResolveResult, nil)
  fakeManager.StatReturnsOnCall(0, &shell.ObjectStats{CumulativeSize: 5000000}, nil)
  fakeManager.StatReturnsOnCall(1, &shell.ObjectStats{CumulativeSize: 5000000}, nil)
+ fakeManager.StatReturnsOnCall(2, &shell.ObjectStats{CumulativeSize: 5000000}, nil)
+ fakeManager.StatReturnsOnCall(3, &shell.ObjectStats{CumulativeSize: 5000000}, nil)
+ fakeManager.RefsReturnsOnCall(0, []string{"QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv", "QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv"}, nil)
+ fakeManager.RefsReturnsOnCall(1, []string{"QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv", "QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv"}, nil)
+ fakeManager.RefsReturnsOnCall(2, []string{"QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv", "QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv"}, nil)
+ fakeManager.RefsReturnsOnCall(3, []string{"QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv", "QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv"}, nil)
  var apiResp apiResponse
  urlValues := url.Values{}
  urlValues.Add("hold_time", tt.args.holdTime)

api/v2/routes_rtfs.go

@@ -54,14 +54,8 @@ func (api *API) pinHashLocally(c *gin.Context) {
  Respond(c, http.StatusBadRequest, gin.H{"response": alreadyUploadedMessage})
  return
  }
- // get object size
- stats, err := api.ipfs.Stat(hash)
- if err != nil {
- api.LogError(c, err, eh.IPFSObjectStatError)(http.StatusBadRequest)
- return
- }
  // determine cost of upload
- cost, err := utils.CalculatePinCost(username, hash, holdTimeInt, api.ipfs, api.usage)
+ cost, size, err := utils.CalculatePinCost(username, hash, holdTimeInt, api.ipfs, api.usage)
  if err != nil {
  api.LogError(c, err, eh.CostCalculationError)(http.StatusBadRequest)
  return
@@ -72,7 +66,7 @@ func (api *API) pinHashLocally(c *gin.Context) {
  return
  }
  // update their data usage
- if err := api.usage.UpdateDataUsage(username, uint64(stats.CumulativeSize)); err != nil {
+ if err := api.usage.UpdateDataUsage(username, uint64(size)); err != nil {
  api.LogError(c, err, eh.CantUploadError)(http.StatusBadRequest)
  api.refundUserCredits(username, "pin", cost)
  return
@@ -83,15 +77,15 @@ func (api *API) pinHashLocally(c *gin.Context) {
  NetworkName: "public",
  UserName: username,
  HoldTimeInMonths: holdTimeInt,
- Size: int64(stats.CumulativeSize),
+ Size: size,
  CreditCost: cost,
  FileName: c.PostForm("file_name"),
  }
  // sent pin message
  if err = api.queues.cluster.PublishMessage(qp); err != nil {
  api.LogError(c, err, eh.QueuePublishError)(http.StatusBadRequest)
  api.refundUserCredits(username, "pin", cost)
- api.usage.ReduceDataUsage(username, uint64(stats.CumulativeSize))
+ api.usage.ReduceDataUsage(username, uint64(size))
  return
  }
  // log success and return
@@ -197,7 +191,7 @@ func (api *API) addFile(c *gin.Context) {
  // if the user is within the free tier, then we throttle on-demand encryption
  // free accounts are limited to a file upload size of 275MB when performing
  // on-demand encryption. Non free accounts do not have this limit
- if userUsage.Tier == models.Free {
+ if userUsage.Tier == models.Free || userUsage.Tier == models.Unverified {
  megabytesUint := datasize.MB.Bytes()
  maxSize := megabytesUint * 275
  if fileHandler.Size > int64(maxSize) {
@@ -381,7 +375,7 @@ func (api *API) extendPin(c *gin.Context) {
  return
  }
  // calculate cost of hold time extension
- cost, err := utils.CalculatePinCost(username, hash, holdTimeInt, api.ipfs, api.usage)
+ cost, _, err := utils.CalculatePinCost(username, hash, holdTimeInt, api.ipfs, api.usage)
  if err != nil {
  api.LogError(c, err, eh.CostCalculationError)(http.StatusBadRequest)
  return

api/v2/routes_utils.go

@@ -17,6 +17,7 @@ import (
  pb "github.com/RTradeLtd/grpc/krab"
  "github.com/RTradeLtd/rtfs/v2"
  "github.com/RTradeLtd/rtfs/v2/beam"
+ "github.com/ezzarghili/recaptcha-go"
  "github.com/gin-gonic/gin"
  gocid "github.com/ipfs/go-cid"
 )
@@ -343,7 +344,7 @@ func (api *API) handleUserCreate(c *gin.Context, forms map[string]string, create
  "<br>",
  "<br>",
  "Lastly let's talk about emails! We try our best to not spam your inbox, so we limit emails to a few things: payment notifications, pin expiration warnings, password/username retrieval and processing failures.\n",
- "But before we do this, you must validate your email. Note that email validation is not required unless you want these notifications\n",
+ "But before we do this, you must validate your email. Additionally before validating your email, you are in the 'unverified' tier which is limited to 100MB of data consumption. Email verification is now mandatory\n",
  "To validate your email, just click the following "+link+"\n",
  "<br>",
  "<br>",
@@ -388,3 +389,15 @@ func (api *API) handleUserCreate(c *gin.Context, forms map[string]string, create
  },
  })
 }
+
+func (api *API) verifyCaptcha(c *gin.Context) {
+ err := api.captcha.VerifyWithOptions(
+ c.PostForm("g-recaptcha-response"),
+ // require a threshold of 0.8, default is 0.5
+ recaptcha.VerifyOption{Threshold: 0.8},
+ )
+ if err != nil {
+ Fail(c, errors.New("captcha validation failed"))
+ }
+ Respond(c, http.StatusOK, gin.H{"response": "captcha validation succeeded"})
+}