From 82c950ecd8eb5339359eb82d7b18c26a6d22c92f Mon Sep 17 00:00:00 2001
From: postables <postables@rtradetechnologies.com>
Date: Mon, 13 Apr 2020 13:57:24 -0700
Subject: [PATCH 01/25] api/v2: add basic captcha verification

---
 api/v2/api.go          | 18 ++++++++++++++----
 api/v2/routes_utils.go | 13 +++++++++++++
 api/v2/utils.go        |  7 +++++++
 go.mod                 |  1 +
 go.sum                 |  2 ++
 5 files changed, 37 insertions(+), 4 deletions(-)

diff --git a/api/v2/api.go b/api/v2/api.go
index c3a1a6e0a..d4aa6f8af 100644
--- a/api/v2/api.go
+++ b/api/v2/api.go
@@ -9,13 +9,13 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/streadway/amqp"
-
 	"github.com/RTradeLtd/Temporal/rtfscluster"
 	pbLens "github.com/RTradeLtd/grpc/lensv2"
 	pbOrch "github.com/RTradeLtd/grpc/nexus"
 	pbSigner "github.com/RTradeLtd/grpc/pay"
+	recaptcha "github.com/ezzarghili/recaptcha-go"
 	pbBchWallet "github.com/gcash/bchwallet/rpc/walletrpc"
+	"github.com/streadway/amqp"
 
 	"github.com/RTradeLtd/kaas/v2"
 	"go.uber.org/zap"
@@ -64,6 +64,8 @@ type API struct {
 	service     string
 	cmcAPIKey   string
 	version     string
+
+	captcha recaptcha.ReCAPTCHA
 }
 
 // Initialize is used ot initialize our API service. debug = true is useful
@@ -108,7 +110,11 @@ func Initialize(
 		return nil, err
 	}
 	api.version = version
-
+	captcha, err := recaptcha.NewReCAPTCHA(api.getCaptchaKey(), recaptcha.V3, time.Second*10)
+	if err != nil {
+		return nil, err
+	}
+	api.captcha = captcha
 	// init routes
 	if err = api.setupRoutes(opts.DebugLogging); err != nil {
 		return nil, err
@@ -124,7 +130,6 @@ func new(cfg *config.TemporalConfig, router *gin.Engine, l *zap.SugaredLogger, c
 		dbm *database.Manager
 		err error
 	)
-
 	// set up database manager
 	dbm, err = database.New(cfg, database.Options{LogMode: debug})
 	if err != nil {
@@ -651,6 +656,11 @@ func (api *API) setupRoutes(debug bool) error {
 		ens.POST("/update", api.UpdateContentHash)
 	}
 
+	recap := v2.Group("/captcha")
+	{
+		recap.POST("/verify", api.verifyCaptcha)
+	}
+
 	api.l.Info("Routes initialized")
 	return nil
 }
diff --git a/api/v2/routes_utils.go b/api/v2/routes_utils.go
index 255e40196..788a932c3 100644
--- a/api/v2/routes_utils.go
+++ b/api/v2/routes_utils.go
@@ -17,6 +17,7 @@ import (
 	pb "github.com/RTradeLtd/grpc/krab"
 	"github.com/RTradeLtd/rtfs/v2"
 	"github.com/RTradeLtd/rtfs/v2/beam"
+	"github.com/ezzarghili/recaptcha-go"
 	"github.com/gin-gonic/gin"
 	gocid "github.com/ipfs/go-cid"
 )
@@ -388,3 +389,15 @@ func (api *API) handleUserCreate(c *gin.Context, forms map[string]string, create
 	},
 	})
 }
+
+func (api *API) verifyCaptcha(c *gin.Context) {
+	err := api.captcha.VerifyWithOptions(
+		c.PostForm("g-recaptcha-response"),
+		// require a threshold of 0.8, default is 0.5
+		recaptcha.VerifyOption{Threshold: 0.8},
+	)
+	if err != nil {
+		Fail(c, errors.New("captcha validation failed"))
+	}
+	Respond(c, http.StatusOK, gin.H{"response": "captcha validation succeeded"})
+}
diff --git a/api/v2/utils.go b/api/v2/utils.go
index 8ba70cbd0..5e0f9be91 100644
--- a/api/v2/utils.go
+++ b/api/v2/utils.go
@@ -277,3 +277,10 @@ func (api *API) getCMCKey() string {
 	}
 	return api.cmcAPIKey
 }
+
+func (api *API) getCaptchaKey() string {
+	if os.Getenv("RECAPTCHA_KEY") != "" {
+		return os.Getenv("RECAPTCHA_KEY")
+	}
+	return ""
+}
diff --git a/go.mod b/go.mod
index 16a3ae1a5..fb458b65c 100644
--- a/go.mod
+++ b/go.mod
@@ -23,6 +23,7 @@ require (
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
 	github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
 	github.com/dvwright/xss-mw v0.0.0-20191029162136-7a0dab86d8f6
+	github.com/ezzarghili/recaptcha-go v4.0.0+incompatible // indirect
 	github.com/fatih/color v1.9.0 // indirect
 	github.com/gcash/bchutil v0.0.0-20191012211144-98e73ec336ba
 	github.com/gcash/bchwallet v0.8.2
diff --git a/go.sum b/go.sum
index 3ac4ec479..c9bd90834 100644
--- a/go.sum
+++ b/go.sum
@@ -220,6 +220,8 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5 h1:Yzb9+7DPaBjB8zlTR87/ElzFsnQfuHnVUVqpZZIcV5Y=
 github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
+github.com/ezzarghili/recaptcha-go v4.0.0+incompatible h1:xmp0ucfDJY9ZAvPUtfQHkC6ahV7oOwRo5asRrczxLGg=
+github.com/ezzarghili/recaptcha-go v4.0.0+incompatible/go.mod h1:7PVEKE9sr6tm+xN2EPMSOJEwJTgYJMnXi75g1h3zMmc=
 github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 h1:BBso6MBKW8ncyZLv37o+KNyy0HrrHgfnOaGQC2qvN+A=
 github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5/go.mod h1:JpoxHjuQauoxiFMl1ie8Xc/7TfLuMZ5eOCONd1sUBHg=
 github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=

From dc4d79dabfb670d9e885436b199a551e130360b5 Mon Sep 17 00:00:00 2001
From: postables <postables@rtradetechnologies.com>
Date: Mon, 13 Apr 2020 14:01:45 -0700
Subject: [PATCH 02/25] add better rate limiter

---
 api/v2/api.go | 23 ++++++++++++++---------
 go.mod        |  5 ++---
 go.sum        | 17 +++++++++++++++--
 3 files changed, 31 insertions(+), 14 deletions(-)

diff --git a/api/v2/api.go b/api/v2/api.go
index d4aa6f8af..2383f9377 100644
--- a/api/v2/api.go
+++ b/api/v2/api.go
@@ -9,23 +9,22 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/RTradeLtd/ChainRider-Go/dash"
+	"github.com/RTradeLtd/Temporal/queue"
 	"github.com/RTradeLtd/Temporal/rtfscluster"
 	pbLens "github.com/RTradeLtd/grpc/lensv2"
 	pbOrch "github.com/RTradeLtd/grpc/nexus"
 	pbSigner "github.com/RTradeLtd/grpc/pay"
+	"github.com/RTradeLtd/kaas/v2"
+	"github.com/RTradeLtd/rtfs/v2"
 	recaptcha "github.com/ezzarghili/recaptcha-go"
 	pbBchWallet "github.com/gcash/bchwallet/rpc/walletrpc"
 	"github.com/streadway/amqp"
-
-	"github.com/RTradeLtd/kaas/v2"
+	"github.com/ulule/limiter/v3"
+	mgin "github.com/ulule/limiter/v3/drivers/middleware/gin"
+	"github.com/ulule/limiter/v3/drivers/store/memory"
 	"go.uber.org/zap"
 
-	"github.com/RTradeLtd/ChainRider-Go/dash"
-	"github.com/RTradeLtd/Temporal/queue"
-	"github.com/RTradeLtd/rtfs/v2"
-
-	limit "github.com/aviddiviner/gin-limit"
-
 	"github.com/RTradeLtd/config/v2"
 	stats "github.com/semihalev/gin-stats"
 
@@ -384,6 +383,12 @@ func (api *API) setupRoutes(debug bool) error {
 			return err
 		}
 	}
+	// 1000 requests per hour
+	rate, err := limiter.NewRateFromFormatted(fmt.Sprintf("%v-H", connLimit))
+	if err != nil {
+		return err
+	}
+
 	// ensure we have valid cors configuration, otherwise default to allow all
 	var allowedOrigins []string
 	if len(api.cfg.API.Connection.CORS.AllowedOrigins) > 0 {
@@ -397,7 +402,7 @@ func (api *API) setupRoutes(debug bool) error {
 		// greater than what can be configured with HTTP Headers
 		xssMdlwr.RemoveXss(),
 		// rate limiting
-		limit.MaxAllowed(connLimit),
+		mgin.NewMiddleware(limiter.New(memory.NewStore(), rate)),
 		// security middleware
 		middleware.NewSecWare(dev),
 		// request id middleware
diff --git a/go.mod b/go.mod
index fb458b65c..d3af32af0 100644
--- a/go.mod
+++ b/go.mod
@@ -17,13 +17,12 @@ require (
 	github.com/RTradeLtd/rtns v0.0.19
 	github.com/appleboy/gin-jwt v2.3.1+incompatible
 	github.com/appleboy/gofight/v2 v2.1.1 // indirect
-	github.com/aviddiviner/gin-limit v0.0.0-20170918012823-43b5f79762c1
 	github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23 // indirect
 	github.com/c2h5oh/datasize v0.0.0-20171227191756-4eba002a5eae
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
 	github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
 	github.com/dvwright/xss-mw v0.0.0-20191029162136-7a0dab86d8f6
-	github.com/ezzarghili/recaptcha-go v4.0.0+incompatible // indirect
+	github.com/ezzarghili/recaptcha-go v4.0.0+incompatible
 	github.com/fatih/color v1.9.0 // indirect
 	github.com/gcash/bchutil v0.0.0-20191012211144-98e73ec336ba
 	github.com/gcash/bchwallet v0.8.2
@@ -58,7 +57,6 @@ require (
 	github.com/microcosm-cc/bluemonday v1.0.2 // indirect
 	github.com/multiformats/go-multiaddr v0.2.0
 	github.com/multiformats/go-multihash v0.0.13
-	github.com/pkg/errors v0.9.1 // indirect
 	github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1 // indirect
 	github.com/prometheus/client_golang v1.4.1 // indirect
 	github.com/rs/cors v1.7.0
@@ -68,6 +66,7 @@ require (
 	github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94
 	github.com/stripe/stripe-go v60.0.1+incompatible
 	github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c // indirect
+	github.com/ulule/limiter/v3 v3.5.0
 	go.bobheadxi.dev/res v0.2.0
 	go.bobheadxi.dev/zapx/zapx v0.6.8
 	go.bobheadxi.dev/zapx/ztest v0.6.4
diff --git a/go.sum b/go.sum
index c9bd90834..bc374ce95 100644
--- a/go.sum
+++ b/go.sum
@@ -94,8 +94,6 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5
 github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM=
 github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg=
 github.com/astaxie/beego v1.11.1/go.mod h1:i69hVzgauOPSw5qeyF4GVZhn7Od0yG5bbCGzmhbWxgQ=
-github.com/aviddiviner/gin-limit v0.0.0-20170918012823-43b5f79762c1 h1:OLrWlPirfG33eUv6tAZBb2SW2K+xBenfJIWJ+nORMTU=
-github.com/aviddiviner/gin-limit v0.0.0-20170918012823-43b5f79762c1/go.mod h1:v4YSuwMq3CcRnBfKwKzvCATH1jq46sgSHJ8EEUx2ne0=
 github.com/awalterschulze/gographviz v0.0.0-20190522210029-fa59802746ab/go.mod h1:GEV5wmg4YquNw7v1kkyoX9etIk8yVmXj+AkDHuuETHs=
 github.com/beego/goyaml2 v0.0.0-20130207012346-5545475820dd/go.mod h1:1b+Y/CofkYwXMUU0OhQqGvsY2Bvgr4j6jfT699wyZKQ=
 github.com/beego/x2j v0.0.0-20131220205130-a0352aadc542/go.mod h1:kSeGC/p1AbBiEp5kat81+DSQrZenVBZXklMLaELspWU=
@@ -282,6 +280,7 @@ github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+
 github.com/go-playground/validator/v10 v10.2.0 h1:KgJ0snyC2R9VXYN2rneOtQcw5aHQB1Vv0sFl1UcHBOY=
 github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI=
 github.com/go-redis/redis v6.14.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
+github.com/go-redis/redis/v7 v7.2.0/go.mod h1:JDNMw23GTyLNC4GZu9njt15ctBQVn7xjRfnwdHj/Dcg=
 github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
 github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
@@ -681,6 +680,9 @@ github.com/kkdai/bstream v0.0.0-20181106074824-b3251f7901ec h1:n1NeQ3SgUHyISrjFF
 github.com/kkdai/bstream v0.0.0-20181106074824-b3251f7901ec/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
 github.com/kkdai/bstream v1.0.0 h1:Se5gHwgp2VT2uHfDrkbbgbgEvV9cimLELwrPJctSjg8=
 github.com/kkdai/bstream v1.0.0/go.mod h1:FDnDOHt5Yx4p3FaHcioFT0QjDOtgUpvjeZqAs+NVZZA=
+github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.9.6/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -1170,12 +1172,14 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
 github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.11.0 h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw=
 github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
 github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.8.1 h1:C5Dqfs/LeauYDX0jJXIe2SWmwCbGzx9yF8C8xy3Lh34=
 github.com/onsi/gomega v1.8.1/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
 github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
@@ -1308,6 +1312,8 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/stripe/stripe-go v60.0.1+incompatible h1:HDVlurz4+i189rsXGgMDryNIlzrFZV+KrY6pV/RFIBQ=
 github.com/stripe/stripe-go v60.0.1+incompatible/go.mod h1:A1dQZmO/QypXmsL0T8axYZkSN/uA/T/A64pfKdBAMiY=
 github.com/syndtr/goleveldb v0.0.0-20181127023241-353a9fca669c/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0=
@@ -1334,12 +1340,16 @@ github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43 h1:BasDe+IErOQKrMV
 github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43/go.mod h1:iT03XoTwV7xq/+UGwKO3UbC1nNNlopQiY61beSdrtOA=
 github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
 github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
+github.com/ulule/limiter/v3 v3.5.0 h1:QRAebbswjlezHIfiSQgM8+jMxaz/zsrxGRuiUJ43MHo=
+github.com/ulule/limiter/v3 v3.5.0/go.mod h1:TgOUQZKZ2KHjemqrC8UHUbKPqpTmSY43/2wbQ7YN1h8=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli/v2 v2.0.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasthttp v1.9.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w=
 github.com/valyala/fasttemplate v1.0.1 h1:tY9CJiPnMXf1ERmG2EyK7gNUd+c6RKGD0IfU8WdUSz8=
 github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8=
+github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
 github.com/vishvananda/netlink v1.0.0 h1:bqNY2lgheFIu1meHUFSH3d7vG93AFyqg3oGbJCOJgSM=
 github.com/vishvananda/netlink v1.0.0/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
 github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0=
@@ -1530,6 +1540,8 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7 h1:fHDIZ2oxGnUZRN6WgWFCbYBjH9uqVPRCUVUDhs0wnbA=
 golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI=
 golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -1585,6 +1597,7 @@ golang.org/x/sys v0.0.0-20190922100055-0a153f010e69 h1:rOhMmluY6kLMhdnrivzec6lLg
 golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191008105621-543471e840be h1:QAcqgptGM8IQBC9K/RC4o+O9YmqEm0diQn9QmZw/0mU=
 golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

From 6e7644f7e7311a68e104abe0bb6813450c871c20 Mon Sep 17 00:00:00 2001
From: postables <postables@rtradetechnologies.com>
Date: Mon, 13 Apr 2020 14:15:40 -0700
Subject: [PATCH 03/25] prevent blank captcha key from failing tests

---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index 823259247..320e43c32 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,6 +9,7 @@ env:
     - SSL_MODE_DISABLE=true
   global:
     - GO111MODULE=on
+    - RECAPTCHA_KEY=cantbeblankortestsfail
     # decodes into SENGRID_API_KEY environment variable
     - secure: GXKJ+1Wa+pqUm1PKHJMbDrejSfIsqhsZ6l88i0paTvVou9T/mv03mx9hzrSLsYXFFVjinfbKysxMtEorTFDl0YBSNxuPc7eaiJinnoptubBh++bvEzF/A8wmWHgIBxroFIrK4SNsnreCXPqT1YBQZWi3JenMkzS68j1oa/uPa5ODZ3rjx1Wmu6hSANSZPqzOoGO6lnRP8G30oiFzbawEnB/52iBcHhPtffUihLFnn3k1wXyL8fpi4JYuyNQe84Br+w91KSS2nMmG+RVvJWumHNGXFyjEVv0n7HbirjCBI1iZo50bTdFtujkGSOCoHqM1hwa2yLWISipkb19Ls4eeWcWZBFpcdBhIIeHEmz9iCXxn0ksIGzuw00Xof/HTcWUpnzzQFq4E8iVE4tmMrFbYEcD5vcGB3S7bURjTD38uZ3/7I4Oyuo33/WQGdDNIzugU/dycGkSNzwTeIB3yJfpr/wbC6lU5RnADh5Ej5hGreAfkng/iuC9T36Fnn9u4fTocj++smOxZSjbbKsyekNNdadYmU0sfd6Ka9mFcv8H6sDs7zwqyZLTDu6P57CN5VFz4Yjc1emupFX5NFWor5jull9cr+ilOmfnBSCQPV0ArKrpKjrL9K9NJAhRKOqxmpLKx3pozGnrQMFKM0xsSuTcOCdyTdZF8JEQESeENHzwyoB4=
 install:

From 902ce4730a1496eb500843de068a41872070e528 Mon Sep 17 00:00:00 2001
From: postables <postables@rtradetechnologies.com>
Date: Mon, 13 Apr 2020 16:09:33 -0700
Subject: [PATCH 04/25] api/v2: set ForwardedByClientIP otherwise we wont get

proper rate limiting results as this will cause gin to skip processing
---
 api/v2/api.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/api/v2/api.go b/api/v2/api.go
index 2383f9377..6d94d9bef 100644
--- a/api/v2/api.go
+++ b/api/v2/api.go
@@ -84,6 +84,8 @@ func Initialize(
 		err    error
 		router = gin.Default()
 	)
+	// if we dont set this, rate limiting wont work properly
+	router.ForwardedByClientIP = true
 	// update dev mode
 	dev = opts.DevMode
 	l = l.Named("api")

From cb35b5d5ba2f68b5b27739fd452000a1f548df4b Mon Sep 17 00:00:00 2001
From: postables <postables@rtradetechnologies.com>
Date: Mon, 13 Apr 2020 16:14:22 -0700
Subject: [PATCH 05/25] update config

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index d3af32af0..a12f09731 100644
--- a/go.mod
+++ b/go.mod
@@ -5,7 +5,7 @@ go 1.14
 require (
 	github.com/RTradeLtd/ChainRider-Go v1.0.8
 	github.com/RTradeLtd/cmd/v2 v2.1.0
-	github.com/RTradeLtd/config/v2 v2.2.0
+	github.com/RTradeLtd/config/v2 v2.2.1-rc1
 	github.com/RTradeLtd/crypto/v2 v2.1.1
 	github.com/RTradeLtd/database/v2 v2.7.4
 	github.com/RTradeLtd/entropy-mnemonics v0.0.0-20170316012907-7b01a644a636
diff --git a/go.sum b/go.sum
index bc374ce95..98efc785b 100644
--- a/go.sum
+++ b/go.sum
@@ -36,8 +36,8 @@ github.com/RTradeLtd/config/v2 v2.1.1/go.mod h1:juSzxBr84ZeNera4QtOZ7khT9AAtqvyP
 github.com/RTradeLtd/config/v2 v2.1.4/go.mod h1:juSzxBr84ZeNera4QtOZ7khT9AAtqvyPPn/rx2dgzp4=
 github.com/RTradeLtd/config/v2 v2.1.5 h1:5RqXYZJNufmsHk9tL1I2wyQHxkgc/fdXjyamoegTI4A=
 github.com/RTradeLtd/config/v2 v2.1.5/go.mod h1:juSzxBr84ZeNera4QtOZ7khT9AAtqvyPPn/rx2dgzp4=
-github.com/RTradeLtd/config/v2 v2.2.0 h1:7657sVBh+aoXDPGTEKYf8vwBA9dl0Jd8BHm4h8cZ4yk=
-github.com/RTradeLtd/config/v2 v2.2.0/go.mod h1:J2vFG/293yeXFaoX51M7hyvU+5NJqZYQ8Mm+aLiV30E=
+github.com/RTradeLtd/config/v2 v2.2.1-rc1 h1:MXI40D1wD2jj9VTD93Xudk0He/jV0lQZY78pC9hukJI=
+github.com/RTradeLtd/config/v2 v2.2.1-rc1/go.mod h1:J2vFG/293yeXFaoX51M7hyvU+5NJqZYQ8Mm+aLiV30E=
 github.com/RTradeLtd/crypto v2.0.0+incompatible h1:3+UEo0upD0p3A+7yLJ14UJpT6aZEhpzQqF5dt9iRivM=
 github.com/RTradeLtd/crypto v2.0.0+incompatible/go.mod h1:xhKwg748pxs2as6Ts65TiBBFrYzntioTqBIZEa1BUio=
 github.com/RTradeLtd/crypto/v2 v2.1.1 h1:P59zYkkNkl6K1KiTRvW52AYwLvwmtzuzZ9+AjLWmKsU=

From e8df5cdf79a9f5fff850afdc0726ea40b26497d9 Mon Sep 17 00:00:00 2001
From: postables <postables@rtradetechnologies.com>
Date: Mon, 13 Apr 2020 16:16:16 -0700
Subject: [PATCH 06/25] remove embedded struct as this is referenced by the cfg

---
 api/v2/api.go   | 1 -
 api/v2/utils.go | 4 ++--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/api/v2/api.go b/api/v2/api.go
index 6d94d9bef..2f9ab9896 100644
--- a/api/v2/api.go
+++ b/api/v2/api.go
@@ -61,7 +61,6 @@ type API struct {
 	dc          *dash.Client
 	queues      queues
 	service     string
-	cmcAPIKey   string
 	version     string
 
 	captcha recaptcha.ReCAPTCHA
diff --git a/api/v2/utils.go b/api/v2/utils.go
index 5e0f9be91..09f7d6457 100644
--- a/api/v2/utils.go
+++ b/api/v2/utils.go
@@ -275,12 +275,12 @@ func (api *API) getCMCKey() string {
 	if os.Getenv("CMC_API") != "" {
 		return os.Getenv("CMC_API")
 	}
-	return api.cmcAPIKey
+	return api.cfg.APIKeys.CoinMarketCap
 }
 
 func (api *API) getCaptchaKey() string {
 	if os.Getenv("RECAPTCHA_KEY") != "" {
 		return os.Getenv("RECAPTCHA_KEY")
 	}
-	return ""
+	return api.cfg.APIKeys.ReCAPTCHA
 }

From d65270224ef13328c79645c60f3a022500600e2f Mon Sep 17 00:00:00 2001
From: postables <postables@rtradetechnologies.com>
Date: Mon, 13 Apr 2020 19:20:38 -0700
Subject: [PATCH 07/25] conditionally enable captcha

---
 api/v2/api.go | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/api/v2/api.go b/api/v2/api.go
index 2f9ab9896..1cb41084d 100644
--- a/api/v2/api.go
+++ b/api/v2/api.go
@@ -63,7 +63,8 @@ type API struct {
 	service     string
 	version     string
 
-	captcha recaptcha.ReCAPTCHA
+	captcha        recaptcha.ReCAPTCHA
+	captchaEnabled bool
 }
 
 // Initialize is used ot initialize our API service. debug = true is useful
@@ -110,11 +111,14 @@ func Initialize(
 		return nil, err
 	}
 	api.version = version
-	captcha, err := recaptcha.NewReCAPTCHA(api.getCaptchaKey(), recaptcha.V3, time.Second*10)
-	if err != nil {
-		return nil, err
+	if api.getCaptchaKey() != "" {
+		captcha, err := recaptcha.NewReCAPTCHA(api.getCaptchaKey(), recaptcha.V3, time.Second*10)
+		if err != nil {
+			return nil, err
+		}
+		api.captcha = captcha
+		api.captchaEnabled = true
 	}
-	api.captcha = captcha
 	// init routes
 	if err = api.setupRoutes(opts.DebugLogging); err != nil {
 		return nil, err
@@ -661,10 +665,12 @@ func (api *API) setupRoutes(debug bool) error {
 		ens.POST("/claim", api.ClaimENSName)
 		ens.POST("/update", api.UpdateContentHash)
 	}
+	if api.captchaEnabled {
+		recap := v2.Group("/captcha")
+		{
+			recap.POST("/verify", api.verifyCaptcha)
+		}
 
-	recap := v2.Group("/captcha")
-	{
-		recap.POST("/verify", api.verifyCaptcha)
 	}
 
 	api.l.Info("Routes initialized")

From 71bdbcdc9e4e7cb12eb55312bd8eeab4f21277e7 Mon Sep 17 00:00:00 2001
From: postables <postables@rtradetechnologies.com>
Date: Tue, 14 Apr 2020 15:11:47 -0700
Subject: [PATCH 08/25] api/middleware: require verified emails

---
 api/middleware/jwt.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/api/middleware/jwt.go b/api/middleware/jwt.go
index 42ac84131..805056ede 100644
--- a/api/middleware/jwt.go
+++ b/api/middleware/jwt.go
@@ -48,6 +48,10 @@ func JwtConfigGenerate(jwtKey, realmName string, db *gorm.DB, l *zap.SugaredLogg
 					return "", false
 				}
 			}
+			if !usr.EmailEnabled {
+				lAuth.Warn("unverified user tried to sign in", "username", usr.UserName)
+				return "", false
+			}
 			lAuth.Info("successful login", "username", usr.UserName)
 			return usr.UserName, true
 		},

From 925dd68f97c4dd02a71f793933ce5ffebf9ce992 Mon Sep 17 00:00:00 2001
From: postables <postables@rtradetechnologies.com>
Date: Tue, 14 Apr 2020 15:24:59 -0700
Subject: [PATCH 09/25] disable verified email requirement

---
 api/middleware/jwt.go | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/api/middleware/jwt.go b/api/middleware/jwt.go
index 805056ede..42ac84131 100644
--- a/api/middleware/jwt.go
+++ b/api/middleware/jwt.go
@@ -48,10 +48,6 @@ func JwtConfigGenerate(jwtKey, realmName string, db *gorm.DB, l *zap.SugaredLogg
 					return "", false
 				}
 			}
-			if !usr.EmailEnabled {
-				lAuth.Warn("unverified user tried to sign in", "username", usr.UserName)
-				return "", false
-			}
 			lAuth.Info("successful login", "username", usr.UserName)
 			return usr.UserName, true
 		},

From 00cbcdae53f5ac04501390edcde5b8ecfd582b49 Mon Sep 17 00:00:00 2001
From: postables <postables@rtradetechnologies.com>
Date: Tue, 14 Apr 2020 15:31:01 -0700
Subject: [PATCH 10/25] api/v2: update usage tier for unverified users during
 email validation

---
 api/v2/utils.go | 11 +++++++++++
 go.mod          |  2 +-
 go.sum          |  4 ++--
 3 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/api/v2/utils.go b/api/v2/utils.go
index 8ba70cbd0..ead5679d4 100644
--- a/api/v2/utils.go
+++ b/api/v2/utils.go
@@ -172,6 +172,17 @@ func (api *API) verifyEmailJWTToken(jwtString, username string) error {
 	if _, err := api.um.ValidateEmailVerificationToken(username, emailVerificationString); err != nil {
 		return err
 	}
+	// upgrade to free tier if unverified
+	usg, err := api.usage.FindByUserName(username)
+	if err != nil {
+		return err
+	}
+	// only update tier if they are an unverified user
+	// this is to provide backwards compatability where some unverified users
+	// may already be in a different tier
+	if usg.Tier == models.Unverified {
+		api.usage.UpdateTier(username, models.Free)
+	}
 	return nil
 }
 
diff --git a/go.mod b/go.mod
index 16a3ae1a5..65ff6a0d6 100644
--- a/go.mod
+++ b/go.mod
@@ -7,7 +7,7 @@ require (
 	github.com/RTradeLtd/cmd/v2 v2.1.0
 	github.com/RTradeLtd/config/v2 v2.2.0
 	github.com/RTradeLtd/crypto/v2 v2.1.1
-	github.com/RTradeLtd/database/v2 v2.7.4
+	github.com/RTradeLtd/database/v2 v2.7.5-rc1
 	github.com/RTradeLtd/entropy-mnemonics v0.0.0-20170316012907-7b01a644a636
 	github.com/RTradeLtd/go-ipfs-api v0.0.0-20190522213636-8e3700e602fd
 	github.com/RTradeLtd/gpaginator v0.0.4
diff --git a/go.sum b/go.sum
index 3ac4ec479..80c32ddf4 100644
--- a/go.sum
+++ b/go.sum
@@ -42,8 +42,8 @@ github.com/RTradeLtd/crypto v2.0.0+incompatible h1:3+UEo0upD0p3A+7yLJ14UJpT6aZEh
 github.com/RTradeLtd/crypto v2.0.0+incompatible/go.mod h1:xhKwg748pxs2as6Ts65TiBBFrYzntioTqBIZEa1BUio=
 github.com/RTradeLtd/crypto/v2 v2.1.1 h1:P59zYkkNkl6K1KiTRvW52AYwLvwmtzuzZ9+AjLWmKsU=
 github.com/RTradeLtd/crypto/v2 v2.1.1/go.mod h1:saIQ67Btn4JWsOdzjn9U6Dl+aZlg+YKgg4RsQKXxjf4=
-github.com/RTradeLtd/database/v2 v2.7.4 h1:7kYdfjMpvnccrzxt/l29DUTRXsfiWBzuNmGsY+BmVJA=
-github.com/RTradeLtd/database/v2 v2.7.4/go.mod h1:2Q64z+Gdas9wgMpO72iKYW3tCsHs2SJHPzdROcj6MLE=
+github.com/RTradeLtd/database/v2 v2.7.5-rc1 h1:1z8eFMSEleC8Rr8mRISdq+3+2NI23FBf8eBzrNVQdk8=
+github.com/RTradeLtd/database/v2 v2.7.5-rc1/go.mod h1:2Q64z+Gdas9wgMpO72iKYW3tCsHs2SJHPzdROcj6MLE=
 github.com/RTradeLtd/entropy-mnemonics v0.0.0-20170316012907-7b01a644a636 h1:i/+1LBA+YMfD1m9UnQP52A7S6y2U3C0xpMBehPkDRug=
 github.com/RTradeLtd/entropy-mnemonics v0.0.0-20170316012907-7b01a644a636/go.mod h1:zpzHNRdCMCG9PM9QO5jVSldXCRMJ7lY42yJ5TEe//7M=
 github.com/RTradeLtd/go-ipfs-api v0.0.0-20190308091756-8b7099fd5e21/go.mod h1:ipDfy60LjYDddlX/zluSwRVtfGR0EB1HqADazGNMUmE=

From b1334217e24a3b85afef9b14175617ec5bc75d4d Mon Sep 17 00:00:00 2001
From: postables <postables@rtradetechnologies.com>
Date: Tue, 14 Apr 2020 16:08:42 -0700
Subject: [PATCH 11/25] api/v2: update wording about email verification

---
 api/v2/routes_utils.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/api/v2/routes_utils.go b/api/v2/routes_utils.go
index 255e40196..f07dd5101 100644
--- a/api/v2/routes_utils.go
+++ b/api/v2/routes_utils.go
@@ -343,7 +343,7 @@ func (api *API) handleUserCreate(c *gin.Context, forms map[string]string, create
 			"<br>",
 			"<br>",
 			"Lastly let's talk about emails! We try our best to not spam your inbox, so we limit emails to a few things: payment notifications, pin expiration warnings, password/username retrieval and processing failures.\n",
-			"But before we do this, you must validate your email. Note that email validation is not required unless you want these notifications\n",
+			"But before we do this, you must validate your email. Additionally before validating your email, you are in the 'unverified' tier which is limited to 100MB of data consumption. Email verification is now mandatory\n",
 			"To validate your email, just click the following "+link+"\n",
 			"<br>",
 			"<br>",

From 49f3b65541bde23cb3c275e216dc00143d4851f1 Mon Sep 17 00:00:00 2001
From: postables <postables@rtradetechnologies.com>
Date: Tue, 14 Apr 2020 16:16:55 -0700
Subject: [PATCH 12/25] update all usage tier management calls

---
 api/v2/routes_account.go |  3 +++
 api/v2/routes_ens.go     |  4 ++--
 api/v2/routes_rtfs.go    |  2 +-
 api/v2/utils.go          | 16 +++++++++++-----
 api/v2/utils_test.go     |  5 +++++
 cmd/temporal/main.go     |  5 +++++
 utils/utils.go           |  4 ++--
 7 files changed, 29 insertions(+), 10 deletions(-)

diff --git a/api/v2/routes_account.go b/api/v2/routes_account.go
index 3ca0680db..f2a82a72d 100644
--- a/api/v2/routes_account.go
+++ b/api/v2/routes_account.go
@@ -328,6 +328,9 @@ func (api *API) upgradeAccount(c *gin.Context) {
 		api.LogError(c, err, eh.UserSearchError)(http.StatusBadRequest)
 		return
 	}
+	if usages.Tier == models.Unverified {
+		Fail(c, errors.New("unverified account upgrade process must be done via email verification"))
+	}
 	// prevent people from repeatedly calling this granting perpetual credits
 	if usages.Tier != models.Free {
 		Fail(c, errors.New("user account is already upgrade"))
diff --git a/api/v2/routes_ens.go b/api/v2/routes_ens.go
index d40b0b053..ba133a0d8 100644
--- a/api/v2/routes_ens.go
+++ b/api/v2/routes_ens.go
@@ -24,7 +24,7 @@ func (api *API) ClaimENSName(c *gin.Context) {
 		return
 	}
 	// prevent processing if account is free tier
-	if usage.Tier == models.Free {
+	if usage.Tier == models.Free || usage.Tier == models.Unverified {
 		Fail(c, errors.New("free accounts not eligible for ens claim"), http.StatusBadRequest)
 		return
 	}
@@ -74,7 +74,7 @@ func (api *API) UpdateContentHash(c *gin.Context) {
 		return
 	}
 	// prevent processing if account is free tier
-	if usage.Tier == models.Free {
+	if usage.Tier == models.Free || usage.Tier == models.Unverified {
 		Fail(c, errors.New("free accounts not eligible for ens claim"), http.StatusBadRequest)
 		return
 	}
diff --git a/api/v2/routes_rtfs.go b/api/v2/routes_rtfs.go
index 70b947137..c41f5ef40 100644
--- a/api/v2/routes_rtfs.go
+++ b/api/v2/routes_rtfs.go
@@ -197,7 +197,7 @@ func (api *API) addFile(c *gin.Context) {
 		// if the user is within the free tier, then we throttle on-demand encryption
 		// free accounts are limited to a file upload size of 275MB when performing
 		// on-demand encryption. Non free accounts do not have this limit
-		if userUsage.Tier == models.Free {
+		if userUsage.Tier == models.Free || userUsage.Tier == models.Unverified {
 			megabytesUint := datasize.MB.Bytes()
 			maxSize := megabytesUint * 275
 			if fileHandler.Size > int64(maxSize) {
diff --git a/api/v2/utils.go b/api/v2/utils.go
index ead5679d4..4c3b98d15 100644
--- a/api/v2/utils.go
+++ b/api/v2/utils.go
@@ -258,10 +258,16 @@ func (api *API) validateHoldTime(username, holdTime string) (int64, error) {
 	if err != nil {
 		return 0, err
 	}
-	if usageTier.Tier == models.Free && holdTimeInt > freeHoldTimeLimitInMonths {
-		return 0, errors.New("free accounts are limited to maximum hold times of 12 month")
-	} else if usageTier.Tier != models.Free && holdTimeInt > nonFreeHoldTimeLimitInMonths {
-		return 0, errors.New("non free accounts are limited to a maximum hold time of 24 months")
+	switch usageTier.Tier {
+	case models.Free, models.Unverified:
+		if holdTimeInt > freeHoldTimeLimitInMonths {
+			return 0, errors.New("free accounts are limited to maximum hold times of 12 month")
+
+		}
+	default:
+		if holdTimeInt > nonFreeHoldTimeLimitInMonths {
+			return 0, errors.New("non free accounts are limited to a maximum hold time of 24 months")
+		}
 	}
 	return holdTimeInt, nil
 }
@@ -269,7 +275,7 @@ func (api *API) validateHoldTime(username, holdTime string) (int64, error) {
 func (api *API) ensureLEMaxPinTime(upload *models.Upload, holdTime int64, tier models.DataUsageTier) error {
 	var limit time.Time
 	switch tier {
-	case models.Free:
+	case models.Free, models.Unverified:
 		limit = time.Now().AddDate(1, 0, 0)
 	case models.Paid, models.Partner, models.WhiteLabeled:
 		limit = time.Now().AddDate(2, 0, 0)
diff --git a/api/v2/utils_test.go b/api/v2/utils_test.go
index 9145dd538..a0919cc09 100644
--- a/api/v2/utils_test.go
+++ b/api/v2/utils_test.go
@@ -148,6 +148,11 @@ func Test_Ensure_Two_Year_Max(t *testing.T) {
 		{"12-Months-paid", args{12, models.Paid}, false},
 		{"22-Months-paid", args{22, models.Paid}, false},
 		{"25-Months-paid", args{25, models.Paid}, true},
+		{"10-Months-unverified", args{10, models.Unverified}, false},
+		{"11-Months-unverified", args{11, models.Unverified}, false},
+		{"12-Months-unverified", args{12, models.Unverified}, true},
+		{"22-Months-unverified", args{22, models.Unverified}, true},
+		{"25-Months-unverified", args{25, models.Unverified}, true},
 		{"10-Months-free", args{10, models.Free}, false},
 		{"11-Months-free", args{11, models.Free}, false},
 		{"12-Months-free", args{12, models.Free}, true},
diff --git a/cmd/temporal/main.go b/cmd/temporal/main.go
index 17b90feb5..07fd205e6 100644
--- a/cmd/temporal/main.go
+++ b/cmd/temporal/main.go
@@ -517,6 +517,11 @@ var commands = map[string]cmd.Cmd{
 				fmt.Println("failed to create user account", err)
 				os.Exit(1)
 			}
+			// update tier
+			if err := models.NewUsageManager(d.DB).UpdateTier(args["user"], models.Free); err != nil {
+				fmt.Println("failed to update user account tier", err)
+				os.Exit(1)
+			}
 			// add credits
 			if _, err := models.NewUserManager(d.DB).AddCredits(args["user"], 99999999); err != nil {
 				fmt.Println("failed to grant credits to user account", err)
diff --git a/utils/utils.go b/utils/utils.go
index b5fafdaca..73157e2df 100644
--- a/utils/utils.go
+++ b/utils/utils.go
@@ -30,7 +30,7 @@ func CalculatePinCost(username, contentHash string, holdTimeInMonths int64, im r
 		return 0, err
 	}
 	// if they are free tier, they don't incur data charges
-	if usage.Tier == models.Free || usage.Tier == models.WhiteLabeled {
+	if usage.Tier == models.Free || usage.Tier == models.WhiteLabeled || usage.Tier == models.Unverified {
 		return 0, nil
 	}
 	// dynamic pricing based on their usage tier
@@ -49,7 +49,7 @@ func CalculateFileCost(username string, holdTimeInMonths, size int64, um *models
 		return 0, err
 	}
 	// if they are free tier, they don't incur data charges
-	if usage.Tier == models.Free || usage.Tier == models.WhiteLabeled {
+	if usage.Tier == models.Free || usage.Tier == models.WhiteLabeled || usage.Tier == models.Unverified {
 		return 0, nil
 	}
 	// dynamic pricing based on their usage tier

From 1843666892e93466ba9b1cd049a4a490b972ba98 Mon Sep 17 00:00:00 2001
From: postables <postables@rtradetechnologies.com>
Date: Tue, 14 Apr 2020 16:58:48 -0700
Subject: [PATCH 13/25] prevent dev rate limit throttling

---
 api/v2/api.go | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/api/v2/api.go b/api/v2/api.go
index 1cb41084d..8aa6e8151 100644
--- a/api/v2/api.go
+++ b/api/v2/api.go
@@ -388,8 +388,15 @@ func (api *API) setupRoutes(debug bool) error {
 			return err
 		}
 	}
+	// make sure we dont throttle dev too much
+	var rateLimit string
+	if dev {
+		rateLimit = "100000-H"
+	} else {
+		fmt.Sprintf("%v-H", connLimit)
+	}
 	// 1000 requests per hour
-	rate, err := limiter.NewRateFromFormatted(fmt.Sprintf("%v-H", connLimit))
+	rate, err := limiter.NewRateFromFormatted(rateLimit)
 	if err != nil {
 		return err
 	}

From 43b94ee5d31150deac387562f3d50613e631c043 Mon Sep 17 00:00:00 2001
From: postables <postables@rtradetechnologies.com>
Date: Tue, 14 Apr 2020 18:45:15 -0700
Subject: [PATCH 14/25] make sure we use sprintf call

---
 api/v2/api.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/api/v2/api.go b/api/v2/api.go
index 8aa6e8151..8111f7943 100644
--- a/api/v2/api.go
+++ b/api/v2/api.go
@@ -393,7 +393,7 @@ func (api *API) setupRoutes(debug bool) error {
 	if dev {
 		rateLimit = "100000-H"
 	} else {
-		fmt.Sprintf("%v-H", connLimit)
+		rateLimit = fmt.Sprintf("%v-H", connLimit)
 	}
 	// 1000 requests per hour
 	rate, err := limiter.NewRateFromFormatted(rateLimit)

From 3287b625c947ddc4a5cd28454e2057f25726cb6f Mon Sep 17 00:00:00 2001
From: postables <postables@rtradetechnologies.com>
Date: Tue, 14 Apr 2020 19:17:36 -0700
Subject: [PATCH 15/25] api/v2: remove invalid comment

---
 api/v2/api.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/api/v2/api.go b/api/v2/api.go
index 8111f7943..051af4d38 100644
--- a/api/v2/api.go
+++ b/api/v2/api.go
@@ -395,7 +395,6 @@ func (api *API) setupRoutes(debug bool) error {
 	} else {
 		rateLimit = fmt.Sprintf("%v-H", connLimit)
 	}
-	// 1000 requests per hour
 	rate, err := limiter.NewRateFromFormatted(rateLimit)
 	if err != nil {
 		return err

From d48cf3fdd445663f0e455a2be959f807a296ae81 Mon Sep 17 00:00:00 2001
From: postables <postables@rtradetechnologies.com>
Date: Tue, 14 Apr 2020 21:28:34 -0700
Subject: [PATCH 16/25] use a 20 second captcha timeout

---
 api/v2/api.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/api/v2/api.go b/api/v2/api.go
index 051af4d38..fd95fa6c7 100644
--- a/api/v2/api.go
+++ b/api/v2/api.go
@@ -112,7 +112,7 @@ func Initialize(
 	}
 	api.version = version
 	if api.getCaptchaKey() != "" {
-		captcha, err := recaptcha.NewReCAPTCHA(api.getCaptchaKey(), recaptcha.V3, time.Second*10)
+		captcha, err := recaptcha.NewReCAPTCHA(api.getCaptchaKey(), recaptcha.V3, time.Second*20)
 		if err != nil {
 			return nil, err
 		}

From a868b19fd9981c650e8577bc8541d159b48f86b2 Mon Sep 17 00:00:00 2001
From: postables <postables@rtradetechnologies.com>
Date: Fri, 17 Apr 2020 01:23:10 -0700
Subject: [PATCH 17/25] api/middleware: require emails to be enabled

---
 api/middleware/jwt.go | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/api/middleware/jwt.go b/api/middleware/jwt.go
index 42ac84131..27974d7e8 100644
--- a/api/middleware/jwt.go
+++ b/api/middleware/jwt.go
@@ -48,16 +48,21 @@ func JwtConfigGenerate(jwtKey, realmName string, db *gorm.DB, l *zap.SugaredLogg
 					return "", false
 				}
 			}
+			// email enabled implies they have verified their email
+			if !usr.EmailEnabled {
+				return "", false
+			}
 			lAuth.Info("successful login", "username", usr.UserName)
 			return usr.UserName, true
 		},
 		Authorizator: func(userId string, c *gin.Context) bool {
 			// as a final security step, ensure that we can find the user in our database
 			userManager := models.NewUserManager(db)
-			if _, err := userManager.FindByUserName(userId); err != nil {
+			usr, err := userManager.FindByUserName(userId)
+			if err != nil {
 				return false
 			}
-			return true
+			return usr.EmailEnabled
 		},
 		Unauthorized: func(c *gin.Context, code int, message string) {
 			l.Error("invalid login detected")

From e0147f53a6289edda84433b4ec07b2e0bf672873 Mon Sep 17 00:00:00 2001
From: postables <postables@rtradetechnologies.com>
Date: Fri, 17 Apr 2020 12:49:06 -0700
Subject: [PATCH 18/25] api/v2: adjust login test

---
 api/v2/api_test.go | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/api/v2/api_test.go b/api/v2/api_test.go
index 70ca0a43c..e9a401d46 100644
--- a/api/v2/api_test.go
+++ b/api/v2/api_test.go
@@ -256,7 +256,8 @@ func Test_API_Setup(t *testing.T) {
 		args     args
 		wantCode int
 	}{
-		{"Login-testuser2", args{"POST", "/v2/auth/login", "testuser2", "password123!@#$%^&&**(!@#!", ""}, 200},
+		// this test should fail due to invalid email
+		{"Login-testuser2", args{"POST", "/v2/auth/login", "testuser2", "password123!@#$%^&&**(!@#!", ""}, 401},
 		{"Login-testuser", args{"POST", "/v2/auth/login", "testuser", "admin", ""}, 200},
 		// tests login via the email instead of using
 		{"Login-TestUser-Email", args{"POST", "/v2/auth/login", "test@email.com", "admin", ""}, 200},
@@ -272,7 +273,7 @@ func Test_API_Setup(t *testing.T) {
 			)
 			api.r.ServeHTTP(testRecorder, req)
 			if testRecorder.Code != tt.wantCode {
-				t.Fatalf("bad http status code from %s", tt.args.call)
+				t.Fatalf("bad http status code from %s. got %v, want %v", tt.args.call, testRecorder.Code, tt.wantCode)
 			}
 			bodBytes, err := ioutil.ReadAll(testRecorder.Result().Body)
 			if err != nil {

From c20bf724b7578cfe19b26f042ff4774ec86fd706 Mon Sep 17 00:00:00 2001
From: postables <postables@rtradetechnologies.com>
Date: Fri, 17 Apr 2020 14:13:06 -0700
Subject: [PATCH 19/25] update database dependency

---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 8b713e26b..590e0b4d1 100644
--- a/go.mod
+++ b/go.mod
@@ -7,7 +7,7 @@ require (
 	github.com/RTradeLtd/cmd/v2 v2.1.0
 	github.com/RTradeLtd/config/v2 v2.2.1-rc1
 	github.com/RTradeLtd/crypto/v2 v2.1.1
-	github.com/RTradeLtd/database/v2 v2.7.5-rc1
+	github.com/RTradeLtd/database/v2 v2.7.6-rc1
 	github.com/RTradeLtd/entropy-mnemonics v0.0.0-20170316012907-7b01a644a636
 	github.com/RTradeLtd/go-ipfs-api v0.0.0-20190522213636-8e3700e602fd
 	github.com/RTradeLtd/gpaginator v0.0.4
diff --git a/go.sum b/go.sum
index ff2c496d2..a39285f35 100644
--- a/go.sum
+++ b/go.sum
@@ -42,8 +42,8 @@ github.com/RTradeLtd/crypto v2.0.0+incompatible h1:3+UEo0upD0p3A+7yLJ14UJpT6aZEh
 github.com/RTradeLtd/crypto v2.0.0+incompatible/go.mod h1:xhKwg748pxs2as6Ts65TiBBFrYzntioTqBIZEa1BUio=
 github.com/RTradeLtd/crypto/v2 v2.1.1 h1:P59zYkkNkl6K1KiTRvW52AYwLvwmtzuzZ9+AjLWmKsU=
 github.com/RTradeLtd/crypto/v2 v2.1.1/go.mod h1:saIQ67Btn4JWsOdzjn9U6Dl+aZlg+YKgg4RsQKXxjf4=
-github.com/RTradeLtd/database/v2 v2.7.5-rc1 h1:1z8eFMSEleC8Rr8mRISdq+3+2NI23FBf8eBzrNVQdk8=
-github.com/RTradeLtd/database/v2 v2.7.5-rc1/go.mod h1:2Q64z+Gdas9wgMpO72iKYW3tCsHs2SJHPzdROcj6MLE=
+github.com/RTradeLtd/database/v2 v2.7.6-rc1 h1:/H0tGPJxnQKb6aVeIyP2qQArA30nr66tEwWY5mMnaA8=
+github.com/RTradeLtd/database/v2 v2.7.6-rc1/go.mod h1:2Q64z+Gdas9wgMpO72iKYW3tCsHs2SJHPzdROcj6MLE=
 github.com/RTradeLtd/entropy-mnemonics v0.0.0-20170316012907-7b01a644a636 h1:i/+1LBA+YMfD1m9UnQP52A7S6y2U3C0xpMBehPkDRug=
 github.com/RTradeLtd/entropy-mnemonics v0.0.0-20170316012907-7b01a644a636/go.mod h1:zpzHNRdCMCG9PM9QO5jVSldXCRMJ7lY42yJ5TEe//7M=
 github.com/RTradeLtd/go-ipfs-api v0.0.0-20190308091756-8b7099fd5e21/go.mod h1:ipDfy60LjYDddlX/zluSwRVtfGR0EB1HqADazGNMUmE=

From e6955acb891a230c69c93630d81ea67a5f795d34 Mon Sep 17 00:00:00 2001
From: postables <postables@rtradetechnologies.com>
Date: Fri, 17 Apr 2020 17:12:47 -0700
Subject: [PATCH 20/25] utils: enable deduplicate size calculation

---
 utils/utils.go | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/utils/utils.go b/utils/utils.go
index 73157e2df..1378aa325 100644
--- a/utils/utils.go
+++ b/utils/utils.go
@@ -1,6 +1,7 @@
 package utils
 
 import (
+	"errors"
 	"math/big"
 
 	"github.com/RTradeLtd/database/v2/models"
@@ -10,12 +11,15 @@ import (
 
 // CalculatePinCost is used to calculate the cost of pining a particular content hash
 func CalculatePinCost(username, contentHash string, holdTimeInMonths int64, im rtfs.Manager, um *models.UsageManager) (float64, error) {
-	objectStat, err := im.Stat(contentHash)
+	// get total size of content hash in bytes ensuring that we calculate size
+	// by following unique references
+	sizeInBytes, _, err := rtfs.DedupAndCalculatePinSize(contentHash, im)
 	if err != nil {
-		return float64(0), err
+		return 0, err
+	}
+	if sizeInBytes <= 0 {
+		return 0, errors.New("failed to calculate object size")
 	}
-	// get total size of content hash in bytes
-	sizeInBytes := objectStat.CumulativeSize
 	// get gigabytes convert to bytes
 	gigaInBytes := datasize.GB.Bytes()
 	// convert size of content hash form int to float64

From b54d5d5d8819efd2fec92abd5746d05dbfef29d1 Mon Sep 17 00:00:00 2001
From: postables <postables@rtradetechnologies.com>
Date: Fri, 17 Apr 2020 17:22:32 -0700
Subject: [PATCH 21/25] api/v2: fix failing test

---
 api/v2/routes_ipns_test.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/api/v2/routes_ipns_test.go b/api/v2/routes_ipns_test.go
index f011c4abe..89009fa33 100644
--- a/api/v2/routes_ipns_test.go
+++ b/api/v2/routes_ipns_test.go
@@ -188,6 +188,12 @@ func Test_API_Routes_IPNS_Pin(t *testing.T) {
 			fakeManager.ResolveReturnsOnCall(0, validResolveResult, nil)
 			fakeManager.StatReturnsOnCall(0, &shell.ObjectStats{CumulativeSize: 5000000}, nil)
 			fakeManager.StatReturnsOnCall(1, &shell.ObjectStats{CumulativeSize: 5000000}, nil)
+			fakeManager.StatReturnsOnCall(2, &shell.ObjectStats{CumulativeSize: 5000000}, nil)
+			fakeManager.StatReturnsOnCall(3, &shell.ObjectStats{CumulativeSize: 5000000}, nil)
+			fakeManager.RefsReturnsOnCall(0, []string{"QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv", "QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv"}, nil)
+			fakeManager.RefsReturnsOnCall(1, []string{"QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv", "QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv"}, nil)
+			fakeManager.RefsReturnsOnCall(2, []string{"QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv", "QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv"}, nil)
+			fakeManager.RefsReturnsOnCall(3, []string{"QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv", "QmS4ustL54uo8FzR9455qaxZwuMiUhyvMcX9Ba8nUH4uVv"}, nil)
 			var apiResp apiResponse
 			urlValues := url.Values{}
 			urlValues.Add("hold_time", tt.args.holdTime)

From 5b5b4086ba1625f58cacd6bb0d4129a6c0660c91 Mon Sep 17 00:00:00 2001
From: postables <postables@rtradetechnologies.com>
Date: Fri, 17 Apr 2020 17:50:39 -0700
Subject: [PATCH 22/25] utils: fall back to cumulative size evaluation

---
 utils/utils.go | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/utils/utils.go b/utils/utils.go
index 1378aa325..1f2863056 100644
--- a/utils/utils.go
+++ b/utils/utils.go
@@ -1,7 +1,6 @@
 package utils
 
 import (
-	"errors"
 	"math/big"
 
 	"github.com/RTradeLtd/database/v2/models"
@@ -17,8 +16,15 @@ func CalculatePinCost(username, contentHash string, holdTimeInMonths int64, im r
 	if err != nil {
 		return 0, err
 	}
+	// if this is true, fall back to default calculation
+	// as it wont always be possible to calculate deduplicated
+	// storage costs if the object is not of a unixfs type
 	if sizeInBytes <= 0 {
-		return 0, errors.New("failed to calculate object size")
+		stats, err := im.Stat(contentHash)
+		if err != nil {
+			return 0, err
+		}
+		sizeInBytes = int64(stats.CumulativeSize)
 	}
 	// get gigabytes convert to bytes
 	gigaInBytes := datasize.GB.Bytes()

From 25376aba1ebf991504ccad362bd273e29bf0b563 Mon Sep 17 00:00:00 2001
From: postables <postables@rtradetechnologies.com>
Date: Fri, 17 Apr 2020 19:00:47 -0700
Subject: [PATCH 23/25] api/middleware: update authorizator to check both email
 & account stat

---
 api/middleware/jwt.go | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/api/middleware/jwt.go b/api/middleware/jwt.go
index 27974d7e8..4ab71eb48 100644
--- a/api/middleware/jwt.go
+++ b/api/middleware/jwt.go
@@ -62,7 +62,10 @@ func JwtConfigGenerate(jwtKey, realmName string, db *gorm.DB, l *zap.SugaredLogg
 			if err != nil {
 				return false
 			}
-			return usr.EmailEnabled
+			if usr.EmailEnabled && usr.AccountEnabled {
+				return true
+			}
+			return false
 		},
 		Unauthorized: func(c *gin.Context, code int, message string) {
 			l.Error("invalid login detected")

From 9145b52c106ceec0c0b5a6a95fb362eca7e7b77e Mon Sep 17 00:00:00 2001
From: postables <postables@rtradetechnologies.com>
Date: Sat, 18 Apr 2020 15:47:20 -0700
Subject: [PATCH 24/25] make sure to calculate deduplicated storage update

---
 api/v2/routes_frontend.go |  2 +-
 api/v2/routes_ipns.go     | 14 ++++----------
 api/v2/routes_rtfs.go     | 16 +++++-----------
 utils/utils.go            | 13 +++++++------
 utils/utils_test.go       |  2 +-
 5 files changed, 18 insertions(+), 29 deletions(-)

diff --git a/api/v2/routes_frontend.go b/api/v2/routes_frontend.go
index 30920bcc7..088ad6888 100644
--- a/api/v2/routes_frontend.go
+++ b/api/v2/routes_frontend.go
@@ -35,7 +35,7 @@ func (api *API) calculatePinCost(c *gin.Context) {
 		return
 	}
 	// calculate pin cost
-	totalCost, err := utils.CalculatePinCost(username, hash, holdTimeInt, api.ipfs, api.usage)
+	totalCost, _, err := utils.CalculatePinCost(username, hash, holdTimeInt, api.ipfs, api.usage)
 	if err != nil {
 		api.LogError(c, err, eh.CostCalculationError)(http.StatusBadRequest)
 		Fail(c, err)
diff --git a/api/v2/routes_ipns.go b/api/v2/routes_ipns.go
index 478521820..00b23081c 100644
--- a/api/v2/routes_ipns.go
+++ b/api/v2/routes_ipns.go
@@ -167,14 +167,8 @@ func (api *API) pinIPNSHash(c *gin.Context) {
 		Respond(c, http.StatusBadRequest, gin.H{"response": alreadyUploadedMessage})
 		return
 	}
-	// get size of object
-	stats, err := api.ipfs.Stat(hash)
-	if err != nil {
-		api.LogError(c, err, eh.IPFSObjectStatError)(http.StatusBadRequest)
-		return
-	}
 	// get the cost of this object
-	cost, err := utils.CalculatePinCost(username, hash, holdTimeInt, api.ipfs, api.usage)
+	cost, size, err := utils.CalculatePinCost(username, hash, holdTimeInt, api.ipfs, api.usage)
 	if err != nil {
 		api.LogError(c, err, eh.CostCalculationError)(http.StatusBadRequest)
 		return
@@ -184,7 +178,7 @@ func (api *API) pinIPNSHash(c *gin.Context) {
 		api.LogError(c, err, eh.InvalidBalanceError)(http.StatusPaymentRequired)
 		return
 	}
-	if err := api.usage.UpdateDataUsage(username, uint64(stats.CumulativeSize)); err != nil {
+	if err := api.usage.UpdateDataUsage(username, uint64(size)); err != nil {
 		api.LogError(c, err, eh.CantUploadError)(http.StatusBadRequest)
 		api.refundUserCredits(username, "pin", cost)
 		return
@@ -196,13 +190,13 @@ func (api *API) pinIPNSHash(c *gin.Context) {
 		UserName:         username,
 		HoldTimeInMonths: holdTimeInt,
 		CreditCost:       cost,
-		Size:             int64(stats.CumulativeSize),
+		Size:             int64(size),
 	}
 	// send message for processing
 	if err = api.queues.cluster.PublishMessage(qp); err != nil {
 		api.LogError(c, err, eh.QueuePublishError)(http.StatusBadRequest)
 		api.refundUserCredits(username, "pin", cost)
-		api.usage.ReduceDataUsage(username, uint64(stats.CumulativeSize))
+		api.usage.ReduceDataUsage(username, uint64(size))
 		return
 	}
 	// log and return
diff --git a/api/v2/routes_rtfs.go b/api/v2/routes_rtfs.go
index c41f5ef40..5823ebe5f 100644
--- a/api/v2/routes_rtfs.go
+++ b/api/v2/routes_rtfs.go
@@ -54,14 +54,8 @@ func (api *API) pinHashLocally(c *gin.Context) {
 		Respond(c, http.StatusBadRequest, gin.H{"response": alreadyUploadedMessage})
 		return
 	}
-	// get object size
-	stats, err := api.ipfs.Stat(hash)
-	if err != nil {
-		api.LogError(c, err, eh.IPFSObjectStatError)(http.StatusBadRequest)
-		return
-	}
 	// determine cost of upload
-	cost, err := utils.CalculatePinCost(username, hash, holdTimeInt, api.ipfs, api.usage)
+	cost, size, err := utils.CalculatePinCost(username, hash, holdTimeInt, api.ipfs, api.usage)
 	if err != nil {
 		api.LogError(c, err, eh.CostCalculationError)(http.StatusBadRequest)
 		return
@@ -72,7 +66,7 @@ func (api *API) pinHashLocally(c *gin.Context) {
 		return
 	}
 	// update their data usage
-	if err := api.usage.UpdateDataUsage(username, uint64(stats.CumulativeSize)); err != nil {
+	if err := api.usage.UpdateDataUsage(username, uint64(size)); err != nil {
 		api.LogError(c, err, eh.CantUploadError)(http.StatusBadRequest)
 		api.refundUserCredits(username, "pin", cost)
 		return
@@ -83,7 +77,7 @@ func (api *API) pinHashLocally(c *gin.Context) {
 		NetworkName:      "public",
 		UserName:         username,
 		HoldTimeInMonths: holdTimeInt,
-		Size:             int64(stats.CumulativeSize),
+		Size:             size,
 		CreditCost:       cost,
 		FileName:         c.PostForm("file_name"),
 	}
@@ -91,7 +85,7 @@ func (api *API) pinHashLocally(c *gin.Context) {
 	if err = api.queues.cluster.PublishMessage(qp); err != nil {
 		api.LogError(c, err, eh.QueuePublishError)(http.StatusBadRequest)
 		api.refundUserCredits(username, "pin", cost)
-		api.usage.ReduceDataUsage(username, uint64(stats.CumulativeSize))
+		api.usage.ReduceDataUsage(username, uint64(size))
 		return
 	}
 	// log success and return
@@ -381,7 +375,7 @@ func (api *API) extendPin(c *gin.Context) {
 		return
 	}
 	// calculate cost of hold time extension
-	cost, err := utils.CalculatePinCost(username, hash, holdTimeInt, api.ipfs, api.usage)
+	cost, _, err := utils.CalculatePinCost(username, hash, holdTimeInt, api.ipfs, api.usage)
 	if err != nil {
 		api.LogError(c, err, eh.CostCalculationError)(http.StatusBadRequest)
 		return
diff --git a/utils/utils.go b/utils/utils.go
index 1f2863056..cbaa4d7c3 100644
--- a/utils/utils.go
+++ b/utils/utils.go
@@ -9,12 +9,13 @@ import (
 )
 
 // CalculatePinCost is used to calculate the cost of pining a particular content hash
-func CalculatePinCost(username, contentHash string, holdTimeInMonths int64, im rtfs.Manager, um *models.UsageManager) (float64, error) {
+// it returns the cost to bill the user, as well as the calculated size of the pin
+func CalculatePinCost(username, contentHash string, holdTimeInMonths int64, im rtfs.Manager, um *models.UsageManager) (float64, int64, error) {
 	// get total size of content hash in bytes ensuring that we calculate size
 	// by following unique references
 	sizeInBytes, _, err := rtfs.DedupAndCalculatePinSize(contentHash, im)
 	if err != nil {
-		return 0, err
+		return 0, 0, err
 	}
 	// if this is true, fall back to default calculation
 	// as it wont always be possible to calculate deduplicated
@@ -22,7 +23,7 @@ func CalculatePinCost(username, contentHash string, holdTimeInMonths int64, im r
 	if sizeInBytes <= 0 {
 		stats, err := im.Stat(contentHash)
 		if err != nil {
-			return 0, err
+			return 0, 0, err
 		}
 		sizeInBytes = int64(stats.CumulativeSize)
 	}
@@ -37,15 +38,15 @@ func CalculatePinCost(username, contentHash string, holdTimeInMonths int64, im r
 	// get the users usage model
 	usage, err := um.FindByUserName(username)
 	if err != nil {
-		return 0, err
+		return 0, 0, err
 	}
 	// if they are free tier, they don't incur data charges
 	if usage.Tier == models.Free || usage.Tier == models.WhiteLabeled || usage.Tier == models.Unverified {
-		return 0, nil
+		return 0, sizeInBytes, nil
 	}
 	// dynamic pricing based on their usage tier
 	costPerMonthFloat := objectSizeInGigabytesFloat * usage.Tier.PricePerGB()
-	return costPerMonthFloat * float64(holdTimeInMonths), nil
+	return costPerMonthFloat * float64(holdTimeInMonths), sizeInBytes, nil
 }
 
 // CalculateFileCost is used to calculate the cost of storing a file
diff --git a/utils/utils_test.go b/utils/utils_test.go
index 6407e5d16..d0cabdb8f 100644
--- a/utils/utils_test.go
+++ b/utils/utils_test.go
@@ -47,7 +47,7 @@ func TestUtils_CalculatePinCost(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if _, err := utils.CalculatePinCost(
+			if _, _, err := utils.CalculatePinCost(
 				tt.args.username,
 				tt.args.hash,
 				tt.args.months,

From 3e6ff4f423a37389654aee97520123f10a3b912c Mon Sep 17 00:00:00 2001
From: postables <postables@rtradetechnologies.com>
Date: Sun, 19 Apr 2020 15:53:43 -0700
Subject: [PATCH 25/25] add code review comment

---
 api/middleware/jwt.go | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/api/middleware/jwt.go b/api/middleware/jwt.go
index 4ab71eb48..7867d744a 100644
--- a/api/middleware/jwt.go
+++ b/api/middleware/jwt.go
@@ -62,10 +62,7 @@ func JwtConfigGenerate(jwtKey, realmName string, db *gorm.DB, l *zap.SugaredLogg
 			if err != nil {
 				return false
 			}
-			if usr.EmailEnabled && usr.AccountEnabled {
-				return true
-			}
-			return false
+			return usr.EmailEnabled && usr.AccountEnabled
 		},
 		Unauthorized: func(c *gin.Context, code int, message string) {
 			l.Error("invalid login detected")