Large refactore (#8)

* add port number to script

* fix parameter count

* set main.Version and main.BuildTime

* optimze webpage

* handle / at end of path

* refactore upload

* large refactore

* refactore storage

* refactore package storage

* refactor: Add script to trigger release pipeline with next version

.github/workflows/release.yml

@@ -22,9 +22,11 @@ jobs:
       - name: Get dependencies
         run: go mod tidy
 
-      - name: Build binary
+      - name: Build binary and set main.Version and main.BuildTime
         run: |
-          GOOS=linux GOARCH=amd64 go build -ldflags="-w -s" -o iot-ephemeral-value-store-server main.go
+          TAG_NAME=${GITHUB_REF#refs/tags/}
+          BUILD_TIME=$(date +%Y-%m-%dT%H:%M:%SZ)
+          GOOS=linux GOARCH=amd64 go build -ldflags="-w -s -X main.Version=${TAG_NAME} -X main.BuildTime=${BUILD_TIME}" -o iot-ephemeral-value-store-server main.go
           chmod +x iot-ephemeral-value-store-server
 
       - name: Package files

.vscode/launch.json

@@ -0,0 +1,19 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Launch Package",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "program": "${workspaceFolder}",
+            "args": [
+                "-port",
+                "8088",
+            ]
+        }
+    ]
+}

httphandler/config.go

@@ -5,12 +5,11 @@ import (
 	"html"
 	"net/http"
 
-	"github.com/dgraph-io/badger/v3"
+	"github.com/dhcgn/iot-ephemeral-value-store/storage"
 )
 
 type Config struct {
-	Db              *badger.DB
-	PersistDuration string
+	StorageInstance storage.Storage
 }
 
 func sanitizeInput(input string) string {

httphandler/downloadHandler.go

@@ -6,7 +6,6 @@ import (
 	"net/http"
 	"strings"
 
-	"github.com/dgraph-io/badger/v3"
 	"github.com/gorilla/mux"
 )
 
@@ -15,17 +14,9 @@ func (c Config) PlainDownloadHandler(w http.ResponseWriter, r *http.Request) {
 	downloadKey := vars["downloadKey"]
 	param := vars["param"]
 
-	var jsonData []byte
-	err := c.Db.View(func(txn *badger.Txn) error {
-		item, err := txn.Get([]byte(downloadKey))
-		if err != nil {
-			return err
-		}
-		jsonData, err = item.ValueCopy(nil)
-		return err
-	})
+	jsonData, err := c.StorageInstance.GetJSON(downloadKey)
 	if err != nil {
-		http.Error(w, "Data not found", http.StatusNotFound)
+		http.Error(w, "Invalid download key or database error", http.StatusNotFound)
 		return
 	}
 
@@ -63,15 +54,7 @@ func (c Config) DownloadHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	downloadKey := vars["downloadKey"]
 
-	var jsonData []byte
-	err := c.Db.View(func(txn *badger.Txn) error {
-		item, err := txn.Get([]byte(downloadKey))
-		if err != nil {
-			return err
-		}
-		jsonData, err = item.ValueCopy(nil)
-		return err
-	})
+	jsonData, err := c.StorageInstance.GetJSON(downloadKey)
 	if err != nil {
 		http.Error(w, "Invalid download key or database error", http.StatusNotFound)
 		return

httphandler/uploadHandler.go

@@ -2,14 +2,12 @@ package httphandler
 
 import (
 	"encoding/hex"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"net/http"
 	"strings"
 	"time"
 
-	"github.com/dgraph-io/badger/v3"
 	"github.com/dhcgn/iot-ephemeral-value-store/domain"
 	"github.com/gorilla/mux"
 )
@@ -19,179 +17,97 @@ func (c Config) UploadAndPatchHandler(w http.ResponseWriter, r *http.Request) {
 	uploadKey := vars["uploadKey"]
 	path := vars["param"]
 
-	err := ValidateUploadKey(uploadKey)
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusBadRequest)
-		return
-	}
-
-	// Derive the download key from the upload key
-	downloadKey, err := domain.DeriveDownloadKey(uploadKey)
-	if err != nil {
-		http.Error(w, "Error deriving download key", http.StatusInternalServerError)
-		return
-	}
-
-	// Parse the duration for setting TTL on data
-	duration, err := time.ParseDuration(c.PersistDuration)
-	if err != nil {
-		http.Error(w, fmt.Sprintf("Invalid duration format: %v", err), http.StatusBadRequest)
-		return
-	}
-
-	// Collect all parameters into a map
-	params := r.URL.Query()
-	paramMap := make(map[string]string)
-	for key, values := range params {
-		if len(values) > 0 {
-			// Sanitize each parameter value
-			sanitizedValue := sanitizeInput(values[0])
-			paramMap[key] = sanitizedValue
-		}
-	}
-
-	// Get the current timestamp
-	timestamp := time.Now().UTC().Format(time.RFC3339)
-
-	// Add the timestamp to the map
-	paramMap["timestamp"] = timestamp
-
-	// Retrieve existing JSON data from the database
-	var existingData map[string]interface{}
-	err = c.Db.View(func(txn *badger.Txn) error {
-		item, err := txn.Get([]byte(downloadKey))
-		if err != nil {
-			// If the key does not exist, initialize an empty map
-			if err == badger.ErrKeyNotFound {
-				existingData = make(map[string]interface{})
-				return nil
-			}
-			return err
-		}
-		jsonData, err := item.ValueCopy(nil)
-		if err != nil {
-			return err
-		}
-		return json.Unmarshal(jsonData, &existingData)
-	})
-	if err != nil {
-		http.Error(w, "Failed to retrieve existing data from database", http.StatusInternalServerError)
-		return
-	}
-
-	// Merge the new data into the existing JSON structure
-	mergeData(existingData, paramMap, strings.Split(path, "/"))
-
-	// Convert the updated data to a JSON string
-	updatedJSONData, err := json.Marshal(existingData)
-	if err != nil {
-		http.Error(w, "Error encoding updated data to JSON", http.StatusInternalServerError)
-		return
-	}
-
-	// Store the updated JSON data in the database using the download key
-	err = c.Db.Update(func(txn *badger.Txn) error {
-		e := badger.NewEntry([]byte(downloadKey), updatedJSONData).WithTTL(duration)
-		return txn.SetEntry(e)
-	})
-	if err != nil {
-		http.Error(w, "Failed to save updated data to database", http.StatusInternalServerError)
-		return
-	}
-
-	// Construct URLs for each parameter for the plainDownloadHandler
-	urls := make(map[string]string)
-	for key := range params {
-		urls[key] = fmt.Sprintf("http://%s/%s/plain/%s", r.Host, downloadKey, key)
-	}
-
-	// Construct the download URL
-	downloadURL := fmt.Sprintf("http://%s/%s/json", r.Host, downloadKey)
-
-	// Return the download URL in the response
-	jsonResponse(w, map[string]interface{}{
-		"message":        "Data uploaded successfully",
-		"download_url":   downloadURL,
-		"parameter_urls": urls,
-	})
+	c.handleUpload(w, r, uploadKey, path, true)
 }
 
 func (c Config) UploadHandler(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	uploadKey := vars["uploadKey"]
 
-	err := ValidateUploadKey(uploadKey)
-	if err != nil {
+	c.handleUpload(w, r, uploadKey, "", false)
+}
+
+func (c Config) handleUpload(w http.ResponseWriter, r *http.Request, uploadKey, path string, isPatch bool) {
+	// Validate upload key
+	if err := ValidateUploadKey(uploadKey); err != nil {
 		http.Error(w, err.Error(), http.StatusBadRequest)
 		return
 	}
 
-	// Derive the download key from the upload key
+	// Derive download key
 	downloadKey, err := domain.DeriveDownloadKey(uploadKey)
 	if err != nil {
 		http.Error(w, "Error deriving download key", http.StatusInternalServerError)
 		return
 	}
 
-	// Parse the duration for setting TTL on data
-	duration, err := time.ParseDuration(c.PersistDuration)
-	if err != nil {
-		http.Error(w, fmt.Sprintf("Invalid duration format: %v", err), http.StatusBadRequest)
+	// Collect parameters
+	paramMap := collectParams(r.URL.Query())
+
+	// Add timestamp to params
+	addTimestamp(paramMap)
+
+	// Handle data storage
+	if err := c.handleDataStorage(downloadKey, paramMap, path, isPatch); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
 
-	// Collect all parameters into a map
-	params := r.URL.Query()
+	// Construct and return response
+	constructAndReturnResponse(w, r, downloadKey, paramMap)
+}
+
+func collectParams(params map[string][]string) map[string]string {
 	paramMap := make(map[string]string)
 	for key, values := range params {
 		if len(values) > 0 {
-			// Sanitize each parameter value
 			sanitizedValue := sanitizeInput(values[0])
 			paramMap[key] = sanitizedValue
 		}
 	}
+	return paramMap
+}
 
-	// Get the current timestamp
+func addTimestamp(paramMap map[string]string) {
 	timestamp := time.Now().UTC().Format(time.RFC3339)
-
-	// Add the timestamp to the map
 	paramMap["timestamp"] = timestamp
+}
 
-	// Convert the parameter map to a JSON string
-	jsonData, err := json.Marshal(paramMap)
-	if err != nil {
-		http.Error(w, "Error encoding parameters to JSON", http.StatusInternalServerError)
-		return
-	}
+func (c Config) handleDataStorage(downloadKey string, paramMap map[string]string, path string, isPatch bool) error {
+	var dataToStore map[string]interface{}
 
-	// Store the JSON data in the database using the download key
-	err = c.Db.Update(func(txn *badger.Txn) error {
-		e := badger.NewEntry([]byte(downloadKey), jsonData).WithTTL(duration)
-		return txn.SetEntry(e)
-	})
-	if err != nil {
-		http.Error(w, "Failed to save to database", http.StatusInternalServerError)
-		return
+	if isPatch {
+		existingData, err := c.StorageInstance.Retrieve(downloadKey)
+		if err != nil {
+			return err
+		}
+		mergeData(existingData, paramMap, strings.Split(path, "/"))
+		dataToStore = existingData
+	} else {
+		dataToStore = make(map[string]interface{})
+		for k, v := range paramMap {
+			dataToStore[k] = v
+		}
 	}
 
-	// Construct URLs for each parameter for the plainDownloadHandler
+	return c.StorageInstance.Store(downloadKey, dataToStore)
+}
+
+func constructAndReturnResponse(w http.ResponseWriter, r *http.Request, downloadKey string, params map[string]string) {
 	urls := make(map[string]string)
 	for key := range params {
-		urls[key] = fmt.Sprintf("http://%s/%s/plain/%s", r.Host, downloadKey, key)
+		urls[key] = fmt.Sprintf("http://%s/d/%s/plain/%s", r.Host, downloadKey, key)
 	}
 
-	// Construct the download URL
-	downloadURL := fmt.Sprintf("http://%s/%s/json", r.Host, downloadKey)
+	downloadURL := fmt.Sprintf("http://%s/d/%s/json", r.Host, downloadKey)
 
-	// Return the download URL in the response
 	jsonResponse(w, map[string]interface{}{
 		"message":        "Data uploaded successfully",
 		"download_url":   downloadURL,
 		"parameter_urls": urls,
 	})
 }
 
-// mergeData merges the new data into the existing JSON structure based on the provided path.
 func mergeData(existingData map[string]interface{}, newData map[string]string, path []string) {
 	if len(path) == 0 || (len(path) == 1 && path[0] == "") {
 		for k, v := range newData {
@@ -212,14 +128,13 @@ func mergeData(existingData map[string]interface{}, newData map[string]string, p
 	}
 }
 
-// ValidateUploadKey checks if the uploadKey is a valid 256 bit hex string
 func ValidateUploadKey(uploadKey string) error {
-	uploadKey = strings.ToLower(uploadKey) // make it case insensitive
+	uploadKey = strings.ToLower(uploadKey)
 	decoded, err := hex.DecodeString(uploadKey)
 	if err != nil {
 		return errors.New("uploadKey must be a 256 bit hex string")
 	}
-	if len(decoded) != 32 { // 256 bits = 32 bytes
+	if len(decoded) != 32 {
 		return errors.New("uploadKey must be a 256 bit hex string")
 	}
 	return nil

install/install-service.sh

@@ -5,6 +5,7 @@ SERVICE_NAME="iot-ephemeral-value-store-server"
 USER="root"
 GROUP="root"
 DEFAULT_STORE_PATH="/var/lib/iot-ephemeral-value-store"
+DEFAULT_PORT=8080
 
 # Check if running as root
 if [ "$(id -u)" != "0" ]; then
@@ -13,13 +14,25 @@ if [ "$(id -u)" != "0" ]; then
 fi
 
 # Check if binary path is provided
-if [ "$#" -ne 1 ]; then
-    echo "Usage: $0 /path/to/binary"
+if [ "$#" -lt 1 ]; then
+    echo "Usage: $0 /path/to/binary [optional port]"
     exit 1
 fi
 
 BINARY_PATH=$1
 
+# Check if optional port is provided set to default port 8080
+if [ "$#" -eq 2 ]; then
+    # check if this is a valid port number
+    if ! [[ $2 =~ ^[0-9]+$ ]] || [ $2 -lt 1 ] || [ $2 -gt 65535 ]; then
+        echo "Error: Invalid port number"
+        exit 1
+    fi
+    PORT=$2
+else
+    PORT=$DEFAULT_PORT
+fi
+
 # Validate that the binary exists
 if [ ! -f "$BINARY_PATH" ]; then
     echo "Error: Binary not found at $BINARY_PATH"
@@ -56,7 +69,7 @@ After=network.target
 Type=simple
 User=$USER
 Group=$GROUP
-ExecStart=$INSTALL_PATH -store $DEFAULT_STORE_PATH
+ExecStart=$INSTALL_PATH -store $DEFAULT_STORE_PATH -port $PORT
 Restart=on-failure
 
 [Install]