Removing collections package and API cleanup (#1501)

cmd/bee-file/main.go

@@ -5,31 +5,13 @@
 package main
 
 import (
-	"bytes"
-	"context"
-	"encoding/json"
+	"errors"
 	"fmt"
-	"io"
-	"io/ioutil"
 	"os"
-	"path/filepath"
 
-	cmdfile "github.com/ethersphere/bee/cmd/internal/file"
-	"github.com/ethersphere/bee/pkg/collection/entry"
-	"github.com/ethersphere/bee/pkg/file"
-	"github.com/ethersphere/bee/pkg/file/joiner"
-	"github.com/ethersphere/bee/pkg/file/splitter"
-	"github.com/ethersphere/bee/pkg/logging"
-	"github.com/ethersphere/bee/pkg/storage"
-	"github.com/ethersphere/bee/pkg/swarm"
 	"github.com/spf13/cobra"
 )
 
-const (
-	defaultMimeType     = "application/octet-stream"
-	limitMetadataLength = swarm.ChunkSize
-)
-
 var (
 	filename     string // flag variable, filename to use in metadata
 	mimeType     string // flag variable, mime type to use in metadata
@@ -41,186 +23,11 @@ var (
 	ssl          bool   // flag variable, uses https for api if set
 	retrieve     bool   // flag variable, if set will resolve and retrieve referenced file
 	verbosity    string // flag variable, debug level
-	logger       logging.Logger
 )
 
-// getEntry handles retrieving and writing a file from the file entry
-// referenced by the given address.
-func getEntry(cmd *cobra.Command, args []string) (err error) {
-	// process the reference to retrieve
-	addr, err := swarm.ParseHexAddress(args[0])
-	if err != nil {
-		return err
-	}
-
-	// initialize interface with HTTP API
-	store := cmdfile.NewApiStore(host, port, ssl)
-
-	buf := bytes.NewBuffer(nil)
-	writeCloser := cmdfile.NopWriteCloser(buf)
-	limitBuf := cmdfile.NewLimitWriteCloser(writeCloser, limitMetadataLength)
-	j, _, err := joiner.New(cmd.Context(), store, addr)
-	if err != nil {
-		return err
-	}
-
-	_, err = file.JoinReadAll(cmd.Context(), j, limitBuf)
-	if err != nil {
-		return err
-	}
-	e := &entry.Entry{}
-	err = e.UnmarshalBinary(buf.Bytes())
-	if err != nil {
-		return err
-	}
-
-	j, _, err = joiner.New(cmd.Context(), store, e.Metadata())
-	if err != nil {
-		return err
-	}
-
-	buf = bytes.NewBuffer(nil)
-
-	_, err = file.JoinReadAll(cmd.Context(), j, buf)
-	if err != nil {
-		return err
-	}
-
-	// retrieve metadata
-	metaData := &entry.Metadata{}
-	err = json.Unmarshal(buf.Bytes(), metaData)
-	if err != nil {
-		return err
-	}
-	logger.Debugf("Filename: %s", metaData.Filename)
-	logger.Debugf("MIME-type: %s", metaData.MimeType)
-
-	if outDir == "" {
-		outDir = "."
-	} else {
-		err := os.MkdirAll(outDir, 0o777) // skipcq: GSC-G301
-		if err != nil {
-			return err
-		}
-	}
-	outFilePath := filepath.Join(outDir, metaData.Filename)
-
-	// create output dir if not exist
-	if outDir != "." {
-		err := os.MkdirAll(outDir, 0o777) // skipcq: GSC-G301
-		if err != nil {
-			return err
-		}
-	}
-
-	// protect any existing file unless explicitly told not to
-	outFileFlags := os.O_CREATE | os.O_WRONLY
-	if outFileForce {
-		outFileFlags |= os.O_TRUNC
-	} else {
-		outFileFlags |= os.O_EXCL
-	}
-
-	// open the file
-	outFile, err := os.OpenFile(outFilePath, outFileFlags, 0o666) // skipcq: GSC-G302
-	if err != nil {
-		return err
-	}
-	defer outFile.Close()
-
-	j, _, err = joiner.New(cmd.Context(), store, e.Reference())
-	if err != nil {
-		return err
-	}
-
-	_, err = file.JoinReadAll(cmd.Context(), j, outFile)
-	return err
-}
-
-// putEntry creates a new file entry with the given reference.
-func putEntry(cmd *cobra.Command, args []string) (err error) {
-	// process the reference to retrieve
-	addr, err := swarm.ParseHexAddress(args[0])
-	if err != nil {
-		return err
-	}
-	// add the fsStore and/or apiStore, depending on flags
-	stores := cmdfile.NewTeeStore()
-	if outDir != "" {
-		err := os.MkdirAll(outDir, 0o777) // skipcq: GSC-G301
-		if err != nil {
-			return err
-		}
-		store := cmdfile.NewFsStore(outDir)
-		stores.Add(store)
-	}
-	if useHttp {
-		store := cmdfile.NewApiStore(host, port, ssl)
-		stores.Add(store)
-	}
-
-	// create metadata object, with defaults for missing values
-	if filename == "" {
-		filename = args[0]
-	}
-	if mimeType == "" {
-		mimeType = defaultMimeType
-	}
-	metadata := entry.NewMetadata(filename)
-	metadata.MimeType = mimeType
-
-	// serialize metadata and send it to splitter
-	metadataBytes, err := json.Marshal(metadata)
-	if err != nil {
-		return err
-	}
-	logger.Debugf("metadata contents: %s", metadataBytes)
-
-	// set up splitter to process the metadata
-	s := splitter.NewSimpleSplitter(stores, storage.ModePutUpload)
-	ctx := context.Background()
-
-	// first add metadata
-	metadataBuf := bytes.NewBuffer(metadataBytes)
-	metadataReader := io.LimitReader(metadataBuf, int64(len(metadataBytes)))
-	metadataReadCloser := ioutil.NopCloser(metadataReader)
-	metadataAddr, err := s.Split(ctx, metadataReadCloser, int64(len(metadataBytes)), false)
-	if err != nil {
-		return err
-	}
-
-	// create entry from given reference and metadata,
-	// serialize and send to splitter
-	fileEntry := entry.New(addr, metadataAddr)
-	fileEntryBytes, err := fileEntry.MarshalBinary()
-	if err != nil {
-		return err
-	}
-	fileEntryBuf := bytes.NewBuffer(fileEntryBytes)
-	fileEntryReader := io.LimitReader(fileEntryBuf, int64(len(fileEntryBytes)))
-	fileEntryReadCloser := ioutil.NopCloser(fileEntryReader)
-	fileEntryAddr, err := s.Split(ctx, fileEntryReadCloser, int64(len(fileEntryBytes)), false)
-	if err != nil {
-		return err
-	}
-
-	// output reference to file entry
-	cmd.Println(fileEntryAddr)
-	return nil
-}
-
 // Entry is the underlying procedure for the CLI command
 func Entry(cmd *cobra.Command, args []string) (err error) {
-	logger, err = cmdfile.SetLogger(cmd, verbosity)
-	if err != nil {
-		fmt.Fprintln(os.Stderr, err.Error())
-		os.Exit(1)
-	}
-
-	if retrieve {
-		return getEntry(cmd, args)
-	}
-	return putEntry(cmd, args)
+	return errors.New("command is deprecated")
 }
 
 func main() {

openapi/Swarm.yaml

@@ -154,22 +154,31 @@ paths:
         default:
           description: Default response
 
-  "/files":
+  "/bzz":
     post:
-      summary: "Upload file"
+      summary: "Upload file or a collection of files"
+      description: "In order to upload a collection, user can send a multipart request with all the files populated in the form data with appropriate headers.\n\n
+        User can also upload a tar file along with the swarm-collection header. This will upload the tar file after extracting the entire directory structure.\n\n
+        If the swarm-collection header is absent, all requests (including tar files) are considered as single file uploads.\n\n
+        A multipart request is treated as a collection regardless of whether the swarm-collection header is present. This means in order to serve single files
+        uploaded as a multipart request, the swarm-index-document header should be used with the name of the file."
       tags:
         - File
+        - Collection
       parameters:
         - in: query
           name: name
           schema:
             $ref: "SwarmCommon.yaml#/components/schemas/FileName"
           required: false
-          description: Filename
+          description: Filename when uploading single file
         - $ref: "SwarmCommon.yaml#/components/parameters/SwarmTagParameter"
         - $ref: "SwarmCommon.yaml#/components/parameters/SwarmPinParameter"
         - $ref: "SwarmCommon.yaml#/components/parameters/SwarmEncryptParameter"
         - $ref: "SwarmCommon.yaml#/components/parameters/ContentTypePreserved"
+        - $ref: "SwarmCommon.yaml#/components/parameters/SwarmCollection"
+        - $ref: "SwarmCommon.yaml#/components/parameters/SwarmIndexDocumentParameter"
+        - $ref: "SwarmCommon.yaml#/components/parameters/SwarmErrorDocumentParameter"
       requestBody:
         content:
           multipart/form-data:
@@ -184,76 +193,6 @@ paths:
             schema:
               type: string
               format: binary
-      responses:
-        "200":
-          description: Ok
-          headers:
-            "swarm-tag":
-              $ref: "SwarmCommon.yaml#/components/headers/SwarmTag"
-            "etag":
-              $ref: "SwarmCommon.yaml#/components/headers/ETag" 
-          content:
-            application/json:
-              schema:
-                $ref: "SwarmCommon.yaml#/components/schemas/ReferenceResponse"
-        "400":
-          $ref: "SwarmCommon.yaml#/components/responses/400"
-        "403":
-          $ref: "SwarmCommon.yaml#/components/responses/403"
-        "500":
-          $ref: "SwarmCommon.yaml#/components/responses/500"
-        default:
-          description: Default response
-
-  "/files/{reference}":
-    get:
-      summary: "Get referenced file"
-      tags:
-        - File
-      parameters:
-        - in: path
-          name: reference
-          schema:
-            $ref: "SwarmCommon.yaml#/components/schemas/SwarmReference"
-          required: true
-          description: Swarm address of content
-        - $ref: "SwarmCommon.yaml#/components/parameters/SwarmRecoveryTargetsParameter"
-      responses:
-        "200":
-          description: Ok
-          headers:
-            "swarm-recovery-targets":
-              $ref: "SwarmCommon.yaml#/components/headers/SwarmRecoveryTargets"
-            "ETag":
-              $ref: "SwarmCommon.yaml#/components/headers/ETag"
-          content:
-            application/octet-stream:
-              schema:
-                type: string
-                format: binary
-        "400":
-          $ref: "SwarmCommon.yaml#/components/responses/400"
-        "404":
-          $ref: "SwarmCommon.yaml#/components/responses/404"
-        "500":
-          $ref: "SwarmCommon.yaml#/components/responses/500"
-        default:
-          description: Default response
-
-  "/dirs":
-    post:
-      summary: "Upload a collection"
-      tags:
-        - Collection
-      parameters:
-        - $ref: "SwarmCommon.yaml#/components/parameters/SwarmTagParameter"
-        - $ref: "SwarmCommon.yaml#/components/parameters/SwarmPinParameter"
-        - $ref: "SwarmCommon.yaml#/components/parameters/SwarmEncryptParameter"
-        - $ref: "SwarmCommon.yaml#/components/parameters/SwarmIndexDocumentParameter"
-        - $ref: "SwarmCommon.yaml#/components/parameters/SwarmErrorDocumentParameter"
-        - $ref: "SwarmCommon.yaml#/components/parameters/ContentTypePreserved"
-      requestBody:
-        content:
           application/x-tar:
             schema:
               type: string
@@ -264,6 +203,8 @@ paths:
           headers:
             "swarm-tag":
               $ref: "SwarmCommon.yaml#/components/headers/SwarmTag"
+            "etag":
+              $ref: "SwarmCommon.yaml#/components/headers/ETag" 
           content:
             application/json:
               schema:
@@ -279,7 +220,7 @@ paths:
 
   "/bzz/{reference}":
     get:
-      summary: "Get index document from a collection of files"
+      summary: "Get file or index document from a collection of files"
       tags:
         - Collection
       parameters:

openapi/SwarmCommon.yaml

@@ -493,6 +493,14 @@ components:
       required: false
       description: Configure custom error document to be returned when a specified path can not be found in collection
 
+    SwarmCollection:
+      in: header
+      name: swarm-collection
+      schema:
+        type: boolean
+      required: false
+      description: Upload file/files as a collection
+
   responses:
     "204":
       description: The resource was deleted successfully.

pkg/api/api.go

@@ -40,6 +40,7 @@ const (
 	SwarmErrorDocumentHeader = "Swarm-Error-Document"
 	SwarmFeedIndexHeader     = "Swarm-Feed-Index"
 	SwarmFeedIndexNextHeader = "Swarm-Feed-Index-Next"
+	SwarmCollectionHeader    = "Swarm-Collection"
 )
 
 // The size of buffer used for prefetching content with Langos.
@@ -54,9 +55,20 @@ const (
 	largeBufferFilesizeThreshold = 10 * 1000000 // ten megs
 )
 
+const (
+	contentTypeHeader = "Content-Type"
+	multiPartFormData = "multipart/form-data"
+	contentTypeTar    = "application/x-tar"
+)
+
 var (
 	errInvalidNameOrAddress = errors.New("invalid name or bzz address")
 	errNoResolver           = errors.New("no resolver connected")
+	invalidRequest          = errors.New("could not validate request")
+	invalidContentType      = errors.New("invalid content-type")
+	invalidContentLength    = errors.New("invalid content-length")
+	directoryStoreError     = errors.New("could not store directory")
+	fileStoreError          = errors.New("could not store file")
 )
 
 // Service is the API service interface.