dedupe cmd Run funcs

Signed-off-by: Ransom Williams <rwilliams@oneconcern.com>

cmd/datamon/cmd/bundle_download.go

@@ -4,12 +4,8 @@ package cmd
 
 import (
  "context"
- "fmt"
 
  "github.com/oneconcern/datamon/pkg/core"
- "github.com/oneconcern/datamon/pkg/storage/gcs"
- "github.com/oneconcern/datamon/pkg/storage/localfs"
- "github.com/spf13/afero"
  "github.com/spf13/cobra"
 )
 
@@ -23,41 +19,25 @@ var BundleDownloadCmd = &cobra.Command{
  Long: "Download a readonly, non-interactive view of the entire data that is part of a bundle. If --bundle is not specified" +
  " the latest bundle will be downloaded",
  Run: func(cmd *cobra.Command, args []string) {
-
- sourceStore, err := gcs.New(params.repo.MetadataBucket, config.Credential)
+ remoteStores, err := paramsToRemoteCmdStores(params)
  if err != nil {
  logFatalln(err)
  }
- blobStore, err := gcs.New(params.repo.BlobBucket, config.Credential)
+ destinationStore, err := paramsToDestStore(params, true, "")
  if err != nil {
  logFatalln(err)
  }
- path, err := sanitizePath(params.bundle.DataPath)
- fmt.Println("Using path: " + path)
- if err != nil {
- logFatalln("Failed path validation: " + err.Error())
- }
- createPath(path)
- fs := afero.NewBasePathFs(afero.NewOsFs(), path+"/")
- empty, err := afero.IsEmpty(fs, "/")
- if err != nil {
- logFatalln("Failed path validation: " + err.Error())
- }
- if !empty {
- logFatalf("%s should be empty", path)
- }
- destinationStore := localfs.New(fs)
 
- err = setLatestOrLabelledBundle(sourceStore)
+ err = setLatestOrLabelledBundle(remoteStores.meta)
  if err != nil {
  logFatalln(err)
  }
  bd := core.NewBDescriptor()
  bundle := core.New(bd,
  core.Repo(params.repo.RepoName),
- core.MetaStore(sourceStore),
+ core.MetaStore(remoteStores.meta),
  core.ConsumableStore(destinationStore),
- core.BlobStore(blobStore),
+ core.BlobStore(remoteStores.blob),
  core.BundleID(params.bundle.ID),
  core.ConcurrentFileDownloads(params.bundle.ConcurrencyFactor/fileDownloadsByConcurrencyFactor),
  )

cmd/datamon/cmd/bundle_download_file.go

@@ -2,13 +2,8 @@ package cmd
 
 import (
  "context"
- "os"
- "path/filepath"
 
  "github.com/oneconcern/datamon/pkg/core"
- "github.com/oneconcern/datamon/pkg/storage/gcs"
- "github.com/oneconcern/datamon/pkg/storage/localfs"
- "github.com/spf13/afero"
  "github.com/spf13/cobra"
 )
 
@@ -17,31 +12,25 @@ var bundleDownloadFileCmd = &cobra.Command{
  Short: "Download a file from bundle",
  Long: "Download a readonly, non-interactive view of a single file from a bundle",
  Run: func(cmd *cobra.Command, args []string) {
-
- metadataStore, err := gcs.New(params.repo.MetadataBucket, config.Credential)
+ remoteStores, err := paramsToRemoteCmdStores(params)
  if err != nil {
  logFatalln(err)
  }
- blobStore, err := gcs.New(params.repo.BlobBucket, config.Credential)
+ destinationStore, err := paramsToDestStore(params, false, "")
  if err != nil {
  logFatalln(err)
  }
- path, err := filepath.Abs(filepath.Clean(params.bundle.DataPath))
- if err != nil {
- logFatalf("Failed path validation: %s", err)
- }
- _ = os.MkdirAll(path, 0700)
- destinationStore := localfs.New(afero.NewBasePathFs(afero.NewOsFs(), path))
- err = setLatestOrLabelledBundle(metadataStore)
+
+ err = setLatestOrLabelledBundle(remoteStores.meta)
  if err != nil {
  logFatalln(err)
  }
  bd := core.NewBDescriptor()
  bundle := core.New(bd,
  core.Repo(params.repo.RepoName),
- core.MetaStore(metadataStore),
+ core.MetaStore(remoteStores.meta),
  core.ConsumableStore(destinationStore),
- core.BlobStore(blobStore),
+ core.BlobStore(remoteStores.blob),
  core.BundleID(params.bundle.ID),
  )
 

cmd/datamon/cmd/bundle_list.go

@@ -6,7 +6,6 @@ import (
  "text/template"
 
  "github.com/oneconcern/datamon/pkg/core"
- "github.com/oneconcern/datamon/pkg/storage/gcs"
 
  "github.com/spf13/cobra"
 )
@@ -18,11 +17,11 @@ var BundleListCommand = &cobra.Command{
  Run: func(cmd *cobra.Command, args []string) {
  const listLineTemplateString = `{{.ID}} , {{.Timestamp}} , {{.Message}}`
  listLineTemplate := template.Must(template.New("list line").Parse(listLineTemplateString))
- store, err := gcs.New(params.repo.MetadataBucket, config.Credential)
+ remoteStores, err := paramsToRemoteCmdStores(params)
  if err != nil {
  logFatalln(err)
  }
- bundleDescriptors, err := core.ListBundles(params.repo.RepoName, store)
+ bundleDescriptors, err := core.ListBundles(params.repo.RepoName, remoteStores.meta)
  if err != nil {
  logFatalln(err)
  }

cmd/datamon/cmd/bundle_list_file.go

@@ -7,7 +7,6 @@ import (
  "github.com/oneconcern/datamon/pkg/core"
 
  "github.com/oneconcern/datamon/pkg/model"
- "github.com/oneconcern/datamon/pkg/storage/gcs"
  "github.com/spf13/cobra"
 )
 
@@ -16,19 +15,18 @@ var bundleFileList = &cobra.Command{
  Short: "List files in a bundle",
  Long: "List all the files in a bundle",
  Run: func(cmd *cobra.Command, args []string) {
-
- store, err := gcs.New(params.repo.MetadataBucket, config.Credential)
+ remoteStores, err := paramsToRemoteCmdStores(params)
  if err != nil {
  logFatalln(err)
  }
- err = setLatestOrLabelledBundle(store)
+ err = setLatestOrLabelledBundle(remoteStores.meta)
  if err != nil {
  logFatalln(err)
  }
  bundle := core.Bundle{
  RepoID: params.repo.RepoName,
  BundleID: params.bundle.ID,
- MetaStore: store,
+ MetaStore: remoteStores.meta,
  ConsumableStore: nil,
  BlobStore: nil,
  BundleDescriptor: model.BundleDescriptor{},

cmd/datamon/cmd/bundle_mount.go

@@ -5,18 +5,13 @@ package cmd
 import (
  "context"
  "fmt"
- "io/ioutil"
  "log"
  "os"
 
  daemonizer "github.com/jacobsa/daemonize"
 
- "github.com/oneconcern/datamon/pkg/dlogger"
-
  "github.com/oneconcern/datamon/pkg/core"
- "github.com/oneconcern/datamon/pkg/storage/gcs"
- "github.com/oneconcern/datamon/pkg/storage/localfs"
- "github.com/spf13/afero"
+ "github.com/oneconcern/datamon/pkg/dlogger"
 
  "github.com/spf13/cobra"
 )
@@ -31,6 +26,18 @@ func undaemonizeArgs(args []string) []string {
  return foregroundArgs
 }
 
+/**
+ * call this function followed by return at any point in a Run: func in order to run the command as a pseudo-daemonized process.
+ * conceptually, pseudo-daemonization is akin to usual daemon processes in C wherein fork() does the job
+ * of splitting the process in two within the control-flow of the given process, copying or sharing memory segments as needed.
+ *
+ * Go doesn't fork() because of the runtime. only exec() is available. so what pseudo-daemonization means is exec()ing
+ * the selfsame binary in a goroutine with some additional IPC communication via pipes to simulate a meaningful fork()-like
+ * return value indicating whether the process started successfully without blocking on the exec()ed process's exit code.
+ *
+ * specifically, `daemonizer.SignalOutcome(nil)` is used to in Run() to bracket the daemonized process and the end of the
+ * initialization code.
+ */
 func runDaemonized() {
  var path string
  path, err := os.Executable()
@@ -60,6 +67,10 @@ func runDaemonized() {
  }
 }
 
+/**
+ * in between runDaemonized() and SignalOutcome(), call this function instead of logFatalln() or similar
+ * in case of errors
+ */
 func onDaemonError(err error) {
  if errSig := daemonizer.SignalOutcome(err); errSig != nil {
  logFatalln(fmt.Errorf("error SignalOutcome: %v, cause: %v", errSig, err))
@@ -73,50 +84,31 @@ var mountBundleCmd = &cobra.Command{
  Short: "Mount a bundle",
  Long: "Mount a readonly, non-interactive view of the entire data that is part of a bundle",
  Run: func(cmd *cobra.Command, args []string) {
+ // cf. comments on runDaemonized
  if params.bundle.Daemonize {
  runDaemonized()
  return
  }
-
- var err error
-
- var consumableStorePath string
- if params.bundle.DataPath == "" {
- consumableStorePath, err = ioutil.TempDir("", "datamon-mount-destination")
- if err != nil {
- log.Fatalf("Couldn't create temporary directory: %v\n", err)
- return
- }
- } else {
- consumableStorePath, err = sanitizePath(params.bundle.DataPath)
- if err != nil {
- log.Fatalf("Failed to sanitize destination: %s\n", params.bundle.DataPath)
- return
- }
- createPath(consumableStorePath)
- }
-
- metadataSource, err := gcs.New(params.repo.MetadataBucket, config.Credential)
+ remoteStores, err := paramsToRemoteCmdStores(params)
  if err != nil {
  onDaemonError(err)
  }
- blobStore, err := gcs.New(params.repo.BlobBucket, config.Credential)
+ consumableStore, err := paramsToDestStore(params, false, "datamon-mount-destination")
  if err != nil {
  onDaemonError(err)
  }
- consumableStore := localfs.New(afero.NewBasePathFs(afero.NewOsFs(), consumableStorePath))
 
- err = setLatestOrLabelledBundle(metadataSource)
+ err = setLatestOrLabelledBundle(remoteStores.meta)
  if err != nil {
  logFatalln(err)
  }
  bd := core.NewBDescriptor()
  bundle := core.New(bd,
  core.Repo(params.repo.RepoName),
  core.BundleID(params.bundle.ID),
- core.BlobStore(blobStore),
+ core.BlobStore(remoteStores.blob),
  core.ConsumableStore(consumableStore),
- core.MetaStore(metadataSource),
+ core.MetaStore(remoteStores.meta),
  core.Streaming(params.bundle.Stream),
  )
  logger, err := dlogger.GetLogger(params.root.logLevel)

cmd/datamon/cmd/bundle_mutable_mount.go

@@ -5,17 +5,10 @@ package cmd
 import (
  "context"
  "fmt"
- "io/ioutil"
- "log"
 
  daemonizer "github.com/jacobsa/daemonize"
 
  "github.com/oneconcern/datamon/pkg/core"
- "github.com/oneconcern/datamon/pkg/model"
- "github.com/oneconcern/datamon/pkg/storage/gcs"
- "github.com/oneconcern/datamon/pkg/storage/localfs"
- "github.com/spf13/afero"
-
  "github.com/spf13/cobra"
 )
 
@@ -25,59 +18,34 @@ var mutableMountBundleCmd = &cobra.Command{
  Short: "Create a bundle incrementally with filesystem operations",
  Long: "Write directories and files to the mountpoint. Unmount or send SIGINT to this process to save.",
  Run: func(cmd *cobra.Command, args []string) {
- if params.repo.ContributorEmail == "" {
- logFatalln(fmt.Errorf("contributor email must be set in config or as a cli param"))
- }
- if params.repo.ContributorName == "" {
- logFatalln(fmt.Errorf("contributor name must be set in config or as a cli param"))
+ contributor, err := paramsToContributor(params)
+ if err != nil {
+ logFatalln(err)
  }
-
+ // cf. comments on runDaemonized in bundle_mount.go
  if params.bundle.Daemonize {
  runDaemonized()
  return
  }
- var err error
- var consumableStorePath string
- if params.bundle.DataPath == "" {
- consumableStorePath, err = ioutil.TempDir("", "datamon-mount-destination")
- if err != nil {
- log.Fatalf("Couldn't create temporary directory: %v\n", err)
- return
- }
- } else {
- consumableStorePath, err = sanitizePath(params.bundle.DataPath)
- if err != nil {
- log.Fatalf("Failed to sanitize destination: %s\n", params.bundle.DataPath)
- return
- }
- createPath(consumableStorePath)
- }
-
- metadataSource, err := gcs.New(params.repo.MetadataBucket, config.Credential)
+ remoteStores, err := paramsToRemoteCmdStores(params)
  if err != nil {
  onDaemonError(err)
  }
- blobStore, err := gcs.New(params.repo.BlobBucket, config.Credential)
+ consumableStore, err := paramsToSrcStore(params, true)
  if err != nil {
  onDaemonError(err)
  }
- consumableStore := localfs.New(afero.NewBasePathFs(afero.NewOsFs(), consumableStorePath))
 
  bd := core.NewBDescriptor(
  core.Message(params.bundle.Message),
- core.Contributors([]model.Contributor{{
- Name: params.repo.ContributorName,
- Email: params.repo.ContributorEmail,
- },
- }),
+ core.Contributor(contributor),
  )
  bundle := core.New(bd,
  core.Repo(params.repo.RepoName),
- core.BlobStore(blobStore),
+ core.BlobStore(remoteStores.blob),
  core.ConsumableStore(consumableStore),
- core.MetaStore(metadataSource),
+ core.MetaStore(remoteStores.meta),
  )
-
  fs, err := core.NewMutableFS(bundle, params.bundle.DataPath)
  if err != nil {
  onDaemonError(err)
@@ -86,7 +54,6 @@ var mutableMountBundleCmd = &cobra.Command{
  if err != nil {
  onDaemonError(err)
  }
-
  registerSIGINTHandlerMount(params.bundle.MountPath)
  if err = daemonizer.SignalOutcome(nil); err != nil {
  logFatalln(err)