Skip to content

Commit

Permalink
fix(export): export even if dag contains missing viz referernce
Browse files Browse the repository at this point in the history
Export has been failing with 'merkledag: not found' thanks to the common culprit of ds.Viz.RenderedPath
not being stored within the underlying dag. To properly fix this, we should instead fix & test the
renderedPath problem, but many datasets have been created with this issue already.

This commit supresses the error generated by referencing a path not found in ds.Viz.RenderedPath,
which has the effect of fixing export.

closes #1161
  • Loading branch information
b5 committed Mar 31, 2020
1 parent 7b5af05 commit 44c696d
Showing 1 changed file with 24 additions and 16 deletions.
40 changes: 24 additions & 16 deletions base/dsfs/dsutil/zip.go
Original file line number Diff line number Diff line change
Expand Up @@ -94,22 +94,7 @@ func WriteZipArchive(ctx context.Context, store cafs.Filestore, ds *dataset.Data
}

if ds.Viz.RenderedPath != "" {
// TODO (b5) - rendered viz isn't always being properly added to the
// encoded DAG, causing this to hang indefinitely on a network lookup.
// Use a short timeout for now to prevent the process from running too
// long. We should come up with a more permanent fix for this.
withTimeout, done := context.WithTimeout(ctx, time.Millisecond*250)
defer done()
rendered, err := store.Get(withTimeout, ds.Viz.RenderedPath)
if err != nil {
return err
}
target, err := zw.Create("index.html")
if err != nil {
return err
}
_, err = io.Copy(target, rendered)
if err != nil {
if err = maybeWriteRenderedViz(ctx, store, zw, ds.Viz.RenderedPath); err != nil {
return err
}
}
Expand All @@ -135,6 +120,29 @@ func WriteZipArchive(ctx context.Context, store cafs.Filestore, ds *dataset.Data
return zw.Close()
}

// TODO (b5) - rendered viz isn't always being properly added to the
// encoded DAG, causing this to hang indefinitely on a network lookup.
// Use a short timeout for now to prevent the process from running too
// long. We should come up with a more permanent fix for this.
func maybeWriteRenderedViz(ctx context.Context, store cafs.Filestore, zw *zip.Writer, vizPath string) error {
withTimeout, done := context.WithTimeout(ctx, time.Millisecond*250)
defer done()
rendered, err := store.Get(withTimeout, vizPath)
if err != nil {
if strings.Contains(err.Error(), "not found") {
return nil
}
return err
}

target, err := zw.Create("index.html")
if err != nil {
return err
}
_, err = io.Copy(target, rendered)
return err
}

// UnzipDatasetBytes is a convenince wrapper for UnzipDataset
func UnzipDatasetBytes(zipData []byte, ds *dataset.Dataset) error {
return UnzipDataset(bytes.NewReader(zipData), int64(len(zipData)), ds)
Expand Down

0 comments on commit 44c696d

Please sign in to comment.