From a64d3ced0b7a5afbeb9993eba2a74c6b1648e911 Mon Sep 17 00:00:00 2001 From: Jesus Aguilar <3589801+giventocode@users.noreply.github.com> Date: Wed, 28 Feb 2018 00:00:19 -0500 Subject: [PATCH 1/7] - Listing of source objects no longer blocks the transfer. - S3 folders are not counted in the batch totals. - Dial connection errors are retried. - TCP connections are closed for small requests. --- README.md | 4 +- blobporter.go | 54 +++++------ internal/azutil.go | 37 ++++++-- internal/const.go | 2 +- pipelinefactory.go | 17 ++-- sources/azblobinfo.go | 74 ++++++++++------ sources/http.go | 45 +--------- sources/multifile.go | 4 +- sources/ostorefactory.go | 75 ---------------- sources/perfsource.go | 4 +- sources/s3info.go | 122 +++++++++++-------------- sources/sourcefactory.go | 182 ++++++++++++++++++++++++++++++++++++++ sources/types.go | 7 +- transfer/transfer_test.go | 72 +++++++++++---- 14 files changed, 417 insertions(+), 282 deletions(-) delete mode 100644 sources/ostorefactory.go create mode 100644 sources/sourcefactory.go diff --git a/README.md b/README.md index 0d25369..9a221e1 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Sources and targets are decoupled, this design enables the composition of variou Download, extract and set permissions: ```bash -wget -O bp_linux.tar.gz https://github.com/Azure/blobporter/releases/download/v0.6.09/bp_linux.tar.gz +wget -O bp_linux.tar.gz https://github.com/Azure/blobporter/releases/download/v0.6.10/bp_linux.tar.gz tar -xvf bp_linux.tar.gz linux_amd64/blobporter chmod +x ~/linux_amd64/blobporter cd ~/linux_amd64 @@ -46,7 +46,7 @@ export ACCOUNT_KEY= ### Windows -Download [BlobPorter.exe](https://github.com/Azure/blobporter/releases/download/v0.6.09/bp_windows.zip) +Download [BlobPorter.exe](https://github.com/Azure/blobporter/releases/download/v0.6.10/bp_windows.zip) Set environment variables (if using the command prompt): diff --git a/blobporter.go b/blobporter.go index 199a409..1f0fc19 100644 --- a/blobporter.go +++ b/blobporter.go @@ -10,13 +10,12 @@ import ( "strconv" "sync/atomic" + "github.com/Azure/blobporter/internal" "github.com/Azure/blobporter/pipeline" "github.com/Azure/blobporter/transfer" "github.com/Azure/blobporter/util" - "github.com/Azure/blobporter/internal" ) - var argsUtil paramParserValidator func init() { @@ -95,33 +94,29 @@ func init() { var dataTransferred uint64 var targetRetries int32 -func displayFilesToTransfer(sourcesInfo []pipeline.SourceInfo, numOfBatches int, batchNumber int) { - if numOfBatches == 1 { - fmt.Printf("Files to Transfer (%v) :\n", argsUtil.params.transferType) - var totalSize uint64 - summary := "" - - for _, source := range sourcesInfo { - //if the source is URL, remove the QS - display := source.SourceName - if u, err := url.Parse(source.SourceName); err == nil { - display = fmt.Sprintf("%v%v", u.Hostname(), u.Path) - } - summary = summary + fmt.Sprintf("Source: %v Size:%v \n", display, source.Size) - totalSize = totalSize + source.Size - } +func displayFilesToTransfer(sourcesInfo []pipeline.SourceInfo) { + fmt.Printf("\nFiles to Transfer (%v) :\n", argsUtil.params.transferType) + var totalSize uint64 + summary := "" - if len(sourcesInfo) < 10 { - fmt.Printf(summary) - return + for _, source := range sourcesInfo { + //if the source is URL, remove the QS + display := source.SourceName + if u, err := url.Parse(source.SourceName); err == nil { + display = fmt.Sprintf("%v%v", u.Hostname(), u.Path) } + summary = summary + fmt.Sprintf("Source: %v Size:%v \n", display, source.Size) + totalSize = totalSize + source.Size + } - fmt.Printf("%v files. Total size:%v\n", len(sourcesInfo), totalSize) - + if len(sourcesInfo) < 10 { + fmt.Printf(summary) return } - fmt.Printf("\nBatch transfer (%v).\nFiles per Batch: %v.\nBatch: %v of %v\n ", argsUtil.params.transferType, len(sourcesInfo), batchNumber+1, numOfBatches) + fmt.Printf("%v files. Total size:%v\n", len(sourcesInfo), totalSize) + + return } func main() { @@ -141,12 +136,17 @@ func main() { stats := transfer.NewStats(argsUtil.params.numberOfWorkers, argsUtil.params.numberOfReaders) - for b, sourcePipeline := range sourcePipelines { - sourcesInfo := sourcePipeline.GetSourcesInfo() + for sourcePipeline := range sourcePipelines { + + if sourcePipeline.Err != nil { + log.Fatal(sourcePipeline.Err) + } + + sourcesInfo := sourcePipeline.Source.GetSourcesInfo() - tfer := transfer.NewTransfer(&sourcePipeline, &targetPipeline, argsUtil.params.numberOfReaders, argsUtil.params.numberOfWorkers, argsUtil.params.blockSize) + tfer := transfer.NewTransfer(&sourcePipeline.Source, &targetPipeline, argsUtil.params.numberOfReaders, argsUtil.params.numberOfWorkers, argsUtil.params.blockSize) - displayFilesToTransfer(sourcesInfo, len(sourcePipelines), b) + displayFilesToTransfer(sourcesInfo) pb := getProgressBarDelegate(tfer.TotalSize, argsUtil.params.quietMode) tfer.StartTransfer(argsUtil.params.dedupeLevel, pb) diff --git a/internal/azutil.go b/internal/azutil.go index 3dd4c33..6ef0505 100644 --- a/internal/azutil.go +++ b/internal/azutil.go @@ -11,7 +11,7 @@ import ( "os" "syscall" "time" - + "github.com/Azure/blobporter/util" "github.com/Azure/azure-pipeline-go/pipeline" "github.com/Azure/azure-storage-blob-go/2016-05-31/azblob" @@ -190,6 +190,7 @@ func (p *AzUtil) PutBlockBlob(blobName string, body io.ReadSeeker, md5 []byte) e h := azblob.BlobHTTPHeaders{} + //16 is md5.Size if len(md5) != 16 { var md5bytes [16]byte @@ -375,6 +376,24 @@ func isWinsockTimeOutError(err error) net.Error { return nil } +func isDialConnectError(err error) net.Error { + if uerr, ok := err.(*url.Error); ok { + if derr, ok := uerr.Err.(*net.OpError); ok { + if serr, ok := derr.Err.(*os.SyscallError); ok && serr.Syscall == "connect" { + return &retriableError{error: err} + } + } + } + return nil +} + +func isRetriableDialError(err error) net.Error { + if derr := isWinsockTimeOutError(err); derr != nil { + return derr + } + return isDialConnectError(err) +} + type retriableError struct { error } @@ -387,11 +406,19 @@ func (*retriableError) Temporary() bool { return true } +const tcpKeepOpenMinLength = 8 * int64(util.MB) + func (p *clientPolicy) Do(ctx context.Context, request pipeline.Request) (pipeline.Response, error) { - r, err := pipelineHTTPClient.Do(request.WithContext(ctx)) + req := request.WithContext(ctx) + + if req.ContentLength < tcpKeepOpenMinLength { + req.Close=true + } + + r, err := pipelineHTTPClient.Do(req) pipresp := pipeline.NewHTTPResponse(r) if err != nil { - if derr := isWinsockTimeOutError(err); derr != nil { + if derr := isRetriableDialError(err); derr != nil { return pipresp, derr } err = pipeline.NewError(err, "HTTP request failed") @@ -411,9 +438,9 @@ func newpipelineHTTPClient() *http.Client { KeepAlive: 30 * time.Second, DualStack: true, }).Dial, - MaxIdleConns: 0, + MaxIdleConns: 100, MaxIdleConnsPerHost: 100, - IdleConnTimeout: 90 * time.Second, + IdleConnTimeout: 60 * time.Second, TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 1 * time.Second, DisableKeepAlives: false, diff --git a/internal/const.go b/internal/const.go index 0cbdcf4..1a6c02b 100644 --- a/internal/const.go +++ b/internal/const.go @@ -6,7 +6,7 @@ import ( ) //ProgramVersion blobporter version -const ProgramVersion = "0.6.09" +const ProgramVersion = "0.6.10" //HTTPClientTimeout HTTP client timeout when reading from HTTP sources and try timeout for blob storage operations. var HTTPClientTimeout = 90 diff --git a/pipelinefactory.go b/pipelinefactory.go index 72da262..c27148c 100644 --- a/pipelinefactory.go +++ b/pipelinefactory.go @@ -9,11 +9,11 @@ import ( "github.com/Azure/blobporter/transfer" ) -func newTransferPipelines(params *validatedParameters) ([]pipeline.SourcePipeline, pipeline.TargetPipeline, error) { +func newTransferPipelines(params *validatedParameters) (<-chan sources.FactoryResult, pipeline.TargetPipeline, error) { fact := newPipelinesFactory(params) - var sourcesp []pipeline.SourcePipeline + var sourcesp <-chan sources.FactoryResult var targetp pipeline.TargetPipeline var err error @@ -72,7 +72,7 @@ func (p *pipelinesFactory) newTargetPipeline() (pipeline.TargetPipeline, error) return nil, fmt.Errorf("Invalid target segment:%v", p.target) } -func (p *pipelinesFactory) newSourcePipelines() ([]pipeline.SourcePipeline, error) { +func (p *pipelinesFactory) newSourcePipelines() (<-chan sources.FactoryResult, error) { params, err := p.newSourceParams() @@ -83,18 +83,19 @@ func (p *pipelinesFactory) newSourcePipelines() ([]pipeline.SourcePipeline, erro switch p.source { case transfer.File: params := params.(sources.FileSystemSourceParams) - return sources.NewFileSystemSourcePipeline(¶ms), nil + return sources.NewFileSystemSourcePipelineFactory(¶ms), nil case transfer.HTTP: params := params.(sources.HTTPSourceParams) - return []pipeline.SourcePipeline{sources.NewHTTPSourcePipeline(params.SourceURIs, params.TargetAliases, params.SourceParams.CalculateMD5)}, nil + return sources.NewHTTPSourcePipelineFactory(params), nil case transfer.S3: params := params.(sources.S3Params) - return sources.NewS3SourcePipeline(¶ms), nil + return sources.NewS3SourcePipelineFactory(¶ms), nil case transfer.Blob: params := params.(sources.AzureBlobParams) - return sources.NewAzureBlobSourcePipeline(¶ms), nil + return sources.NewAzBlobSourcePipelineFactory(¶ms), nil case transfer.Perf: - return sources.NewPerfSourcePipeline(params.(sources.PerfSourceParams)), nil + params := params.(sources.PerfSourceParams) + return sources.NewPerfSourcePipelineFactory(params), nil } return nil, fmt.Errorf("Invalid source segment:%v", p.source) diff --git a/sources/azblobinfo.go b/sources/azblobinfo.go index 02991dd..f7417b2 100644 --- a/sources/azblobinfo.go +++ b/sources/azblobinfo.go @@ -1,7 +1,6 @@ package sources import ( - "fmt" "log" "path" "time" @@ -39,35 +38,52 @@ func newazBlobInfoProvider(params *AzureBlobParams) *azBlobInfoProvider { return &azBlobInfoProvider{params: params, azUtil: azutil} } -//getSourceInfo gets a list of SourceInfo that represent the list of azure blobs returned by the service -// based on the provided criteria (container/prefix). If the exact match flag is set, then a specific match is -// performed instead of the prefix. Marker semantics are also honored so a complete list is expected -func (b *azBlobInfoProvider) getSourceInfo() ([]pipeline.SourceInfo, error) { - var err error +func (b *azBlobInfoProvider) toSourceInfo(obj *azblob.Blob) (*pipeline.SourceInfo, error) { exp := b.params.SasExp if exp == 0 { exp = defaultSasExpHours } date := time.Now().Add(time.Duration(exp) * time.Minute).UTC() - sourceURIs := make([]pipeline.SourceInfo, 0) + + sourceURLWithSAS := b.azUtil.GetBlobURLWithReadOnlySASToken(obj.Name, date) + + targetAlias := obj.Name + if !b.params.KeepDirStructure { + targetAlias = path.Base(obj.Name) + } + + return &pipeline.SourceInfo{ + SourceName: sourceURLWithSAS.String(), + Size: uint64(*obj.Properties.ContentLength), + TargetAlias: targetAlias}, nil +} + +func (b *azBlobInfoProvider) listObjects(filter SourceFilter) <-chan ObjectListingResult { + sources := make(chan ObjectListingResult, 2) + list := make([]pipeline.SourceInfo, 0) + bsize := 0 blobCallback := func(blob *azblob.Blob, prefix string) (bool, error) { include := true if b.params.UseExactNameMatch { include = blob.Name == prefix } - if include { - sourceURLWithSAS := b.azUtil.GetBlobURLWithReadOnlySASToken(blob.Name, date) + if include && filter.IsIncluded(blob.Name) { + + si, err := b.toSourceInfo(blob) + + if err != nil { + return true, err + } + + list = append(list, *si) - targetAlias := blob.Name - if !b.params.KeepDirStructure { - targetAlias = path.Base(blob.Name) + if bsize++; bsize == b.params.FilesPerPipeline { + sources <- ObjectListingResult{Sources: list} + list = make([]pipeline.SourceInfo, 0) + bsize = 0 } - sourceURIs = append(sourceURIs, pipeline.SourceInfo{ - SourceName: sourceURLWithSAS.String(), - Size: uint64(*blob.Properties.ContentLength), - TargetAlias: targetAlias}) if b.params.UseExactNameMatch { //true, stops iteration return true, nil @@ -78,19 +94,21 @@ func (b *azBlobInfoProvider) getSourceInfo() ([]pipeline.SourceInfo, error) { return false, nil } - for _, blobName := range b.params.BlobNames { - if err = b.azUtil.IterateBlobList(blobName, blobCallback); err != nil { - return nil, err + go func() { + for _, blobName := range b.params.BlobNames { + if err := b.azUtil.IterateBlobList(blobName, blobCallback); err != nil { + sources <- ObjectListingResult{Err: err} + return + } + if bsize > 0 { + sources <- ObjectListingResult{Sources: list} + list = make([]pipeline.SourceInfo, 0) + bsize = 0 + } } - } + close(sources) - if len(sourceURIs) == 0 { - nameMatchMode := "prefix" - if b.params.UseExactNameMatch { - nameMatchMode = "name" - } - return nil, fmt.Errorf(" the %v %s did not match any blob names ", nameMatchMode, b.params.BlobNames) - } + }() - return sourceURIs, nil + return sources } diff --git a/sources/http.go b/sources/http.go index 94b7c95..1712579 100644 --- a/sources/http.go +++ b/sources/http.go @@ -30,51 +30,10 @@ type HTTPSource struct { includeMD5 bool } -type sourceHTTPPipelineFactory func(httpSource HTTPSource) (pipeline.SourcePipeline, error) -func newHTTPSource(sourceListManager objectListManager, pipelineFactory sourceHTTPPipelineFactory, numOfFilePerPipeline int, includeMD5 bool) ([]pipeline.SourcePipeline, error) { - var err error - var sourceInfos []pipeline.SourceInfo - - if sourceInfos, err = sourceListManager.getSourceInfo(); err != nil { - return nil, err - } - - if numOfFilePerPipeline <= 0 { - return nil, fmt.Errorf("Invalid operation. The number of files per batch must be greater than zero") - } - - numOfBatches := (len(sourceInfos) + numOfFilePerPipeline - 1) / numOfFilePerPipeline - pipelines := make([]pipeline.SourcePipeline, numOfBatches) - numOfFilesInBatch := numOfFilePerPipeline - filesSent := len(sourceInfos) - start := 0 - - for b := 0; b < numOfBatches; b++ { - - start = b * numOfFilesInBatch - - if filesSent < numOfFilesInBatch { - numOfFilesInBatch = filesSent - } - - httpSource := HTTPSource{Sources: sourceInfos[start : start+numOfFilesInBatch], HTTPClient: httpSourceHTTPClient, includeMD5: includeMD5} - - pipelines[b], err = pipelineFactory(httpSource) - - if err != nil { - return nil, err - } - - filesSent = filesSent - numOfFilesInBatch - } - - return pipelines, err -} - -//NewHTTPSourcePipeline creates a new instance of an HTTP source +//newHTTPSourcePipeline creates a new instance of an HTTP source //To get the file size, a HTTP HEAD request is issued and the Content-Length header is inspected. -func NewHTTPSourcePipeline(sourceURIs []string, targetAliases []string, md5 bool) pipeline.SourcePipeline { +func newHTTPSourcePipeline(sourceURIs []string, targetAliases []string, md5 bool) pipeline.SourcePipeline { setTargetAlias := len(sourceURIs) == len(targetAliases) sources := make([]pipeline.SourceInfo, len(sourceURIs)) for i := 0; i < len(sourceURIs); i++ { diff --git a/sources/multifile.go b/sources/multifile.go index cf66a3e..1b25658 100644 --- a/sources/multifile.go +++ b/sources/multifile.go @@ -50,10 +50,10 @@ type FileSystemSourceParams struct { KeepDirStructure bool } -// NewFileSystemSourcePipeline creates a new MultiFilePipeline. +// newFileSystemSourcePipeline creates a new MultiFilePipeline. // If the sourcePattern results in a single file and the targetAlias is set, the alias will be used as the target name. // Otherwise the original file name will be used. -func NewFileSystemSourcePipeline(params *FileSystemSourceParams) []pipeline.SourcePipeline { +func newFileSystemSourcePipeline(params *FileSystemSourceParams) []pipeline.SourcePipeline { var files []string var err error //get files from patterns diff --git a/sources/ostorefactory.go b/sources/ostorefactory.go deleted file mode 100644 index 46dfde7..0000000 --- a/sources/ostorefactory.go +++ /dev/null @@ -1,75 +0,0 @@ -package sources - -import ( - "fmt" - "log" - - "github.com/Azure/blobporter/pipeline" -) - -//AzureBlobSource constructs parts channel and implements data readers for Azure Blobs exposed via HTTP -type AzureBlobSource struct { - HTTPSource - container string - blobNames []string - exactNameMatch bool -} - -//NewAzureBlobSourcePipeline creates a new instance of the HTTPPipeline for Azure Blobs -func NewAzureBlobSourcePipeline(params *AzureBlobParams) []pipeline.SourcePipeline { - var err error - var azObjStorage objectListManager - azObjStorage = newazBlobInfoProvider(params) - - if params.FilesPerPipeline <= 0 { - log.Fatal(fmt.Errorf("Invalid operation. The number of files per batch must be greater than zero")) - } - - factory := func(httpSource HTTPSource) (pipeline.SourcePipeline, error) { - return &AzureBlobSource{container: params.Container, - blobNames: params.BlobNames, - HTTPSource: httpSource, - exactNameMatch: params.SourceParams.UseExactNameMatch}, nil - } - - var pipelines []pipeline.SourcePipeline - if pipelines, err = newHTTPSource(azObjStorage, factory, params.SourceParams.FilesPerPipeline, params.SourceParams.CalculateMD5); err != nil { - log.Fatal(err) - } - - return pipelines -} - -//S3Source S3 source HTTP based pipeline -type S3Source struct { - HTTPSource - exactNameMatch bool -} - -//NewS3SourcePipeline creates a new instance of the HTTPPipeline for S3 -func NewS3SourcePipeline(params *S3Params) []pipeline.SourcePipeline { - var err error - var s3ObjStorage objectListManager - s3ObjStorage, err = newS3InfoProvider(params) - - if err != nil { - log.Fatal(err) - } - - if params.FilesPerPipeline <= 0 { - log.Fatal(fmt.Errorf("Invalid operation. The number of files per batch must be greater than zero")) - } - - factory := func(httpSource HTTPSource) (pipeline.SourcePipeline, error) { - return &S3Source{ - HTTPSource: httpSource, - exactNameMatch: params.SourceParams.UseExactNameMatch}, nil - } - - var pipelines []pipeline.SourcePipeline - if pipelines, err = newHTTPSource(s3ObjStorage, factory, params.SourceParams.FilesPerPipeline, params.SourceParams.CalculateMD5); err != nil { - log.Fatal(err) - } - - return pipelines -} diff --git a/sources/perfsource.go b/sources/perfsource.go index 37cba4e..ff8cb84 100644 --- a/sources/perfsource.go +++ b/sources/perfsource.go @@ -80,8 +80,8 @@ type PerfSourceParams struct { BlockSize uint64 } -//NewPerfSourcePipeline TODO -func NewPerfSourcePipeline(params PerfSourceParams) []pipeline.SourcePipeline { +//newPerfSourcePipeline TODO +func newPerfSourcePipeline(params PerfSourceParams) []pipeline.SourcePipeline { ssps := make([]pipeline.SourcePipeline, 1) ssp := PerfSourcePipeline{ definitions: params.Definitions, diff --git a/sources/s3info.go b/sources/s3info.go index 3ab879c..4e69979 100644 --- a/sources/s3info.go +++ b/sources/s3info.go @@ -1,15 +1,14 @@ package sources import ( - "fmt" "log" "net/url" "path" + "strings" "time" - "github.com/minio/minio-go" - "github.com/Azure/blobporter/pipeline" + "github.com/minio/minio-go" ) //S3Params parameters used to create a new instance of a S3 source pipeline @@ -41,89 +40,76 @@ func newS3InfoProvider(params *S3Params) (*s3InfoProvider, error) { } -//getSourceInfo gets a list of SourceInfo that represent the list of objects returned by the service -// based on the provided criteria (bucket/prefix). If the exact match flag is set, then a specific match is -// performed instead of the prefix. Marker semantics are also honored so a complete list is expected -func (s *s3InfoProvider) getSourceInfo() ([]pipeline.SourceInfo, error) { +func (s *s3InfoProvider) toSourceInfo(obj *minio.ObjectInfo) (*pipeline.SourceInfo, error) { + exp := time.Duration(s.params.PreSignedExpMin) * time.Minute + + u, err := s.s3client.PresignedGetObject(s.params.Bucket, obj.Key, exp, url.Values{}) - objLists, err := s.getObjectLists() if err != nil { return nil, err } - exp := time.Duration(s.params.PreSignedExpMin) * time.Minute - - sourceURIs := make([]pipeline.SourceInfo, 0) - for prefix, objList := range objLists { - - for _, obj := range objList { - - include := true - - if s.params.SourceParams.UseExactNameMatch { - include = obj.Key == prefix - } + targetAlias := obj.Key + if !s.params.KeepDirStructure { + targetAlias = path.Base(obj.Key) + } - if include { + return &pipeline.SourceInfo{ + SourceName: u.String(), + Size: uint64(obj.Size), + TargetAlias: targetAlias}, nil - var u *url.URL - u, err = s.s3client.PresignedGetObject(s.params.Bucket, obj.Key, exp, url.Values{}) +} - if err != nil { - return nil, err +func (s *s3InfoProvider) listObjects(filter SourceFilter) <-chan ObjectListingResult { + sources := make(chan ObjectListingResult, 2) + go func() { + list := make([]pipeline.SourceInfo, 0) + bsize := 0 + + for _, prefix := range s.params.Prefixes { + // Create a done channel to control 'ListObjects' go routine. + done := make(chan struct{}) + defer close(done) + for object := range s.s3client.ListObjects(s.params.Bucket, prefix, true, done) { + if object.Err != nil { + sources <- ObjectListingResult{Err: object.Err} + return } + include := true - targetAlias := obj.Key - if !s.params.KeepDirStructure { - targetAlias = path.Base(obj.Key) + if s.params.SourceParams.UseExactNameMatch { + include = object.Key == prefix } - sourceURIs = append(sourceURIs, pipeline.SourceInfo{ - SourceName: u.String(), - Size: uint64(obj.Size), - TargetAlias: targetAlias}) + isfolder := strings.HasSuffix(object.Key, "/") && object.Size == 0 - } - } - } - - if len(sourceURIs) == 0 { - nameMatchMode := "prefix" - if s.params.UseExactNameMatch { - nameMatchMode = "name" - } - return nil, fmt.Errorf(" the %v %s did not match any object key(s) ", nameMatchMode, s.params.Prefixes) - } - - return sourceURIs, nil - -} -func (s *s3InfoProvider) getObjectLists() (map[string][]minio.ObjectInfo, error) { - listLength := 1 + if include && filter.IsIncluded(object.Key) && !isfolder { + si, err := s.toSourceInfo(&object) - if len(s.params.Prefixes) > 1 { - listLength = len(s.params.Prefixes) - } - - listOfLists := make(map[string][]minio.ObjectInfo, listLength) + if err != nil { + sources <- ObjectListingResult{Err: err} + return + } + list = append(list, *si) - for _, prefix := range s.params.Prefixes { - list := make([]minio.ObjectInfo, 0) + if bsize++; bsize == s.params.FilesPerPipeline { + sources <- ObjectListingResult{Sources: list} + list = make([]pipeline.SourceInfo, 0) + bsize = 0 + } + } - // Create a done channel to control 'ListObjects' go routine. - done := make(chan struct{}) + } - defer close(done) - for object := range s.s3client.ListObjects(s.params.Bucket, prefix, true, done) { - if object.Err != nil { - return nil, object.Err + if bsize > 0 { + sources <- ObjectListingResult{Sources: list} + list = make([]pipeline.SourceInfo, 0) + bsize = 0 } - list = append(list, object) } + close(sources) + }() - listOfLists[prefix] = list - } - - return listOfLists, nil - + return sources } diff --git a/sources/sourcefactory.go b/sources/sourcefactory.go new file mode 100644 index 0000000..2ac7395 --- /dev/null +++ b/sources/sourcefactory.go @@ -0,0 +1,182 @@ +package sources + +import ( + "fmt" + "path/filepath" + + "github.com/Azure/blobporter/pipeline" +) + +//AzureBlobSource constructs parts channel and implements data readers for Azure Blobs exposed via HTTP +type AzureBlobSource struct { + HTTPSource + container string + blobNames []string + exactNameMatch bool +} + +//S3Source S3 source HTTP based pipeline +type S3Source struct { + HTTPSource + exactNameMatch bool +} + +//FactoryResult TODO +type FactoryResult struct { + Source pipeline.SourcePipeline + Err error +} + +//NewHTTPSourcePipelineFactory TODO +func NewHTTPSourcePipelineFactory(params HTTPSourceParams) <-chan FactoryResult { + result := make(chan FactoryResult, 1) + defer close(result) + + result <- FactoryResult{ + Source: newHTTPSourcePipeline(params.SourceURIs, + params.TargetAliases, + params.SourceParams.CalculateMD5), + } + return result +} + +//NewPerfSourcePipelineFactory TODO +func NewPerfSourcePipelineFactory(params PerfSourceParams) <-chan FactoryResult { + result := make(chan FactoryResult, 1) + defer close(result) + + perf := newPerfSourcePipeline(params) + result <- FactoryResult{Source: perf[0]} + return result +} + +//NewS3SourcePipelineFactory returns TODO +func NewS3SourcePipelineFactory(params *S3Params) <-chan FactoryResult { + var err error + s3Provider, err := newS3InfoProvider(params) + + if err != nil { + return factoryError(err) + } + + filter := &defaultItemFilter{} + return newObjectListPipelineFactory(s3Provider, filter, params.CalculateMD5, params.FilesPerPipeline) +} + +//NewAzBlobSourcePipelineFactory TODO +func NewAzBlobSourcePipelineFactory(params *AzureBlobParams) <-chan FactoryResult { + azProvider := newazBlobInfoProvider(params) + + filter := &defaultItemFilter{} + return newObjectListPipelineFactory(azProvider, filter, params.CalculateMD5, params.FilesPerPipeline) +} + +//NewFileSystemSourcePipelineFactory TODO +func NewFileSystemSourcePipelineFactory(params *FileSystemSourceParams) <-chan FactoryResult { + var files []string + var err error + result := make(chan FactoryResult, 1) + + //get files from patterns + for i := 0; i < len(params.SourcePatterns); i++ { + var sourceFiles []string + if sourceFiles, err = filepath.Glob(params.SourcePatterns[i]); err != nil { + return factoryError(err) + } + files = append(files, sourceFiles...) + } + + if params.FilesPerPipeline <= 0 { + err = fmt.Errorf("Invalid operation. The number of files per batch must be greater than zero") + return factoryError(err) + } + + if len(files) == 0 { + err = fmt.Errorf("The pattern(s) %v did not match any files", fmt.Sprint(params.SourcePatterns)) + return factoryError(err) + } + + go func() { + numOfBatches := (len(files) + params.FilesPerPipeline - 1) / params.FilesPerPipeline + numOfFilesInBatch := params.FilesPerPipeline + filesSent := len(files) + start := 0 + for b := 0; b < numOfBatches; b++ { + var targetAlias []string + + if filesSent < numOfFilesInBatch { + numOfFilesInBatch = filesSent + } + start = b * numOfFilesInBatch + + if len(params.TargetAliases) == len(files) { + targetAlias = params.TargetAliases[start : start+numOfFilesInBatch] + } + result <- FactoryResult{Source: newMultiFilePipeline(files[start:start+numOfFilesInBatch], + targetAlias, + params.BlockSize, + params.NumOfPartitions, + params.MD5, + params.KeepDirStructure), + } + filesSent = filesSent - numOfFilesInBatch + } + close(result) + }() + + return result +} + +func newObjectListPipelineFactory(provider objectListProvider, filter SourceFilter, includeMD5 bool, filesPerPipeline int) <-chan FactoryResult { + result := make(chan FactoryResult, 1) + var err error + + if filesPerPipeline <= 0 { + err = fmt.Errorf("Invalid operation. The number of files per batch must be greater than zero") + return factoryError(err) + } + + go func() { + defer close(result) + for lst := range provider.listObjects(filter) { + if lst.Err != nil { + result <- FactoryResult{Err: lst.Err} + return + } + + httpSource := &HTTPSource{Sources: lst.Sources, + HTTPClient: httpSourceHTTPClient, + includeMD5: includeMD5, + } + result <- FactoryResult{Source: httpSource} + } + }() + + return result +} + +func factoryError(err error) chan FactoryResult { + result := make(chan FactoryResult, 1) + defer close(result) + result <- FactoryResult{Err: err} + return result +} + +type defaultItemFilter struct { +} + +//IsIncluded TODO +func (f *defaultItemFilter) IsIncluded(key string) bool { + return true +} + +//SourceFilter TODO +type SourceFilter interface { + IsIncluded(key string) bool +} + +//ObjectListingResult TODO +type ObjectListingResult struct { + Sources []pipeline.SourceInfo + Err error +} diff --git a/sources/types.go b/sources/types.go index 65c60f0..ff268f4 100644 --- a/sources/types.go +++ b/sources/types.go @@ -1,10 +1,9 @@ package sources -import "github.com/Azure/blobporter/pipeline" -//objectListManager abstracs the oerations required to get a list of sources/objects from a underlying service such as Azure Object storage and S3 -type objectListManager interface { - getSourceInfo() ([]pipeline.SourceInfo, error) + +type objectListProvider interface { + listObjects(filter SourceFilter) <-chan ObjectListingResult } //SourceParams input base parameters for blob and S3 based pipelines diff --git a/transfer/transfer_test.go b/transfer/transfer_test.go index 23a703e..6bec792 100644 --- a/transfer/transfer_test.go +++ b/transfer/transfer_test.go @@ -52,7 +52,9 @@ func TestFileToPageHTTPToPage(t *testing.T) { KeepDirStructure: true, MD5: true} - fp := sources.NewFileSystemSourcePipeline(sourceParams)[0] + fpf := <-sources.NewFileSystemSourcePipelineFactory(sourceParams) + fp := fpf.Source + tparams := targets.AzureTargetParams{ AccountName: accountName, AccountKey: accountKey, @@ -73,7 +75,15 @@ func TestFileToPageHTTPToPage(t *testing.T) { BaseBlobURL: targetBaseBlobURL} ap = targets.NewAzurePageTargetPipeline(tparams) - fp = sources.NewHTTPSourcePipeline([]string{sourceURI}, []string{sourceFile}, true) + + httpparams := sources.HTTPSourceParams{SourceParams: sources.SourceParams{ + CalculateMD5: true}, + SourceURIs:[]string{sourceURI}, + TargetAliases:[]string{sourceFile}, + } + + fpf = <- sources.NewHTTPSourcePipelineFactory(httpparams) + fp = fpf.Source tfer = NewTransfer(&fp, &ap, numOfReaders, numOfWorkers, blockSize) tfer.StartTransfer(None, delegate) tfer.WaitForCompletion() @@ -93,7 +103,8 @@ func TestFileToPage(t *testing.T) { KeepDirStructure: true, MD5: true} - fp := sources.NewFileSystemSourcePipeline(sourceParams)[0] + fpf := <-sources.NewFileSystemSourcePipelineFactory(sourceParams) + fp := fpf.Source tparams := targets.AzureTargetParams{ AccountName: accountName, AccountKey: accountKey, @@ -122,7 +133,8 @@ func TestFileToFile(t *testing.T) { KeepDirStructure: true, MD5: true} - fp := sources.NewFileSystemSourcePipeline(sourceParams)[0] + fpf := <-sources.NewFileSystemSourcePipelineFactory(sourceParams) + fp := fpf.Source ft := targets.NewFileSystemTargetPipeline(true, numOfWorkers) tfer := NewTransfer(&fp, &ft, numOfReaders, numOfWorkers, blockSize) @@ -146,7 +158,8 @@ func TestFileToBlob(t *testing.T) { KeepDirStructure: true, MD5: true} - fp := sources.NewFileSystemSourcePipeline(sourceParams)[0] + fpf := <-sources.NewFileSystemSourcePipelineFactory(sourceParams) + fp := fpf.Source tparams := targets.AzureTargetParams{ AccountName: accountName, AccountKey: accountKey, @@ -173,7 +186,8 @@ func TestFileToBlobToBlock(t *testing.T) { KeepDirStructure: true, MD5: true} - fp := sources.NewFileSystemSourcePipeline(sourceParams)[0] + fpf := <-sources.NewFileSystemSourcePipelineFactory(sourceParams) + fp := fpf.Source tparams := targets.AzureTargetParams{ AccountName: accountName, AccountKey: accountKey, @@ -200,7 +214,8 @@ func TestFileToBlobWithLargeBlocks(t *testing.T) { KeepDirStructure: true, MD5: true} - fp := sources.NewFileSystemSourcePipeline(sourceParams)[0] + fpf := <-sources.NewFileSystemSourcePipelineFactory(sourceParams) + fp := fpf.Source tparams := targets.AzureTargetParams{ AccountName: accountName, AccountKey: accountKey, @@ -229,8 +244,8 @@ func TestFilesToBlob(t *testing.T) { NumOfPartitions: numOfReaders, KeepDirStructure: true, MD5: true} - - fp := sources.NewFileSystemSourcePipeline(sourceParams)[0] + fpf := <-sources.NewFileSystemSourcePipelineFactory(sourceParams) + fp := fpf.Source tparams := targets.AzureTargetParams{ AccountName: accountName, AccountKey: accountKey, @@ -262,7 +277,8 @@ func TestFileToBlobHTTPToBlob(t *testing.T) { KeepDirStructure: true, MD5: true} - fp := sources.NewFileSystemSourcePipeline(sourceParams)[0] + fpf := <-sources.NewFileSystemSourcePipelineFactory(sourceParams) + fp := fpf.Source tparams := targets.AzureTargetParams{ AccountName: accountName, @@ -287,7 +303,14 @@ func TestFileToBlobHTTPToBlob(t *testing.T) { ap = targets.NewAzureBlockTargetPipeline(tparams) - fp = sources.NewHTTPSourcePipeline([]string{sourceURI}, []string{sourceFile}, true) + httpparams := sources.HTTPSourceParams{SourceParams: sources.SourceParams{ + CalculateMD5: true}, + SourceURIs:[]string{sourceURI}, + TargetAliases:[]string{sourceFile}, + } + + fpf = <- sources.NewHTTPSourcePipelineFactory(httpparams) + fp = fpf.Source tfer = NewTransfer(&fp, &ap, numOfReaders, numOfWorkers, blockSize) tfer.StartTransfer(None, delegate) tfer.WaitForCompletion() @@ -306,7 +329,8 @@ func TestFileToBlobHTTPToFile(t *testing.T) { KeepDirStructure: true, MD5: true} - fp := sources.NewFileSystemSourcePipeline(sourceParams)[0] + fpf := <-sources.NewFileSystemSourcePipelineFactory(sourceParams) + fp := fpf.Source tparams := targets.AzureTargetParams{ AccountName: accountName, AccountKey: accountKey, @@ -324,7 +348,16 @@ func TestFileToBlobHTTPToFile(t *testing.T) { sourceFiles[0] = sourceFile + "d" ap = targets.NewFileSystemTargetPipeline(true, numOfWorkers) - fp = sources.NewHTTPSourcePipeline([]string{sourceURI}, sourceFiles, true) + + + httpparams := sources.HTTPSourceParams{SourceParams: sources.SourceParams{ + CalculateMD5: true}, + SourceURIs:[]string{sourceURI}, + TargetAliases:[]string{sourceFile}, + } + + fpf = <- sources.NewHTTPSourcePipelineFactory(httpparams) + fp = fpf.Source tfer = NewTransfer(&fp, &ap, numOfReaders, numOfWorkers, blockSize) tfer.StartTransfer(None, delegate) tfer.WaitForCompletion() @@ -345,7 +378,8 @@ func TestFileToBlobToFile(t *testing.T) { KeepDirStructure: true, MD5: true} - fp := sources.NewFileSystemSourcePipeline(sourceParams)[0] + fpf := <-sources.NewFileSystemSourcePipelineFactory(sourceParams) + fp := fpf.Source tparams := targets.AzureTargetParams{ AccountName: accountName, AccountKey: accountKey, @@ -368,7 +402,9 @@ func TestFileToBlobToFile(t *testing.T) { CalculateMD5: true, KeepDirStructure: true, UseExactNameMatch: false}} - fp = sources.NewAzureBlobSourcePipeline(azureBlobParams)[0] + + fpf = <-sources.NewAzBlobSourcePipelineFactory(azureBlobParams) + fp = fpf.Source tfer = NewTransfer(&fp, &ap, numOfReaders, numOfWorkers, blockSize) tfer.StartTransfer(None, delegate) tfer.WaitForCompletion() @@ -390,7 +426,8 @@ func TestFileToBlobToFileWithAlias(t *testing.T) { KeepDirStructure: true, MD5: true} - fp := sources.NewFileSystemSourcePipeline(sourceParams)[0] + fpf := <-sources.NewFileSystemSourcePipelineFactory(sourceParams) + fp := fpf.Source tparams := targets.AzureTargetParams{ AccountName: accountName, AccountKey: accountKey, @@ -414,7 +451,8 @@ func TestFileToBlobToFileWithAlias(t *testing.T) { CalculateMD5: true, KeepDirStructure: true, UseExactNameMatch: true}} - fp = sources.NewAzureBlobSourcePipeline(azureBlobParams)[0] + fpf = <- sources.NewAzBlobSourcePipelineFactory(azureBlobParams) + fp = fpf.Source tfer = NewTransfer(&fp, &ap, numOfReaders, numOfWorkers, blockSize) tfer.StartTransfer(None, delegate) tfer.WaitForCompletion() From 400281c17e459aed3d1d664e6aaf6a690b78415c Mon Sep 17 00:00:00 2001 From: Jesus Aguilar <3589801+giventocode@users.noreply.github.com> Date: Sun, 4 Mar 2018 00:13:36 -0500 Subject: [PATCH 2/7] - transfers now resumable - file stats are read async to define the shape of the transfer. - refactoring. --- .gitignore | 2 + args.go | 19 +++ blobporter.go | 14 +- internal/const.go | 2 +- internal/tracker.go | 341 ++++++++++++++++++++++++++++++++++++++ internal/tracker_test.go | 78 +++++++++ pipelinefactory.go | 19 ++- sources/azblobinfo.go | 11 +- sources/fileinfo.go | 153 +++++++++++++++++ sources/http.go | 22 ++- sources/multifile.go | 131 +-------------- sources/s3info.go | 12 +- sources/sourcefactory.go | 88 ++++------ sources/types.go | 5 +- transfer/transfer.go | 46 +++-- transfer/transfer_test.go | 208 ++++++++++++----------- util/util.go | 75 +-------- 17 files changed, 840 insertions(+), 386 deletions(-) create mode 100644 internal/tracker.go create mode 100644 internal/tracker_test.go create mode 100644 sources/fileinfo.go diff --git a/.gitignore b/.gitignore index 39a633b..ce8be6b 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ *.o *.a *.so +__*.* # Folders _obj @@ -11,6 +12,7 @@ _build/windows_amd64 _wd _old .vscode +*_testdata # Architecture specific extensions/prefixes *.[568vq] diff --git a/args.go b/args.go index d94b749..a41bbc1 100644 --- a/args.go +++ b/args.go @@ -72,6 +72,7 @@ type arguments struct { readTokenExp int numberOfHandlesPerFile int //numberOfHandlesPerFile = defaultNumberOfHandlesPerFile numberOfFilesInBatch int //numberOfFilesInBatch = defaultNumberOfFilesInBatch + transferStatusPath string } //represents validated parameters @@ -95,6 +96,7 @@ type validatedParameters struct { blobSource blobParams blobTarget blobParams perfSourceDefinitions []sources.SourceDefinition + tracker *internal.TransferTracker } type s3Source struct { @@ -166,6 +168,7 @@ func (p *paramParserValidator) parseAndValidate() error { p.targetSegment = t err = p.runParseAndValidationRules( p.pvgCalculateReadersAndWorkers, + p.pvgTransferStatusPathIsPresent, p.pvgBatchLimits, p.pvgHTTPTimeOut, p.pvgDupCheck, @@ -254,6 +257,22 @@ func (p *paramParserValidator) getSourceRules() ([]parseAndValidationRule, error //************************** //Global rules.... +func (p *paramParserValidator) pvgTransferStatusPathIsPresent() error { + + if p.args.transferStatusPath != "" { + if !p.args.quietMode{ + fmt.Printf("Transfer is resumable. Transfer status file:%v \n", p.args.transferStatusPath) + } + tracker, err := internal.NewTransferTracker(p.args.transferStatusPath) + + if err != nil { + return err + } + + p.params.tracker = tracker + } + return nil +} func (p *paramParserValidator) pvgKeepDirectoryStructure() error { p.params.keepDirStructure = !p.args.removeDirStructure return nil diff --git a/blobporter.go b/blobporter.go index 1f0fc19..13a764c 100644 --- a/blobporter.go +++ b/blobporter.go @@ -45,6 +45,7 @@ func init() { numberOfHandlersPerFileMsg = "Number of open handles for concurrent reads and writes per file." numberOfFilesInBatchMsg = "Maximum number of files in a transfer.\n\tIf the number is exceeded new transfers are created" readTokenExpMsg = "Expiration in minutes of the read-only access token that will be generated to read from S3 or Azure Blob sources." + transferStatusFileMsg = "Transfer status file location. If set, blobporter will use this file to track the status of the transfer.\n\tIn case of failure and if the option is set the same status file, source files that were transferred will be skipped.\n\tIf the transfer is successful a summary will be created at then." ) flag.Usage = func() { @@ -67,6 +68,7 @@ func init() { util.PrintUsageDefaults("h", "handles_per_file", strconv.Itoa(argsUtil.args.numberOfHandlesPerFile), numberOfHandlersPerFileMsg) util.PrintUsageDefaults("x", "files_per_transfer", strconv.Itoa(argsUtil.args.numberOfFilesInBatch), numberOfFilesInBatchMsg) util.PrintUsageDefaults("o", "read_token_exp", strconv.Itoa(defaultReadTokenExp), readTokenExpMsg) + util.PrintUsageDefaults("l", "transfer_status", "", transferStatusFileMsg) } util.StringListVarAlias(&argsUtil.args.sourceURIs, "f", "source_file", "", fileMsg) @@ -88,7 +90,7 @@ func init() { util.IntVarAlias(&argsUtil.args.numberOfHandlesPerFile, "h", "handles_per_file", defaultNumberOfHandlesPerFile, numberOfHandlersPerFileMsg) util.IntVarAlias(&argsUtil.args.numberOfFilesInBatch, "x", "files_per_transfer", defaultNumberOfFilesInBatch, numberOfFilesInBatchMsg) util.IntVarAlias(&argsUtil.args.readTokenExp, "o", "read_token_exp", defaultReadTokenExp, readTokenExpMsg) - + util.StringVarAlias(&argsUtil.args.transferStatusPath, "l", "transfer_status", "", transferStatusFileMsg) } var dataTransferred uint64 @@ -144,18 +146,24 @@ func main() { sourcesInfo := sourcePipeline.Source.GetSourcesInfo() - tfer := transfer.NewTransfer(&sourcePipeline.Source, &targetPipeline, argsUtil.params.numberOfReaders, argsUtil.params.numberOfWorkers, argsUtil.params.blockSize) + tfer := transfer.NewTransfer(sourcePipeline.Source, targetPipeline, argsUtil.params.numberOfReaders, argsUtil.params.numberOfWorkers, argsUtil.params.blockSize) + tfer.SetTransferTracker(argsUtil.params.tracker) displayFilesToTransfer(sourcesInfo) pb := getProgressBarDelegate(tfer.TotalSize, argsUtil.params.quietMode) tfer.StartTransfer(argsUtil.params.dedupeLevel, pb) - tfer.WaitForCompletion() stats.AddTransferInfo(tfer.GetStats()) } + if argsUtil.params.tracker != nil { + if err = argsUtil.params.tracker.TrackTransferComplete(); err != nil { + log.Fatal(err) + } + } + stats.DisplaySummary() } diff --git a/internal/const.go b/internal/const.go index 1a6c02b..1c137a2 100644 --- a/internal/const.go +++ b/internal/const.go @@ -6,7 +6,7 @@ import ( ) //ProgramVersion blobporter version -const ProgramVersion = "0.6.10" +const ProgramVersion = "0.6.11" //HTTPClientTimeout HTTP client timeout when reading from HTTP sources and try timeout for blob storage operations. var HTTPClientTimeout = 90 diff --git a/internal/tracker.go b/internal/tracker.go new file mode 100644 index 0000000..adf6dcc --- /dev/null +++ b/internal/tracker.go @@ -0,0 +1,341 @@ +package internal + +import ( + "bufio" + "fmt" + "os" + "strconv" + "strings" + "time" +) + +type fileStatus struct { + sourcename string + size int64 + source string + status TransferStatus + tid string +} + +//TransferStatus TODO +type TransferStatus int + +const ( + //None No status avaiable. + None = iota + //Started Indicates that the file was included in the transfer. + Started + //Completed Indicates that the file was transferred succesfully. + Completed + //Ignored Indicates that the file was ignored - e.g. empty files... + Ignored +) + +func (t TransferStatus) String() string { + switch t { + case None: + return "0" + case Started: + return "1" + case Completed: + return "2" + case Ignored: + return "3" + } + + return "" +} + +func parseTransferStatus(val string) (TransferStatus, error) { + var ts int + var err error + if ts, err = strconv.Atoi(val); err != nil { + return None, fmt.Errorf("Invalid transfer status, %v, err:%v", val, err) + } + switch ts { + case None: + return None, nil + case Completed: + return Completed, nil + case Started: + return Started, nil + case Ignored: + return Ignored, nil + } + return None, fmt.Errorf("Invalid transfer status, %v", val) +} + +func newLogEntry(line string) (*fileStatus, error) { + f := fileStatus{} + + if line == summaryheader { + return nil, fmt.Errorf("Invalid transfer status file. The file contains information from a completed transfer.\nUse a new file name") + } + + tokens := strings.Split(line, "\t") + + if len(tokens) < 5 { + return nil, fmt.Errorf("Invalid log entry. Less than 5 tokens were found. Value:%v ", line) + } + + f.sourcename = tokens[1] + var size int64 + var err error + if size, err = strconv.ParseInt(tokens[3], 10, 64); err != nil { + return nil, err + } + + f.size = size + var status TransferStatus + + status, err = parseTransferStatus(tokens[2]) + + f.status = status + + f.tid = tokens[4] + + if err != nil { + return nil, err + } + + return &f, nil +} + +//Timestamp, sourcename, status, size, id +const formatString = "%s\t%s\t%s\t%v\t%s\t" + +func (f *fileStatus) toLogEntry() string { + return fmt.Sprintf(formatString, time.Now().Format(time.RFC3339Nano), f.sourcename, f.status, f.size, f.tid) +} + +func (f *fileStatus) key() string { + return fmt.Sprintf("%s%v%v", f.sourcename, f.size, f.status) +} + +type transferCompletedRequest struct { + duration time.Duration + response chan error +} + +type fileTransferredRequest struct { + fileName string + response chan error +} + +type isInLogRequest struct { + fileName string + size int64 + status TransferStatus + response chan isInLogResponse +} +type isInLogResponse struct { + inlog bool + err error +} + +//TransferTracker TODO +type TransferTracker struct { + id string + loghandle *os.File + restoredStatus map[string]fileStatus + currentStatus map[string]fileStatus + isInLogReq chan isInLogRequest + fileTransferredReq chan fileTransferredRequest + complete chan error + //transferCompletedReq chan transferCompletedRequest +} + +//NewTransferTracker TODO +func NewTransferTracker(logPath string) (*TransferTracker, error) { + var loghandle *os.File + var err error + load := true + + if loghandle, err = os.OpenFile(logPath, os.O_APPEND|os.O_RDWR, os.ModePerm); err != nil { + if os.IsNotExist(err) { + if loghandle, err = os.Create(logPath); err != nil { + return nil, err + } + load = false + } else { + return nil, err + } + } + + tt := TransferTracker{loghandle: loghandle, + restoredStatus: make(map[string]fileStatus), + currentStatus: make(map[string]fileStatus), + isInLogReq: make(chan isInLogRequest, 500), + fileTransferredReq: make(chan fileTransferredRequest, 500), + complete: make(chan error), + id: fmt.Sprintf("%v_%s", time.Now().Nanosecond(), logPath), + } + if load { + err = tt.load() + if err != nil { + return nil, err + } + } + tt.startTracker() + + return &tt, err +} + +//IsTransferredAndTrackIfNot returns true if the file was previously transferred. If not, track/log that the transferred +//was started. +func (t *TransferTracker) IsTransferredAndTrackIfNot(name string, size int64) (bool, error) { + req := isInLogRequest{fileName: name, + size: size, status: Completed, + response: make(chan isInLogResponse), + } + t.isInLogReq <- req + + resp := <-req.response + + return resp.inlog, resp.err +} + +//TrackFileTransferComplete TODO +func (t *TransferTracker) TrackFileTransferComplete(name string) error { + req := fileTransferredRequest{fileName: name, + response: make(chan error), + } + t.fileTransferredReq <- req + + return <-req.response +} + +//TrackTransferComplete TODO +func (t *TransferTracker) TrackTransferComplete() error { + t.closeChannels() + return <-t.complete +} + +func (t *TransferTracker) isInLog(name string, size int64, status TransferStatus) bool { + fs := fileStatus{sourcename: name, size: size, status: status} + + _, ok := t.restoredStatus[fs.key()] + + return ok +} + +func (t *TransferTracker) load() error { + scanner := bufio.NewScanner(t.loghandle) + for scanner.Scan() { + line := scanner.Text() + s, err := newLogEntry(line) + + if err != nil { + return err + } + + t.restoredStatus[s.key()] = *s + } + + return scanner.Err() +} + +const summaryheader = "----------------------------------------------------------" + +func (t *TransferTracker) writeSummary() error { + t.loghandle.Write([]byte(fmt.Sprintf("%v\n", summaryheader))) + t.loghandle.Write([]byte(fmt.Sprintf("Transfer Completed----------------------------------------\n"))) + t.loghandle.Write([]byte(fmt.Sprintf("Start Summary---------------------------------------------\n"))) + t.loghandle.Write([]byte(fmt.Sprintf("Last Transfer ID:%s\n", t.id))) + t.loghandle.Write([]byte(fmt.Sprintf("Date:%v\n", time.Now().Format(time.UnixDate)))) + + tsize, f, err := t.writeSummaryEntries(t.currentStatus) + if err != nil { + return err + } + tsize2, f2, err := t.writeSummaryEntries(t.restoredStatus) + if err != nil { + return err + } + + t.loghandle.Write([]byte(fmt.Sprintf("Transferred Files:%v\tTotal Size:%d\n", f+f2, tsize+tsize2))) + t.loghandle.Write([]byte(fmt.Sprintf("End Summary-----------------------------------------------\n"))) + + return t.loghandle.Close() +} +func (t *TransferTracker) writeSummaryEntries(entries map[string]fileStatus) (size int64, n int, err error) { + for _, entry := range entries { + if entry.status == Completed { + size = size + entry.size + _, err = t.loghandle.Write([]byte(fmt.Sprintf("File:%s\tSize:%v\tTID:%s\n", entry.sourcename, entry.size, entry.tid))) + if err != nil { + return + } + n++ + } + } + + return +} +func (t *TransferTracker) writeStartedEntry(name string, size int64) error { + + var status TransferStatus = Started + if size == 0 { + status = Ignored + } + + fs := fileStatus{sourcename: name, size: size, status: status, tid: t.id} + t.currentStatus[name] = fs + + line := fmt.Sprintf("%v\n", fs.toLogEntry()) + _, err := t.loghandle.Write([]byte(line)) + return err +} + +func (t *TransferTracker) writeCompleteEntry(name string) error { + fs, ok := t.currentStatus[name] + + if !ok { + return fmt.Errorf("The current status tracker is not consistent. Started entry was not found") + } + + fs.status = Completed + + line := fmt.Sprintf("%v\n", fs.toLogEntry()) + _, err := t.loghandle.Write([]byte(line)) + t.currentStatus[name] = fs + return err +} + +func (t *TransferTracker) startTracker() { + + go func() { + for { + select { + case incReq, ok := <-t.isInLogReq: + if !ok { + break + } + + inlog := t.isInLog(incReq.fileName, incReq.size, Completed) + resp := isInLogResponse{inlog: inlog} + if !inlog { + resp.err = t.writeStartedEntry(incReq.fileName, incReq.size) + } + + incReq.response <- resp + case ftReq, ok := <-t.fileTransferredReq: + if !ok { + t.complete <- t.writeSummary() + return + } + ftReq.response <- t.writeCompleteEntry(ftReq.fileName) + } + } + }() +} + +func (t *TransferTracker) closeChannels() { + close(t.isInLogReq) + close(t.fileTransferredReq) +} + +//SourceFilter TODO +type SourceFilter interface { + IsTransferredAndTrackIfNot(name string, size int64) (bool, error) +} diff --git a/internal/tracker_test.go b/internal/tracker_test.go new file mode 100644 index 0000000..044cdfe --- /dev/null +++ b/internal/tracker_test.go @@ -0,0 +1,78 @@ +package internal + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +//IMPORTANT: This test will create local files in internal_testdata/ +//make sure *_testdata is in gitinore. +const workdir = "internal_testdata" +const logfile = workdir + "/my_log" + +func TestBasicTracking(t *testing.T) { + os.Mkdir(workdir, 466) + os.Remove(logfile) + + tracker, err := NewTransferTracker(logfile) + + assert.NoError(t, err, "unexpected error") + + istrans, err := tracker.IsTransferredAndTrackIfNot("file1", 10) + assert.NoError(t, err, "unexpected error") + assert.False(t, istrans, "must be fales, i.e. not in the log") + + err = tracker.TrackFileTransferComplete("file1") + assert.NoError(t, err, "unexpected error") + + err = tracker.TrackTransferComplete() + assert.NoError(t, err, "unexpected error") + os.Remove(logfile) + +} + +func TestBasicTrackingAndResume(t *testing.T) { + os.Mkdir(workdir, 0666) + os.Remove(logfile) + + tracker, err := NewTransferTracker(logfile) + + assert.NoError(t, err, "unexpected error") + + istrans, err := tracker.IsTransferredAndTrackIfNot("file1", 10) + assert.NoError(t, err, "unexpected error") + assert.False(t, istrans, "must be false, i.e. not in the log") + + istrans, err = tracker.IsTransferredAndTrackIfNot("file2", 10) + assert.NoError(t, err, "unexpected error") + assert.False(t, istrans, "must be false, i.e. not in the log") + + //only one succeeds. + err = tracker.TrackFileTransferComplete("file1") + assert.NoError(t, err, "unexpected error") + + //closing the handle manually as the program will continue (handle to the file will exists) and need to simulate a crash + err = tracker.loghandle.Close() + assert.NoError(t, err, "unexpected error") + + //create another instance of a tracker at this point only one file should be transferred. + tracker, err = NewTransferTracker(logfile) + assert.NoError(t, err, "unexpected error") + + istrans, err = tracker.IsTransferredAndTrackIfNot("file1", 10) + assert.NoError(t, err, "unexpected error") + assert.True(t, istrans, "must be true, as this file was signaled as succesfully transferred") + + istrans, err = tracker.IsTransferredAndTrackIfNot("file2", 10) + assert.NoError(t, err, "unexpected error") + assert.False(t, istrans, "must be false, this was was not signaled as complete") + + err = tracker.TrackFileTransferComplete("file2") + assert.NoError(t, err, "unexpected error") + + err = tracker.TrackTransferComplete() + assert.NoError(t, err, "unexpected error") + os.Remove(logfile) +} diff --git a/pipelinefactory.go b/pipelinefactory.go index c27148c..52cf23c 100644 --- a/pipelinefactory.go +++ b/pipelinefactory.go @@ -105,13 +105,16 @@ func (p *pipelinesFactory) newSourceParams() (interface{}, error) { switch p.source { case transfer.File: return sources.FileSystemSourceParams{ - SourcePatterns: p.valParams.sourceURIs, - BlockSize: p.valParams.blockSize, - TargetAliases: p.valParams.targetAliases, - NumOfPartitions: p.valParams.numberOfReaders, //TODO make this more explicit by numofpartitions as param.. - MD5: p.valParams.calculateMD5, - KeepDirStructure: p.valParams.keepDirStructure, - FilesPerPipeline: p.valParams.numberOfFilesInBatch}, nil + SourcePatterns: p.valParams.sourceURIs, + BlockSize: p.valParams.blockSize, + TargetAliases: p.valParams.targetAliases, + NumOfPartitions: p.valParams.numberOfReaders, //TODO make this more explicit by numofpartitions as param.. + SourceParams: sources.SourceParams{ + Tracker: p.valParams.tracker, + CalculateMD5: p.valParams.calculateMD5, + UseExactNameMatch: p.valParams.useExactMatch, + FilesPerPipeline: p.valParams.numberOfFilesInBatch, + KeepDirStructure: p.valParams.keepDirStructure}}, nil case transfer.HTTP: return sources.HTTPSourceParams{ SourceURIs: p.valParams.sourceURIs, @@ -127,6 +130,7 @@ func (p *pipelinesFactory) newSourceParams() (interface{}, error) { AccessKey: p.valParams.s3Source.accessKey, SecretKey: p.valParams.s3Source.secretKey, SourceParams: sources.SourceParams{ + Tracker: p.valParams.tracker, CalculateMD5: p.valParams.calculateMD5, UseExactNameMatch: p.valParams.useExactMatch, FilesPerPipeline: p.valParams.numberOfFilesInBatch, @@ -141,6 +145,7 @@ func (p *pipelinesFactory) newSourceParams() (interface{}, error) { BaseBlobURL: p.valParams.blobSource.baseBlobURL, SasExp: p.valParams.blobSource.sasExpMin, SourceParams: sources.SourceParams{ + Tracker: p.valParams.tracker, CalculateMD5: p.valParams.calculateMD5, UseExactNameMatch: p.valParams.useExactMatch, FilesPerPipeline: p.valParams.numberOfFilesInBatch, diff --git a/sources/azblobinfo.go b/sources/azblobinfo.go index f7417b2..4d3ef98 100644 --- a/sources/azblobinfo.go +++ b/sources/azblobinfo.go @@ -58,7 +58,7 @@ func (b *azBlobInfoProvider) toSourceInfo(obj *azblob.Blob) (*pipeline.SourceInf TargetAlias: targetAlias}, nil } -func (b *azBlobInfoProvider) listObjects(filter SourceFilter) <-chan ObjectListingResult { +func (b *azBlobInfoProvider) listObjects(filter internal.SourceFilter) <-chan ObjectListingResult { sources := make(chan ObjectListingResult, 2) list := make([]pipeline.SourceInfo, 0) bsize := 0 @@ -68,7 +68,14 @@ func (b *azBlobInfoProvider) listObjects(filter SourceFilter) <-chan ObjectListi if b.params.UseExactNameMatch { include = blob.Name == prefix } - if include && filter.IsIncluded(blob.Name) { + + transferred, err := filter.IsTransferredAndTrackIfNot(blob.Name, int64(*blob.Properties.ContentLength)) + + if err != nil { + return true, err + } + + if include && !transferred { si, err := b.toSourceInfo(blob) diff --git a/sources/fileinfo.go b/sources/fileinfo.go new file mode 100644 index 0000000..df16c4b --- /dev/null +++ b/sources/fileinfo.go @@ -0,0 +1,153 @@ +package sources + +import ( + "fmt" + "os" + "path/filepath" +) + +type fileInfoProvider struct { + params *FileSystemSourceParams +} + +type fileInfoResponse struct { + fileInfos map[string]FileInfo + totalNumOfBlocks int64 + totalSize int64 + err error +} + +//FileInfo Contains the metadata associated with a file to be transferred +type FileInfo struct { + FileStats os.FileInfo + SourceURI string + TargetAlias string + NumOfBlocks int +} + +func newfileInfoProvider(params *FileSystemSourceParams) *fileInfoProvider { + return &fileInfoProvider{params: params} +} + +func (f *fileInfoProvider) listSourcesInfo() <-chan fileInfoResponse { + ret := make(chan fileInfoResponse, 1) + finfos := make(map[string]FileInfo) + var tblocks int64 + var tsize int64 + + walkFunc := func(path string, n int, info *FileInfo) (bool, error) { + tblocks += int64(info.NumOfBlocks) + tsize += info.FileStats.Size() + finfos[info.SourceURI] = *info + if (n+1)%f.params.FilesPerPipeline == 0 { //n counter is zero based + ret <- fileInfoResponse{fileInfos: finfos, totalNumOfBlocks: tblocks, totalSize: tsize} + finfos = make(map[string]FileInfo) + tblocks = 0 + tsize = 0 + } + + return false, nil + } + + go func() { + defer close(ret) + for _, pattern := range f.params.SourcePatterns { + if err := f.walkPattern(pattern, walkFunc); err != nil { + ret <- fileInfoResponse{err: err} + break + } + } + + if len(finfos) > 0 { + ret <- fileInfoResponse{fileInfos: finfos, totalNumOfBlocks: tblocks, totalSize: tsize} + } + }() + + return ret +} + +func (f *fileInfoProvider) newFileInfo(sourceURI string, fileStat os.FileInfo, alias string) (*FileInfo, error) { + + //directories are not allowed... + if fileStat.IsDir() { + return nil, nil + } + + if fileStat.Size() == 0 { + return nil, fmt.Errorf("Empty files are not allowed. The file %v is empty", fileStat.Name()) + } + + numOfBlocks := int(fileStat.Size()+int64(f.params.BlockSize-1)) / int(f.params.BlockSize) + + targetName := fileStat.Name() + + if f.params.KeepDirStructure { + targetName = sourceURI + } + + if alias != "" { + targetName = alias + } + return &FileInfo{FileStats: fileStat, SourceURI: sourceURI, TargetAlias: targetName, NumOfBlocks: numOfBlocks}, nil +} + +func (f *fileInfoProvider) walkPattern(pattern string, walkFunc func(path string, n int, info *FileInfo) (bool, error)) error { + matches, err := filepath.Glob(pattern) + + if err != nil { + return err + } + + if len(matches) == 0 { + return fmt.Errorf(" the pattern %v did not match any files", pattern) + } + + include := true + useAlias := len(f.params.SourcePatterns) == 1 && len(f.params.TargetAliases) == len(matches) + n := 0 + for fi := 0; fi < len(matches); fi++ { + fsinfo, err := os.Stat(matches[fi]) + + if err != nil { + return err + } + + if f.params.Tracker != nil { + var transferred bool + if transferred, err = f.params.Tracker.IsTransferredAndTrackIfNot(matches[fi], fsinfo.Size()); err != nil { + return err + } + + include = !transferred + } + + alias := "" + + if useAlias { + alias = f.params.TargetAliases[fi] + } + + if include { + info, err := f.newFileInfo(matches[fi], fsinfo, alias) + + if err != nil { + return err + } + + if info != nil { + + stop, err := walkFunc(matches[fi], n, info) + + if err != nil { + return err + } + n++ + if stop { + return nil + } + } + } + } + + return nil +} diff --git a/sources/http.go b/sources/http.go index 1712579..2f6e531 100644 --- a/sources/http.go +++ b/sources/http.go @@ -11,6 +11,7 @@ import ( "fmt" "net/http" + "net/url" "github.com/Azure/blobporter/internal" "github.com/Azure/blobporter/pipeline" @@ -30,7 +31,6 @@ type HTTPSource struct { includeMD5 bool } - //newHTTPSourcePipeline creates a new instance of an HTTP source //To get the file size, a HTTP HEAD request is issued and the Content-Length header is inspected. func newHTTPSourcePipeline(sourceURIs []string, targetAliases []string, md5 bool) pipeline.SourcePipeline { @@ -42,7 +42,7 @@ func newHTTPSourcePipeline(sourceURIs []string, targetAliases []string, md5 bool targetAlias = targetAliases[i] } else { var err error - targetAlias, err = util.GetFileNameFromURL(sourceURIs[i]) + targetAlias, err = getFileNameFromURL(sourceURIs[i]) if err != nil { log.Fatal(err) @@ -57,6 +57,24 @@ func newHTTPSourcePipeline(sourceURIs []string, targetAliases []string, md5 bool return &HTTPSource{Sources: sources, HTTPClient: httpSourceHTTPClient, includeMD5: md5} } +// returns last part of URL (filename) +func getFileNameFromURL(sourceURI string) (string, error) { + + purl, err := url.Parse(sourceURI) + + if err != nil { + return "", err + } + + parts := strings.Split(purl.Path, "/") + + if len(parts) == 0 { + return "", fmt.Errorf("Invalid URL file was not found in the path") + } + + return parts[len(parts)-1], nil +} + func getSourceSize(sourceURI string) (size int) { client := &http.Client{} resp, err := client.Head(sourceURI) diff --git a/sources/multifile.go b/sources/multifile.go index 1b25658..ca6a3a7 100644 --- a/sources/multifile.go +++ b/sources/multifile.go @@ -5,8 +5,6 @@ import ( "os" "sync" - "path/filepath" - "fmt" "io" @@ -31,132 +29,17 @@ type FileSystemSource struct { handlePool *internal.FileHandlePool } -//FileInfo Contains the metadata associated with a file to be transferred -type FileInfo struct { - FileStats *os.FileInfo - SourceURI string - TargetAlias string - NumOfBlocks int -} - //FileSystemSourceParams parameters used to create a new instance of multi-file source pipeline type FileSystemSourceParams struct { - SourcePatterns []string - BlockSize uint64 - TargetAliases []string - NumOfPartitions int - MD5 bool - FilesPerPipeline int - KeepDirStructure bool -} - -// newFileSystemSourcePipeline creates a new MultiFilePipeline. -// If the sourcePattern results in a single file and the targetAlias is set, the alias will be used as the target name. -// Otherwise the original file name will be used. -func newFileSystemSourcePipeline(params *FileSystemSourceParams) []pipeline.SourcePipeline { - var files []string - var err error - //get files from patterns - for i := 0; i < len(params.SourcePatterns); i++ { - var sourceFiles []string - if sourceFiles, err = filepath.Glob(params.SourcePatterns[i]); err != nil { - log.Fatal(err) - } - files = append(files, sourceFiles...) - } - - if params.FilesPerPipeline <= 0 { - log.Fatal(fmt.Errorf("Invalid operation. The number of files per batch must be greater than zero")) - } - - if len(files) == 0 { - log.Fatal(fmt.Errorf("The pattern(s) %v did not match any files", fmt.Sprint(params.SourcePatterns))) - } - - numOfBatches := (len(files) + params.FilesPerPipeline - 1) / params.FilesPerPipeline - pipelines := make([]pipeline.SourcePipeline, numOfBatches) - numOfFilesInBatch := params.FilesPerPipeline - filesSent := len(files) - start := 0 - for b := 0; b < numOfBatches; b++ { - var targetAlias []string - - if filesSent < numOfFilesInBatch { - numOfFilesInBatch = filesSent - } - start = b * numOfFilesInBatch - - if len(params.TargetAliases) == len(files) { - targetAlias = params.TargetAliases[start : start+numOfFilesInBatch] - } - pipelines[b] = newMultiFilePipeline(files[start:start+numOfFilesInBatch], - targetAlias, - params.BlockSize, - params.NumOfPartitions, - params.MD5, - params.KeepDirStructure) - filesSent = filesSent - numOfFilesInBatch - } - - return pipelines + SourceParams + SourcePatterns []string + BlockSize uint64 + TargetAliases []string + NumOfPartitions int } const maxNumOfHandlesPerFile int = 4 -func newMultiFilePipeline(files []string, targetAliases []string, blockSize uint64, numOfPartitions int, md5 bool, keepDirStructure bool) pipeline.SourcePipeline { - totalNumberOfBlocks := 0 - var totalSize uint64 - var err error - fileInfos := make(map[string]FileInfo) - useTargetAlias := len(targetAliases) == len(files) - for f := 0; f < len(files); f++ { - var fileStat os.FileInfo - var sName string - - if fileStat, err = os.Stat(files[f]); err != nil { - log.Fatalf("Error: %v", err) - } - - //directories are not allowed... so skipping them - if fileStat.IsDir() { - continue - } - - if fileStat.Size() == 0 { - log.Fatalf("Empty files are not allowed. The file %v is empty", files[f]) - } - numOfBlocks := int(uint64(fileStat.Size())+(blockSize-1)) / int(blockSize) - totalSize = totalSize + uint64(fileStat.Size()) - totalNumberOfBlocks = totalNumberOfBlocks + numOfBlocks - - //use the param instead of the original filename only when - //the number of targets matches the number files to transfer - if useTargetAlias { - sName = targetAliases[f] - } else { - sName = fileStat.Name() - if keepDirStructure { - sName = files[f] - } - - } - - fileInfo := FileInfo{FileStats: &fileStat, SourceURI: files[f], TargetAlias: sName, NumOfBlocks: numOfBlocks} - fileInfos[files[f]] = fileInfo - } - - handlePool := internal.NewFileHandlePool(maxNumOfHandlesPerFile, internal.Read, false) - - return &FileSystemSource{filesInfo: fileInfos, - totalNumberOfBlocks: totalNumberOfBlocks, - blockSize: blockSize, - totalSize: totalSize, - numOfPartitions: numOfPartitions, - includeMD5: md5, - handlePool: handlePool, - } -} - //ExecuteReader implements ExecuteReader from the pipeline.SourcePipeline Interface. //For each file the reader will maintain a open handle from which data will be read. // This implementation uses partitions (group of parts that can be read sequentially). @@ -234,7 +117,7 @@ func (f *FileSystemSource) GetSourcesInfo() []pipeline.SourceInfo { sources := make([]pipeline.SourceInfo, len(f.filesInfo)) var i = 0 for _, file := range f.filesInfo { - sources[i] = pipeline.SourceInfo{SourceName: file.SourceURI, TargetAlias: file.TargetAlias, Size: uint64((*file.FileStats).Size())} + sources[i] = pipeline.SourceInfo{SourceName: file.SourceURI, TargetAlias: file.TargetAlias, Size: uint64(file.FileStats.Size())} i++ } @@ -283,7 +166,7 @@ func (f *FileSystemSource) ConstructBlockInfoQueue(blockSize uint64) (partitions pindex := 0 maxpartitionNumber := 0 for _, source := range f.filesInfo { - partitions := pipeline.ConstructPartsPartition(f.numOfPartitions, (*source.FileStats).Size(), int64(blockSize), source.SourceURI, source.TargetAlias, bufferQ) + partitions := pipeline.ConstructPartsPartition(f.numOfPartitions, source.FileStats.Size(), int64(blockSize), source.SourceURI, source.TargetAlias, bufferQ) allPartitions[pindex] = partitions if len(partitions) > maxpartitionNumber { maxpartitionNumber = len(partitions) diff --git a/sources/s3info.go b/sources/s3info.go index 4e69979..03a9307 100644 --- a/sources/s3info.go +++ b/sources/s3info.go @@ -7,6 +7,7 @@ import ( "strings" "time" + "github.com/Azure/blobporter/internal" "github.com/Azure/blobporter/pipeline" "github.com/minio/minio-go" ) @@ -61,7 +62,7 @@ func (s *s3InfoProvider) toSourceInfo(obj *minio.ObjectInfo) (*pipeline.SourceIn } -func (s *s3InfoProvider) listObjects(filter SourceFilter) <-chan ObjectListingResult { +func (s *s3InfoProvider) listObjects(filter internal.SourceFilter) <-chan ObjectListingResult { sources := make(chan ObjectListingResult, 2) go func() { list := make([]pipeline.SourceInfo, 0) @@ -84,7 +85,14 @@ func (s *s3InfoProvider) listObjects(filter SourceFilter) <-chan ObjectListingRe isfolder := strings.HasSuffix(object.Key, "/") && object.Size == 0 - if include && filter.IsIncluded(object.Key) && !isfolder { + transferred, err := filter.IsTransferredAndTrackIfNot(object.Key, object.Size) + + if err != nil { + sources <- ObjectListingResult{Err: err} + return + } + + if include && !isfolder && !transferred { si, err := s.toSourceInfo(&object) if err != nil { diff --git a/sources/sourcefactory.go b/sources/sourcefactory.go index 2ac7395..1014997 100644 --- a/sources/sourcefactory.go +++ b/sources/sourcefactory.go @@ -2,8 +2,8 @@ package sources import ( "fmt" - "path/filepath" + "github.com/Azure/blobporter/internal" "github.com/Azure/blobporter/pipeline" ) @@ -59,7 +59,12 @@ func NewS3SourcePipelineFactory(params *S3Params) <-chan FactoryResult { return factoryError(err) } - filter := &defaultItemFilter{} + var filter internal.SourceFilter + filter = &defaultItemFilter{} + + if params.Tracker != nil { + filter = params.Tracker + } return newObjectListPipelineFactory(s3Provider, filter, params.CalculateMD5, params.FilesPerPipeline) } @@ -67,67 +72,49 @@ func NewS3SourcePipelineFactory(params *S3Params) <-chan FactoryResult { func NewAzBlobSourcePipelineFactory(params *AzureBlobParams) <-chan FactoryResult { azProvider := newazBlobInfoProvider(params) - filter := &defaultItemFilter{} + var filter internal.SourceFilter + filter = &defaultItemFilter{} + + if params.Tracker != nil { + filter = params.Tracker + } + return newObjectListPipelineFactory(azProvider, filter, params.CalculateMD5, params.FilesPerPipeline) } //NewFileSystemSourcePipelineFactory TODO func NewFileSystemSourcePipelineFactory(params *FileSystemSourceParams) <-chan FactoryResult { - var files []string - var err error result := make(chan FactoryResult, 1) - - //get files from patterns - for i := 0; i < len(params.SourcePatterns); i++ { - var sourceFiles []string - if sourceFiles, err = filepath.Glob(params.SourcePatterns[i]); err != nil { - return factoryError(err) - } - files = append(files, sourceFiles...) - } - - if params.FilesPerPipeline <= 0 { - err = fmt.Errorf("Invalid operation. The number of files per batch must be greater than zero") - return factoryError(err) - } - - if len(files) == 0 { - err = fmt.Errorf("The pattern(s) %v did not match any files", fmt.Sprint(params.SourcePatterns)) - return factoryError(err) - } + provider := newfileInfoProvider(params) go func() { - numOfBatches := (len(files) + params.FilesPerPipeline - 1) / params.FilesPerPipeline - numOfFilesInBatch := params.FilesPerPipeline - filesSent := len(files) - start := 0 - for b := 0; b < numOfBatches; b++ { - var targetAlias []string - - if filesSent < numOfFilesInBatch { - numOfFilesInBatch = filesSent - } - start = b * numOfFilesInBatch + defer close(result) + for fInfoResp := range provider.listSourcesInfo() { - if len(params.TargetAliases) == len(files) { - targetAlias = params.TargetAliases[start : start+numOfFilesInBatch] + if fInfoResp.err != nil { + result <- FactoryResult{Err: fInfoResp.err} + return } - result <- FactoryResult{Source: newMultiFilePipeline(files[start:start+numOfFilesInBatch], - targetAlias, - params.BlockSize, - params.NumOfPartitions, - params.MD5, - params.KeepDirStructure), + + handlePool := internal.NewFileHandlePool(maxNumOfHandlesPerFile, internal.Read, false) + result <- FactoryResult{ + Source: &FileSystemSource{filesInfo: fInfoResp.fileInfos, + totalNumberOfBlocks: int(fInfoResp.totalNumOfBlocks), + blockSize: params.BlockSize, + totalSize: uint64(fInfoResp.totalSize), + numOfPartitions: params.NumOfPartitions, + includeMD5: params.CalculateMD5, + handlePool: handlePool, + }, } - filesSent = filesSent - numOfFilesInBatch } - close(result) + return }() return result } -func newObjectListPipelineFactory(provider objectListProvider, filter SourceFilter, includeMD5 bool, filesPerPipeline int) <-chan FactoryResult { +func newObjectListPipelineFactory(provider objectListProvider, filter internal.SourceFilter, includeMD5 bool, filesPerPipeline int) <-chan FactoryResult { result := make(chan FactoryResult, 1) var err error @@ -166,13 +153,8 @@ type defaultItemFilter struct { } //IsIncluded TODO -func (f *defaultItemFilter) IsIncluded(key string) bool { - return true -} - -//SourceFilter TODO -type SourceFilter interface { - IsIncluded(key string) bool +func (f *defaultItemFilter) IsTransferredAndTrackIfNot(name string, size int64) (bool, error) { + return false, nil } //ObjectListingResult TODO diff --git a/sources/types.go b/sources/types.go index ff268f4..cd4b9ac 100644 --- a/sources/types.go +++ b/sources/types.go @@ -1,9 +1,9 @@ package sources - +import "github.com/Azure/blobporter/internal" type objectListProvider interface { - listObjects(filter SourceFilter) <-chan ObjectListingResult + listObjects(filter internal.SourceFilter) <-chan ObjectListingResult } //SourceParams input base parameters for blob and S3 based pipelines @@ -12,6 +12,7 @@ type SourceParams struct { UseExactNameMatch bool KeepDirStructure bool FilesPerPipeline int + Tracker *internal.TransferTracker } //HTTPSourceParams input parameters for HTTP pipelines diff --git a/transfer/transfer.go b/transfer/transfer.go index e21ee15..ccdfa72 100644 --- a/transfer/transfer.go +++ b/transfer/transfer.go @@ -11,6 +11,8 @@ import ( "sync/atomic" "time" + "github.com/Azure/blobporter/internal" + "github.com/Azure/blobporter/pipeline" "github.com/Azure/blobporter/util" ) @@ -266,8 +268,8 @@ type ProgressUpdate func(results pipeline.WorkerResult, committedCount int, buff //Transfer top data structure holding the state of the transfer. type Transfer struct { - SourcePipeline *pipeline.SourcePipeline - TargetPipeline *pipeline.TargetPipeline + SourcePipeline pipeline.SourcePipeline + TargetPipeline pipeline.TargetPipeline NumOfReaders int NumOfWorkers int TotalNumOfBlocks int @@ -276,6 +278,7 @@ type Transfer struct { SyncWaitGroups *WaitGroups ControlChannels *Channels TimeStats *TimeStatsInfo + tracker *internal.TransferTracker readPartsBufferSize int blockSize uint64 totalNumberOfRetries int32 @@ -309,7 +312,7 @@ const extraThreadTarget = 4 //NewTransfer creates a new Transfer, this will adjust the thread target //and initialize the channels and the wait groups for the writers, readers and the committers -func NewTransfer(source *pipeline.SourcePipeline, target *pipeline.TargetPipeline, readers int, workers int, blockSize uint64) *Transfer { +func NewTransfer(source pipeline.SourcePipeline, target pipeline.TargetPipeline, readers int, workers int, blockSize uint64) *Transfer { threadTarget := readers + workers + extraThreadTarget runtime.GOMAXPROCS(runtime.NumCPU()) @@ -321,7 +324,7 @@ func NewTransfer(source *pipeline.SourcePipeline, target *pipeline.TargetPipelin blockSize: blockSize} //validate that all sourceURIs are unique - sources := (*source).GetSourcesInfo() + sources := source.GetSourcesInfo() if s := getFirstDuplicateSource(sources); s != nil { log.Fatalf("Invalid input. You can't start a transfer with duplicate sources.\nFirst duplicate detected:%v\n", s) @@ -332,7 +335,7 @@ func NewTransfer(source *pipeline.SourcePipeline, target *pipeline.TargetPipelin t.SyncWaitGroups.Commits.Add(1) //Create buffered channels - channels.Partitions, channels.Parts, t.TotalNumOfBlocks, t.TotalSize = (*source).ConstructBlockInfoQueue(blockSize) + channels.Partitions, channels.Parts, t.TotalNumOfBlocks, t.TotalSize = source.ConstructBlockInfoQueue(blockSize) readParts := make(chan pipeline.Part, t.getReadPartsBufferSize()) results := make(chan pipeline.WorkerResult, t.TotalNumOfBlocks) @@ -382,14 +385,14 @@ func (t *Transfer) StartTransfer(dupeLevel DupeCheckLevel, progressBarDelegate P //Concurrely calls the PreProcessSourceInfo implementation of the target pipeline for each source in the transfer. func (t *Transfer) preprocessSources() { - sourcesInfo := (*t.SourcePipeline).GetSourcesInfo() + sourcesInfo := t.SourcePipeline.GetSourcesInfo() var wg sync.WaitGroup wg.Add(len(sourcesInfo)) for i := 0; i < len(sourcesInfo); i++ { go func(s *pipeline.SourceInfo, b uint64) { defer wg.Done() - if err := (*t.TargetPipeline).PreProcessSourceInfo(s, b); err != nil { + if err := t.TargetPipeline.PreProcessSourceInfo(s, b); err != nil { log.Fatal(err) } }(&sourcesInfo[i], t.blockSize) @@ -408,14 +411,13 @@ func (t *Transfer) WaitForCompletion() (time.Duration, time.Duration) { close(t.ControlChannels.Results) t.SyncWaitGroups.Commits.Wait() // Ensure all commits complete t.TimeStats.Duration = time.Now().Sub(t.TimeStats.StartTime) - return t.TimeStats.Duration, t.TimeStats.CumWriteDuration } //GetStats returns the statistics of the transfer. func (t *Transfer) GetStats() *StatInfo { return &StatInfo{ - NumberOfFiles: len((*t.SourcePipeline).GetSourcesInfo()), + NumberOfFiles: len(t.SourcePipeline.GetSourcesInfo()), Duration: t.TimeStats.Duration, CumWriteDuration: t.TimeStats.CumWriteDuration, TotalNumberOfBlocks: t.TotalNumOfBlocks, @@ -425,7 +427,7 @@ func (t *Transfer) GetStats() *StatInfo { // StartWorkers creates and starts the set of Workers to send data blocks // from the to the target. Workers are started after waiting workerDelayStarTime. -func (t *Transfer) startWorkers(workerQ chan pipeline.Part, resultQ chan pipeline.WorkerResult, numOfWorkers int, wg *sync.WaitGroup, d DupeCheckLevel, target *pipeline.TargetPipeline) { +func (t *Transfer) startWorkers(workerQ chan pipeline.Part, resultQ chan pipeline.WorkerResult, numOfWorkers int, wg *sync.WaitGroup, d DupeCheckLevel, target pipeline.TargetPipeline) { for w := 0; w < numOfWorkers; w++ { worker := newWorker(w, workerQ, resultQ, wg, d) go worker.startWorker(target) @@ -434,7 +436,7 @@ func (t *Transfer) startWorkers(workerQ chan pipeline.Part, resultQ chan pipelin //ProcessAndCommitResults reads from the results channel and calls the target's implementations of the ProcessWrittenPart // and CommitList, if the last part is received. The update delegate is called as well. -func (t *Transfer) processAndCommitResults(resultQ chan pipeline.WorkerResult, update ProgressUpdate, target *pipeline.TargetPipeline, commitWg *sync.WaitGroup) { +func (t *Transfer) processAndCommitResults(resultQ chan pipeline.WorkerResult, update ProgressUpdate, target pipeline.TargetPipeline, commitWg *sync.WaitGroup) { lists := make(map[string]pipeline.TargetCommittedListInfo) blocksProcessed := make(map[string]int) @@ -456,7 +458,7 @@ func (t *Transfer) processAndCommitResults(resultQ chan pipeline.WorkerResult, u list = lists[result.SourceURI] numOfBlocks = blocksProcessed[result.SourceURI] - if requeue, err = (*target).ProcessWrittenPart(&result, &list); err == nil { + if requeue, err = target.ProcessWrittenPart(&result, &list); err == nil { lists[result.SourceURI] = list if requeue { resultQ <- result @@ -471,10 +473,15 @@ func (t *Transfer) processAndCommitResults(resultQ chan pipeline.WorkerResult, u } if numOfBlocks == result.NumberOfBlocks { - if _, err := (*target).CommitList(&list, numOfBlocks, result.TargetName); err != nil { + if _, err := target.CommitList(&list, numOfBlocks, result.TargetName); err != nil { log.Fatal(err) } committedCount++ + if t.tracker != nil { + if err := t.tracker.TrackFileTransferComplete(result.TargetName); err != nil { + log.Fatal(err) + } + } } workerBufferLevel = int(float64(len(t.ControlChannels.ReadParts)) / float64(t.getReadPartsBufferSize()) * 100) @@ -483,23 +490,28 @@ func (t *Transfer) processAndCommitResults(resultQ chan pipeline.WorkerResult, u } } +//SetTransferTracker TODO +func (t *Transfer) SetTransferTracker(tracker *internal.TransferTracker) { + t.tracker = tracker +} + // StartReaders starts 'n' readers. Each reader is a routine that executes the source's implementations of ExecuteReader. -func (t *Transfer) startReaders(partitionsQ chan pipeline.PartsPartition, partsQ chan pipeline.Part, workerQ chan pipeline.Part, numberOfReaders int, wg *sync.WaitGroup, pipeline *pipeline.SourcePipeline) { +func (t *Transfer) startReaders(partitionsQ chan pipeline.PartsPartition, partsQ chan pipeline.Part, workerQ chan pipeline.Part, numberOfReaders int, wg *sync.WaitGroup, pipeline pipeline.SourcePipeline) { for i := 0; i < numberOfReaders; i++ { - go (*pipeline).ExecuteReader(partitionsQ, partsQ, workerQ, i, wg) + go pipeline.ExecuteReader(partitionsQ, partsQ, workerQ, i, wg) } } // startWorker starts a worker that reads from the worker queue channel. Which contains the read parts from the source. // Calls the target's WritePart implementation and sends the result to the results channel. -func (w *Worker) startWorker(target *pipeline.TargetPipeline) { +func (w *Worker) startWorker(target pipeline.TargetPipeline) { var tb pipeline.Part var ok bool var duration time.Duration var startTime time.Time var retries int var err error - var t = (*target) + var t = target defer w.Wg.Done() for { tb, ok = <-w.WorkerQueue diff --git a/transfer/transfer_test.go b/transfer/transfer_test.go index 6bec792..6960af3 100644 --- a/transfer/transfer_test.go +++ b/transfer/transfer_test.go @@ -21,7 +21,7 @@ import ( //******** //IMPORTANT: // -- Tests require a valid storage account and env variables set up accordingly. -// -- Tests create a working directory named _wd and temp files under it. Plase make sure the working directory is in .gitignore +// -- Tests create a working directory named transfer_testdata and temp files under it. Plase make sure the working directory is in .gitignore //******** var sourceFiles = make([]string, 1) var accountName = os.Getenv("ACCOUNT_NAME") @@ -36,7 +36,7 @@ var targetBaseBlobURL = "" const ( containerName1 = "bptest" containerName2 = "bphttptest" - tempDir = "_wd" + tempDir = "transfer_testdata" ) func TestFileToPageHTTPToPage(t *testing.T) { @@ -45,12 +45,13 @@ func TestFileToPageHTTPToPage(t *testing.T) { var sourceFile = createPageFile("tb", 1) sourceParams := &sources.FileSystemSourceParams{ - SourcePatterns: []string{sourceFile}, - BlockSize: blockSize, - FilesPerPipeline: filesPerPipeline, - NumOfPartitions: numOfReaders, - KeepDirStructure: true, - MD5: true} + SourcePatterns: []string{sourceFile}, + BlockSize: blockSize, + NumOfPartitions: numOfReaders, + SourceParams: sources.SourceParams{ + FilesPerPipeline: filesPerPipeline, + KeepDirStructure: true, + CalculateMD5: true}} fpf := <-sources.NewFileSystemSourcePipelineFactory(sourceParams) fp := fpf.Source @@ -62,7 +63,7 @@ func TestFileToPageHTTPToPage(t *testing.T) { BaseBlobURL: targetBaseBlobURL} ap := targets.NewAzurePageTargetPipeline(tparams) - tfer := NewTransfer(&fp, &ap, numOfReaders, numOfWorkers, blockSize) + tfer := NewTransfer(fp, ap, numOfReaders, numOfWorkers, blockSize) tfer.StartTransfer(None, delegate) tfer.WaitForCompletion() sourceURI, err := createSasTokenURL(sourceFile, container) @@ -75,16 +76,16 @@ func TestFileToPageHTTPToPage(t *testing.T) { BaseBlobURL: targetBaseBlobURL} ap = targets.NewAzurePageTargetPipeline(tparams) - + httpparams := sources.HTTPSourceParams{SourceParams: sources.SourceParams{ CalculateMD5: true}, - SourceURIs:[]string{sourceURI}, - TargetAliases:[]string{sourceFile}, + SourceURIs: []string{sourceURI}, + TargetAliases: []string{sourceFile}, } - fpf = <- sources.NewHTTPSourcePipelineFactory(httpparams) + fpf = <-sources.NewHTTPSourcePipelineFactory(httpparams) fp = fpf.Source - tfer = NewTransfer(&fp, &ap, numOfReaders, numOfWorkers, blockSize) + tfer = NewTransfer(fp, ap, numOfReaders, numOfWorkers, blockSize) tfer.StartTransfer(None, delegate) tfer.WaitForCompletion() @@ -96,12 +97,13 @@ func TestFileToPage(t *testing.T) { container, _ := getContainers() sourceParams := &sources.FileSystemSourceParams{ - SourcePatterns: []string{sourceFile}, - BlockSize: blockSize, - FilesPerPipeline: filesPerPipeline, - NumOfPartitions: numOfReaders, - KeepDirStructure: true, - MD5: true} + SourcePatterns: []string{sourceFile}, + BlockSize: blockSize, + NumOfPartitions: numOfReaders, + SourceParams: sources.SourceParams{ + FilesPerPipeline: filesPerPipeline, + KeepDirStructure: true, + CalculateMD5: true}} fpf := <-sources.NewFileSystemSourcePipelineFactory(sourceParams) fp := fpf.Source @@ -113,7 +115,7 @@ func TestFileToPage(t *testing.T) { pt := targets.NewAzurePageTargetPipeline(tparams) - tfer := NewTransfer(&fp, &pt, numOfReaders, numOfWorkers, blockSize) + tfer := NewTransfer(fp, pt, numOfReaders, numOfWorkers, blockSize) tfer.StartTransfer(None, delegate) tfer.WaitForCompletion() @@ -125,19 +127,20 @@ func TestFileToFile(t *testing.T) { var destFile = sourceFile + "d" sourceParams := &sources.FileSystemSourceParams{ - SourcePatterns: []string{sourceFile}, - BlockSize: blockSize, - FilesPerPipeline: filesPerPipeline, - NumOfPartitions: numOfReaders, - TargetAliases: []string{destFile}, - KeepDirStructure: true, - MD5: true} + SourcePatterns: []string{sourceFile}, + BlockSize: blockSize, + NumOfPartitions: numOfReaders, + TargetAliases: []string{destFile}, + SourceParams: sources.SourceParams{ + FilesPerPipeline: filesPerPipeline, + KeepDirStructure: true, + CalculateMD5: true}} fpf := <-sources.NewFileSystemSourcePipelineFactory(sourceParams) fp := fpf.Source ft := targets.NewFileSystemTargetPipeline(true, numOfWorkers) - tfer := NewTransfer(&fp, &ft, numOfReaders, numOfWorkers, blockSize) + tfer := NewTransfer(fp, ft, numOfReaders, numOfWorkers, blockSize) tfer.StartTransfer(None, delegate) tfer.WaitForCompletion() @@ -151,12 +154,13 @@ func TestFileToBlob(t *testing.T) { var sourceFile = createFile("tb", 1) sourceParams := &sources.FileSystemSourceParams{ - SourcePatterns: []string{sourceFile}, - BlockSize: blockSize, - FilesPerPipeline: filesPerPipeline, - NumOfPartitions: numOfReaders, - KeepDirStructure: true, - MD5: true} + SourcePatterns: []string{sourceFile}, + BlockSize: blockSize, + NumOfPartitions: numOfReaders, + SourceParams: sources.SourceParams{ + FilesPerPipeline: filesPerPipeline, + KeepDirStructure: true, + CalculateMD5: true}} fpf := <-sources.NewFileSystemSourcePipelineFactory(sourceParams) fp := fpf.Source @@ -167,7 +171,7 @@ func TestFileToBlob(t *testing.T) { ap := targets.NewAzureBlockTargetPipeline(tparams) - tfer := NewTransfer(&fp, &ap, numOfReaders, numOfWorkers, blockSize) + tfer := NewTransfer(fp, ap, numOfReaders, numOfWorkers, blockSize) tfer.StartTransfer(None, delegate) tfer.WaitForCompletion() @@ -179,12 +183,13 @@ func TestFileToBlobToBlock(t *testing.T) { var sourceFile = createFile("tb", 1) sourceParams := &sources.FileSystemSourceParams{ - SourcePatterns: []string{sourceFile}, - BlockSize: blockSize, - FilesPerPipeline: filesPerPipeline, - NumOfPartitions: numOfReaders, - KeepDirStructure: true, - MD5: true} + SourcePatterns: []string{sourceFile}, + BlockSize: blockSize, + NumOfPartitions: numOfReaders, + SourceParams: sources.SourceParams{ + FilesPerPipeline: filesPerPipeline, + KeepDirStructure: true, + CalculateMD5: true}} fpf := <-sources.NewFileSystemSourcePipelineFactory(sourceParams) fp := fpf.Source @@ -194,7 +199,7 @@ func TestFileToBlobToBlock(t *testing.T) { Container: container} ap := targets.NewAzureBlockTargetPipeline(tparams) - tfer := NewTransfer(&fp, &ap, numOfReaders, numOfWorkers, blockSize) + tfer := NewTransfer(fp, ap, numOfReaders, numOfWorkers, blockSize) tfer.StartTransfer(None, delegate) tfer.WaitForCompletion() @@ -207,12 +212,13 @@ func TestFileToBlobWithLargeBlocks(t *testing.T) { bsize := uint64(16 * util.MiByte) sourceParams := &sources.FileSystemSourceParams{ - SourcePatterns: []string{sourceFile}, - BlockSize: blockSize, - FilesPerPipeline: filesPerPipeline, - NumOfPartitions: numOfReaders, - KeepDirStructure: true, - MD5: true} + SourcePatterns: []string{sourceFile}, + BlockSize: blockSize, + NumOfPartitions: numOfReaders, + SourceParams: sources.SourceParams{ + FilesPerPipeline: filesPerPipeline, + KeepDirStructure: true, + CalculateMD5: true}} fpf := <-sources.NewFileSystemSourcePipelineFactory(sourceParams) fp := fpf.Source @@ -223,7 +229,7 @@ func TestFileToBlobWithLargeBlocks(t *testing.T) { ap := targets.NewAzureBlockTargetPipeline(tparams) - tfer := NewTransfer(&fp, &ap, numOfReaders, numOfWorkers, bsize) + tfer := NewTransfer(fp, ap, numOfReaders, numOfWorkers, bsize) tfer.StartTransfer(None, delegate) tfer.WaitForCompletion() @@ -238,12 +244,13 @@ func TestFilesToBlob(t *testing.T) { var sf4 = createFile("tbm", 1) sourceParams := &sources.FileSystemSourceParams{ - SourcePatterns: []string{tempDir + "/tbm*"}, - BlockSize: blockSize, - FilesPerPipeline: filesPerPipeline, - NumOfPartitions: numOfReaders, - KeepDirStructure: true, - MD5: true} + SourcePatterns: []string{tempDir + "/tbm*"}, + BlockSize: blockSize, + NumOfPartitions: numOfReaders, + SourceParams: sources.SourceParams{ + FilesPerPipeline: filesPerPipeline, + KeepDirStructure: true, + CalculateMD5: true}} fpf := <-sources.NewFileSystemSourcePipelineFactory(sourceParams) fp := fpf.Source tparams := targets.AzureTargetParams{ @@ -254,7 +261,7 @@ func TestFilesToBlob(t *testing.T) { ap := targets.NewAzureBlockTargetPipeline(tparams) - tfer := NewTransfer(&fp, &ap, numOfReaders, numOfWorkers, blockSize) + tfer := NewTransfer(fp, ap, numOfReaders, numOfWorkers, blockSize) tfer.StartTransfer(None, delegate) tfer.WaitForCompletion() @@ -270,12 +277,13 @@ func TestFileToBlobHTTPToBlob(t *testing.T) { var sourceFile = createFile("tb", 1) sourceParams := &sources.FileSystemSourceParams{ - SourcePatterns: []string{sourceFile}, - BlockSize: blockSize, - FilesPerPipeline: filesPerPipeline, - NumOfPartitions: numOfReaders, - KeepDirStructure: true, - MD5: true} + SourcePatterns: []string{sourceFile}, + BlockSize: blockSize, + NumOfPartitions: numOfReaders, + SourceParams: sources.SourceParams{ + FilesPerPipeline: filesPerPipeline, + KeepDirStructure: true, + CalculateMD5: true}} fpf := <-sources.NewFileSystemSourcePipelineFactory(sourceParams) fp := fpf.Source @@ -288,7 +296,7 @@ func TestFileToBlobHTTPToBlob(t *testing.T) { ap := targets.NewAzureBlockTargetPipeline(tparams) - tfer := NewTransfer(&fp, &ap, numOfReaders, numOfWorkers, blockSize) + tfer := NewTransfer(fp, ap, numOfReaders, numOfWorkers, blockSize) tfer.StartTransfer(None, delegate) tfer.WaitForCompletion() sourceURI, err := createSasTokenURL(sourceFile, container) @@ -305,13 +313,13 @@ func TestFileToBlobHTTPToBlob(t *testing.T) { httpparams := sources.HTTPSourceParams{SourceParams: sources.SourceParams{ CalculateMD5: true}, - SourceURIs:[]string{sourceURI}, - TargetAliases:[]string{sourceFile}, + SourceURIs: []string{sourceURI}, + TargetAliases: []string{sourceFile}, } - fpf = <- sources.NewHTTPSourcePipelineFactory(httpparams) + fpf = <-sources.NewHTTPSourcePipelineFactory(httpparams) fp = fpf.Source - tfer = NewTransfer(&fp, &ap, numOfReaders, numOfWorkers, blockSize) + tfer = NewTransfer(fp, ap, numOfReaders, numOfWorkers, blockSize) tfer.StartTransfer(None, delegate) tfer.WaitForCompletion() @@ -322,12 +330,13 @@ func TestFileToBlobHTTPToFile(t *testing.T) { var sourceFile = createFile("tb", 1) sourceParams := &sources.FileSystemSourceParams{ - SourcePatterns: []string{sourceFile}, - BlockSize: blockSize, - FilesPerPipeline: filesPerPipeline, - NumOfPartitions: numOfReaders, - KeepDirStructure: true, - MD5: true} + SourcePatterns: []string{sourceFile}, + BlockSize: blockSize, + NumOfPartitions: numOfReaders, + SourceParams: sources.SourceParams{ + FilesPerPipeline: filesPerPipeline, + KeepDirStructure: true, + CalculateMD5: true}} fpf := <-sources.NewFileSystemSourcePipelineFactory(sourceParams) fp := fpf.Source @@ -339,7 +348,7 @@ func TestFileToBlobHTTPToFile(t *testing.T) { ap := targets.NewAzureBlockTargetPipeline(tparams) - tfer := NewTransfer(&fp, &ap, numOfReaders, numOfWorkers, blockSize) + tfer := NewTransfer(fp, ap, numOfReaders, numOfWorkers, blockSize) tfer.StartTransfer(None, delegate) tfer.WaitForCompletion() sourceURI, err := createSasTokenURL(sourceFile, container) @@ -349,16 +358,15 @@ func TestFileToBlobHTTPToFile(t *testing.T) { sourceFiles[0] = sourceFile + "d" ap = targets.NewFileSystemTargetPipeline(true, numOfWorkers) - httpparams := sources.HTTPSourceParams{SourceParams: sources.SourceParams{ CalculateMD5: true}, - SourceURIs:[]string{sourceURI}, - TargetAliases:[]string{sourceFile}, + SourceURIs: []string{sourceURI}, + TargetAliases: []string{sourceFile}, } - fpf = <- sources.NewHTTPSourcePipelineFactory(httpparams) + fpf = <-sources.NewHTTPSourcePipelineFactory(httpparams) fp = fpf.Source - tfer = NewTransfer(&fp, &ap, numOfReaders, numOfWorkers, blockSize) + tfer = NewTransfer(fp, ap, numOfReaders, numOfWorkers, blockSize) tfer.StartTransfer(None, delegate) tfer.WaitForCompletion() @@ -371,12 +379,13 @@ func TestFileToBlobToFile(t *testing.T) { var sourceFile = createFile("tb", 1) sourceParams := &sources.FileSystemSourceParams{ - SourcePatterns: []string{sourceFile}, - BlockSize: blockSize, - FilesPerPipeline: filesPerPipeline, - NumOfPartitions: numOfReaders, - KeepDirStructure: true, - MD5: true} + SourcePatterns: []string{sourceFile}, + BlockSize: blockSize, + NumOfPartitions: numOfReaders, + SourceParams: sources.SourceParams{ + FilesPerPipeline: filesPerPipeline, + KeepDirStructure: true, + CalculateMD5: true}} fpf := <-sources.NewFileSystemSourcePipelineFactory(sourceParams) fp := fpf.Source @@ -387,7 +396,7 @@ func TestFileToBlobToFile(t *testing.T) { BaseBlobURL: ""} ap := targets.NewAzureBlockTargetPipeline(tparams) - tfer := NewTransfer(&fp, &ap, numOfReaders, numOfWorkers, blockSize) + tfer := NewTransfer(fp, ap, numOfReaders, numOfWorkers, blockSize) tfer.StartTransfer(None, delegate) tfer.WaitForCompletion() @@ -402,10 +411,10 @@ func TestFileToBlobToFile(t *testing.T) { CalculateMD5: true, KeepDirStructure: true, UseExactNameMatch: false}} - + fpf = <-sources.NewAzBlobSourcePipelineFactory(azureBlobParams) fp = fpf.Source - tfer = NewTransfer(&fp, &ap, numOfReaders, numOfWorkers, blockSize) + tfer = NewTransfer(fp, ap, numOfReaders, numOfWorkers, blockSize) tfer.StartTransfer(None, delegate) tfer.WaitForCompletion() @@ -418,13 +427,14 @@ func TestFileToBlobToFileWithAlias(t *testing.T) { var alias = sourceFile + "alias" sourceParams := &sources.FileSystemSourceParams{ - SourcePatterns: []string{sourceFile}, - BlockSize: blockSize, - TargetAliases: []string{alias}, - FilesPerPipeline: filesPerPipeline, - NumOfPartitions: numOfReaders, - KeepDirStructure: true, - MD5: true} + SourcePatterns: []string{sourceFile}, + BlockSize: blockSize, + TargetAliases: []string{alias}, + NumOfPartitions: numOfReaders, + SourceParams: sources.SourceParams{ + FilesPerPipeline: filesPerPipeline, + KeepDirStructure: true, + CalculateMD5: true}} fpf := <-sources.NewFileSystemSourcePipelineFactory(sourceParams) fp := fpf.Source @@ -436,7 +446,7 @@ func TestFileToBlobToFileWithAlias(t *testing.T) { ap := targets.NewAzureBlockTargetPipeline(tparams) - tfer := NewTransfer(&fp, &ap, numOfReaders, numOfWorkers, blockSize) + tfer := NewTransfer(fp, ap, numOfReaders, numOfWorkers, blockSize) tfer.StartTransfer(None, delegate) tfer.WaitForCompletion() @@ -451,9 +461,9 @@ func TestFileToBlobToFileWithAlias(t *testing.T) { CalculateMD5: true, KeepDirStructure: true, UseExactNameMatch: true}} - fpf = <- sources.NewAzBlobSourcePipelineFactory(azureBlobParams) + fpf = <-sources.NewAzBlobSourcePipelineFactory(azureBlobParams) fp = fpf.Source - tfer = NewTransfer(&fp, &ap, numOfReaders, numOfWorkers, blockSize) + tfer = NewTransfer(fp, ap, numOfReaders, numOfWorkers, blockSize) tfer.StartTransfer(None, delegate) tfer.WaitForCompletion() diff --git a/util/util.go b/util/util.go index 8f1097e..f9ac1cf 100644 --- a/util/util.go +++ b/util/util.go @@ -5,12 +5,9 @@ import ( "fmt" "log" "os" - "regexp" "strconv" "strings" "time" - - "net/url" ) // Verbose mode active? @@ -202,62 +199,10 @@ func RetriableOperation(operation func(r int) error) (duration time.Duration, st } } -//AskUser places a yes/no question to the user provided by the stdin -func AskUser(question string) bool { - fmt.Printf(question) - for { - var input string - n, err := fmt.Scanln(&input) - if n < 1 || err != nil { - fmt.Println("invalid input") - } - input = strings.ToLower(input) - switch input { - case "y": - return true - case "n": - return false - default: - fmt.Printf("Invalid response.\n") - fmt.Printf(question) - } - } -} - -//isValidContainerName is true if the name of the container is valid, false if not -func isValidContainerName(name string) bool { - if len(name) < 3 { - return false - } - expr := "^[a-z0-9]+([-]?[a-z0-9]){1,63}$" - valid := regexp.MustCompile(expr) - resp := valid.MatchString(name) - if !resp { - fmt.Printf("The name provided for the container is invalid, it must conform the following rules:\n") - fmt.Printf("1. Container names must start with a letter or number, and can contain only letters, numbers, and the dash (-) character.\n") - fmt.Printf("2. Every dash (-) character must be immediately preceded and followed by a letter or number; consecutive dashes are not permitted in container names.\n") - fmt.Printf("3. All letters in a container name must be lowercase.\n") - fmt.Printf("4. Container names must be from 3 through 63 characters long.\n") - } - return resp -} - func handleExceededRetries(err error) { - errMsg := fmt.Sprintf("The number of retries has exceeded the maximum allowed.\nError: %v\nSuggestion:%v\n", err.Error(), getSuggestion(err)) + errMsg := fmt.Sprintf("The number of retries has exceeded the maximum allowed.\nError: %v\n", err.Error()) log.Fatal(errMsg) } -func getSuggestion(err error) string { - switch { - case strings.Contains(err.Error(), "ErrorMessage=The specified blob or block content is invalid"): - return "Try using a different container or upload and then delete a small blob with the same name." - case strings.Contains(err.Error(), "Client.Timeout"): - return "Try increasing the timeout using the -s option or reducing the number of workers and readers, options: -r and -g" - case strings.Contains(err.Error(), "too many open files"): - return "Try reducing the number of sources or batch size" - default: - return "" - } -} //PrintfIfDebug TODO func PrintfIfDebug(format string, values ...interface{}) { @@ -266,21 +211,3 @@ func PrintfIfDebug(format string, values ...interface{}) { fmt.Printf("%v\t%s\n", time.Now(), msg) } } - -//GetFileNameFromURL returns last part of URL (filename) -func GetFileNameFromURL(sourceURI string) (string, error) { - - purl, err := url.Parse(sourceURI) - - if err != nil { - return "", err - } - - parts := strings.Split(purl.Path, "/") - - if len(parts) == 0 { - return "", fmt.Errorf("Invalid URL file was not found in the path") - } - - return parts[len(parts)-1], nil -} From 1877f3b3da619f9a6bdcc75d9c269d1ee87562ec Mon Sep 17 00:00:00 2001 From: Jesus Aguilar <3589801+giventocode@users.noreply.github.com> Date: Sun, 4 Mar 2018 23:19:42 -0500 Subject: [PATCH 3/7] - doc updates --- args.go | 8 +++-- docs/perfmode.rst | 19 +++--------- docs/resumable_transfers.rst | 59 ++++++++++++++++++++++++++++++++++++ sources/s3info.go | 1 + 4 files changed, 71 insertions(+), 16 deletions(-) create mode 100644 docs/resumable_transfers.rst diff --git a/args.go b/args.go index a41bbc1..530d76e 100644 --- a/args.go +++ b/args.go @@ -522,7 +522,7 @@ func (p *paramParserValidator) pvSourceInfoForS3IsReq() error { burl, err := url.Parse(p.params.sourceURIs[0]) if err != nil { - return fmt.Errorf("Invalid S3 endpoint URL. Parsing error: %v.\nThe format is s3://[END_POINT]/[BUCKET]/[OBJECT]", err) + return fmt.Errorf("Invalid S3 endpoint URL. Parsing error: %v.\nThe format is s3://[END_POINT]/[BUCKET]/[PREFIX]", err) } p.params.s3Source.endpoint = burl.Hostname() @@ -533,10 +533,14 @@ func (p *paramParserValidator) pvSourceInfoForS3IsReq() error { segments := strings.Split(burl.Path, "/") + if len(segments) < 2 { + return fmt.Errorf("Invalid S3 endpoint URL. Bucket not specified. The format is s3://[END_POINT]/[BUCKET]/[PREFIX]") + } + p.params.s3Source.bucket = segments[1] if p.params.s3Source.bucket == "" { - return fmt.Errorf("Invalid source S3 URI. Bucket name could be parsed") + return fmt.Errorf("Invalid source S3 URI. Bucket name could not be parsed") } prefix := "" diff --git a/docs/perfmode.rst b/docs/perfmode.rst index 47df12b..4d6d3fd 100644 --- a/docs/perfmode.rst +++ b/docs/perfmode.rst @@ -1,14 +1,7 @@ Performance Mode ====================================== - -If you want to maximize performance, and your source and target are public HTTP based end-points (Blob, S3, and HTTP), running the transfer in a high bandwidth environment such as a VM on the cloud, is strongly recommended. This recommendation comes from the fact that blob to blob, S3 to blob or HTTP to blob transfers are bidirectional where BlobPorter downloads the data (without writing to disk) and uploads it as it is received. - -When running in the cloud, consider the region where the transfer VM ( where BlobPorter will be running), will be deployed. When possible, deploy the transfer VM in the same the same region as the target of the transfer. Running in the same region as the target minimizes the transfer costs (egress from the VM to the target storage account) and the network performance impact (lower latency) for the upload operation. - -For downloads or uploads of multiple or large files the disk i/o could be the constraining resource that slows down the transfer. And often identifying if this is the case, is a cumbersome process. But if done, it could lead to informed decisions about the environment where BlobPorter runs. - -To help with this indentification process, BlobPorter has a performance mode that uploads random data generated in memory and measures the performance of the operation without the impact of disk i/o. -The performance mode for uploads could help you identify the potential upper limit of throughput that the network and the target storage account can provide. +BlobPorter has a performance mode that uploads random data generated in memory and measures the performance of the operation without the impact of disk i/o. +The performance mode for uploads could help you identify the potential upper limit of throughput that the network and the target storage account can provide. For example, the following command will upload 10 x 10GB files to a storage account. @@ -24,19 +17,17 @@ blobporter -f "1GB:10" -c perft -t perf-blockblob -g 20 Similarly, for downloads, you can simulate downloading data from a storage account without writing to disk. This mode could also help you fine-tune the number of readers (-r option) and get an idea of the maximum download throughput. -The following command will download the data we previously uploaded. +The following command downloads the data previously uploaded. ``` export SRC_ACCOUNT_KEY=$ACCOUNT_KEY blobporter -f "https://myaccount.blob.core.windows.net/perft" -t blob-perf ``` -Then you can try downloading to disk. +Then you can download to disk. ``` blobporter -c perft -t blob-file ``` -If the performance difference is significant then you can conclude that disk i/o is the bottleneck, at which point you can consider an SSD backed VM. - - +The performance difference will you a measurement of the impact of disk i/o. \ No newline at end of file diff --git a/docs/resumable_transfers.rst b/docs/resumable_transfers.rst new file mode 100644 index 0000000..31c7b08 --- /dev/null +++ b/docs/resumable_transfers.rst @@ -0,0 +1,59 @@ +Resumable Transfers +====================================== +BlobPorter supports resumable transfers. To enable this feature you need to set the -l option with a path to the transfer status file. + +``` +blobporter -f "manyfiles/*" -c "many" -l mylog +``` + +The status transfer file contains entries for when a file is queued and when it was succesfully tranferred. + +The log entries are created with the following tab-delimited format: + +``` +[Timestamp] [Filename] [Status (1:Started,2:Completed,3:Ignored)] [Size] [Transfer ID ] +``` + +The following output from a transfer status file shows that three files were included in the transfer (file10, file11 and file15). +However, only two were successfully transferred: file10 and file11. + +``` +2018-03-05T03:31:13.034245807Z file10 1 104857600 938520246_mylog +2018-03-05T03:31:13.034390509Z file11 1 104857600 938520246_mylog +2018-03-05T03:31:13.034437109Z file15 1 104857600 938520246_mylog +2018-03-05T03:31:25.232572306Z file10 2 104857600 938520246_mylog +2018-03-05T03:31:25.591239355Z file11 2 104857600 938520246_mylog +``` + +In case of failure, you can reference the same status file and BlobPorter will skip files that were already transferred. + +Consider the previous scenario. After executing the transfer again, the status file has entries only for the missing file (file15). + +``` +2018-03-05T03:31:13.034245807Z file10 1 104857600 938520246_mylog +2018-03-05T03:31:13.034390509Z file11 1 104857600 938520246_mylog +2018-03-05T03:31:13.034437109Z file15 1 104857600 938520246_mylog +2018-03-05T03:31:25.232572306Z file10 2 104857600 938520246_mylog +2018-03-05T03:31:25.591239355Z file11 2 104857600 938520246_mylog +2018-03-05T03:54:33.660161772Z file15 1 104857600 495675852_mylog +2018-03-05T03:54:34.579295059Z file15 2 104857600 495675852_mylog +``` + +When the transfer is sucessful, a summary is created at the end of the transfer status file. + +``` +---------------------------------------------------------- +Transfer Completed---------------------------------------- +Start Summary--------------------------------------------- +Last Transfer ID:495675852_mylog +Date:Mon Mar 5 03:54:34 UTC 2018 +File:file15 Size:104857600 TID:495675852_mylog +File:file10 Size:104857600 TID:938520246_mylog +File:file11 Size:104857600 TID:938520246_mylog +Transferred Files:3 Total Size:314572800 +End Summary----------------------------------------------- +``` + + + + diff --git a/sources/s3info.go b/sources/s3info.go index 03a9307..a281d07 100644 --- a/sources/s3info.go +++ b/sources/s3info.go @@ -33,6 +33,7 @@ type s3InfoProvider struct { func newS3InfoProvider(params *S3Params) (*s3InfoProvider, error) { s3client, err := minio.New(params.Endpoint, params.AccessKey, params.SecretKey, true) + if err != nil { log.Fatalln(err) } From 6f211915f9deb949988dd9e01738237752d4b64a Mon Sep 17 00:00:00 2001 From: Jesus Aguilar <3589801+giventocode@users.noreply.github.com> Date: Sun, 4 Mar 2018 23:34:12 -0500 Subject: [PATCH 4/7] - update docs --- docs/resumable_transfers.rst | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/resumable_transfers.rst b/docs/resumable_transfers.rst index 31c7b08..9c63324 100644 --- a/docs/resumable_transfers.rst +++ b/docs/resumable_transfers.rst @@ -2,34 +2,34 @@ Resumable Transfers ====================================== BlobPorter supports resumable transfers. To enable this feature you need to set the -l option with a path to the transfer status file. -``` +`` blobporter -f "manyfiles/*" -c "many" -l mylog -``` +`` The status transfer file contains entries for when a file is queued and when it was succesfully tranferred. The log entries are created with the following tab-delimited format: -``` +`` [Timestamp] [Filename] [Status (1:Started,2:Completed,3:Ignored)] [Size] [Transfer ID ] -``` +`` The following output from a transfer status file shows that three files were included in the transfer (file10, file11 and file15). However, only two were successfully transferred: file10 and file11. -``` +`` 2018-03-05T03:31:13.034245807Z file10 1 104857600 938520246_mylog 2018-03-05T03:31:13.034390509Z file11 1 104857600 938520246_mylog 2018-03-05T03:31:13.034437109Z file15 1 104857600 938520246_mylog 2018-03-05T03:31:25.232572306Z file10 2 104857600 938520246_mylog 2018-03-05T03:31:25.591239355Z file11 2 104857600 938520246_mylog -``` +`` In case of failure, you can reference the same status file and BlobPorter will skip files that were already transferred. Consider the previous scenario. After executing the transfer again, the status file has entries only for the missing file (file15). -``` +`` 2018-03-05T03:31:13.034245807Z file10 1 104857600 938520246_mylog 2018-03-05T03:31:13.034390509Z file11 1 104857600 938520246_mylog 2018-03-05T03:31:13.034437109Z file15 1 104857600 938520246_mylog @@ -37,11 +37,11 @@ Consider the previous scenario. After executing the transfer again, the status f 2018-03-05T03:31:25.591239355Z file11 2 104857600 938520246_mylog 2018-03-05T03:54:33.660161772Z file15 1 104857600 495675852_mylog 2018-03-05T03:54:34.579295059Z file15 2 104857600 495675852_mylog -``` +`` When the transfer is sucessful, a summary is created at the end of the transfer status file. -``` +`` ---------------------------------------------------------- Transfer Completed---------------------------------------- Start Summary--------------------------------------------- @@ -52,7 +52,7 @@ File:file10 Size:104857600 TID:938520246_mylog File:file11 Size:104857600 TID:938520246_mylog Transferred Files:3 Total Size:314572800 End Summary----------------------------------------------- -``` +`` From fe5c66e30d59448f4b66bd6fc888bca89f4c5aee Mon Sep 17 00:00:00 2001 From: Jesus Aguilar <3589801+giventocode@users.noreply.github.com> Date: Fri, 9 Mar 2018 18:37:56 -0500 Subject: [PATCH 5/7] - documentation update --- .gitignore | 1 + blobporter.go | 2 +- docs/bptransfer.png | Bin 0 -> 47437 bytes docs/conf.py | 4 +- docs/examples.rst | 118 ++++++++++++++++++++++++++++ docs/gettingstarted.rst | 38 +++++++++ docs/index.rst | 19 ++--- docs/options.rst | 52 ++++++++++++ docs/{ => performance}/perfmode.rst | 27 +++---- docs/resumable_transfers.rst | 86 +++++++++----------- 10 files changed, 271 insertions(+), 76 deletions(-) create mode 100644 docs/bptransfer.png create mode 100644 docs/examples.rst create mode 100644 docs/gettingstarted.rst create mode 100644 docs/options.rst rename docs/{ => performance}/perfmode.rst (65%) diff --git a/.gitignore b/.gitignore index ce8be6b..6b85f4e 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ __*.* # Folders _obj _test +_build/ _build/linux_amd64 _build/windows_amd64 _wd diff --git a/blobporter.go b/blobporter.go index 13a764c..50c1428 100644 --- a/blobporter.go +++ b/blobporter.go @@ -45,7 +45,7 @@ func init() { numberOfHandlersPerFileMsg = "Number of open handles for concurrent reads and writes per file." numberOfFilesInBatchMsg = "Maximum number of files in a transfer.\n\tIf the number is exceeded new transfers are created" readTokenExpMsg = "Expiration in minutes of the read-only access token that will be generated to read from S3 or Azure Blob sources." - transferStatusFileMsg = "Transfer status file location. If set, blobporter will use this file to track the status of the transfer.\n\tIn case of failure and if the option is set the same status file, source files that were transferred will be skipped.\n\tIf the transfer is successful a summary will be created at then." + transferStatusFileMsg = "Transfer status file location. If set, blobporter will use this file to track the status of the transfer.\n\tIn case of failure and if the option is set the same status file, source files that were transferred will be skipped.\n\tIf the transfer is successful a summary will be appended." ) flag.Usage = func() { diff --git a/docs/bptransfer.png b/docs/bptransfer.png new file mode 100644 index 0000000000000000000000000000000000000000..fd4321c0b484b04ffd9c820679d9105575622c84 GIT binary patch literal 47437 zcmZU*byQUE7xqmv^dJb*3`j^zH#mTxG*W_;fOL1)&{85I-67rGUDDkt-JSCuzQ5;r z{&~+@x|TDC#hf$u+56g`>$(k9Qjo^MB*jEPK*0GRBdLObfTW3lfJg*F0$$;1@qT^- z#pIp*I|PLCD69tqRN&tjwlbd`5D+L}&wmhUzKnu^7fBqYG#x+Ld~PhV0+8OjjZjCfKWH^LGqodtL|Yls^`nkiJ}jO!$&^O&Mx0v7G1LQJR0)y zfWOGT%Gibmqk!H82(|MZrGi0{A*-+JP~z**K@geby2iol$>OAjxCV8;>t$~H)Rpsq zNfEWY0)>=w(QT1CIL-_3^WoV>4V4iGj+1u*AP6FGw4>wDz)+qKD((>Qe`k4){=btX zUc>6u4&P@8R7Qy!o04B092EvVA+BMdI;G_Hrj{#~HGUp~P1jyq3c?d9%$@GKlX}<; zPF7cAx-gyd-^nO^c^AuJY0J@AG$kw?@8y5LG@dGIa6pHXf)yoZ5`5I8GCIAa7>pFW zJ~*49|22d0Oje-!eMTn33|_y$bzQE_&e;Bf2XR`TA+-JSZKaanp4Ohoi z_*hLIodUB<_>V!be|+jRW+xhY_?{3kPL$_Jbw|ySs9;q_mW$N|v-8fL1AEL5M@C!D zEG>3TLQGvRbp~uQ@}}#8vDcQf!~#DwhhLLE%{m&)61T_B)L_sa?;X0-f663gxm!P{ zaY*}FJD$;&_0TidX+!bj3k3L47MndS?>kf>$G`O&2Yq;pj7Gt1A?}gc=@NcP?4;3% zvkc^lrKAHN13nuUI|OZWN8RUye0n9!Z|aa0LSgz6U> zz+8X3cxjE{V9|usA}ZwAoe;nEcd28wy*=uG2sB;-LE7I55rcC{bsWg`P_(<=zE{jh+`cpNAOKkS1ir#ziQn&t^~$2~oiK z6^v9N`<(Ck#`_TDHjN{6Twl=>N|G3G3Sj^H72at{%849nDy}(jl1wDblDDN=^Or!_ zP5ev3NggBw#!+5&d%v&|&06ziC@)mb*S=3Px+0fN28jT(gOfAkZ%VVw7w7zq!}Yz4 zY;-o^KgPkCnw)=-n~JQZYxL8=aq8Wg(F6ailV)k084vJDE6Fde^LnSi5!d~^M%dY z|M;@l6mcolHoJW(@i}w=yRig!5Sv5yzqRe^=(>4Rm(Miz?xtuvZE}@>D?^56lbb$^ zXxhoTl?4+u2DhWE{oQ}7d-B#9gZ2c0g+hqXc+)M8G4GwRfTFixWn#ZsAFqs&^UfN~ zzNRg0py*q$e~8j6`aZJ*%kL<#HZCHR%e+Uvpg4TZ@#PFji^IKhAJraQDOMu7L|jz; z@4&ZCnQ(!sWW;Ga;zDfUM+U80`P1o5oN2n>c5M1p9VK zg`J6wSOt;Lh-trZQO7@O{iaT2{Pg_6Uomz8zbWS3pKJ*O z1O2yi861Rtw~mYmogKqwt10m`mG?yz5r!NwB(MC?ap`_mO4!+VN4IzFr}Yrs3^TV^fkh zaSG=_46}cQRJ=9!&fj_Hy{d~d(Zlp1MbZE{6wXJVo31bY=EPp5IKd*otPo>*^8fQS zc-^Sy4uSt`u)kN4elioYeYtk!UmD-oxeV(2Wml}>ExEVIC&YaA)6!D;Hy-6BzbjZZRN zul?3t?8r+=`B9{v1LmL2PJ@jE77!vd(qnxy79>^jh4)t+3Bp{Atb3_JOr=D-fu520 z4}%zjdJYiS=WOkmj+qM~x$m9xW=NQqz|%Jd0m>f)_2!jtnLdauCsE6S)^H_2=lhbE zNdF!F*J#G_I$|-buqtU8ShO#2tTszpX-B=2iQM*5_F=+V$NyiC1n*F#2UV6y=Z}Fg zAfE5j&~1L7;{Ml1QK*n4|JP*2PZ0h8e_a-Z^v|2y$tI(MLf*vo)>t}b}k2XFu z1}H2OQxbSe^Ls;-O(M^G8yZ{QmNhyGSY7alLY? zRo31)3$vkf+08bz61UOAT~X(Q09DdhuKom@V$nRs(}a(8I!5ztycp|!Z7U0WSEEC( zEeX#7mgh}8yJk-mwMptov5TN*J3C(U-yNV9^tdY(D>bq4lmw(usfCbwnn(Atkt7-; zf}hS7?pGb>GLQ<)6X>*P7ygcjb*FG#_jl70n7m zMn<-{mi9g#QD4v*zQBMfB}!|4T6uAQVy3^TGqO4xfbU}6D^ghS@-PK73auOA%B?G6 z`}e-VX12Jq_jv6^WN;Tglyri~eZ8G+d4P}-50~Zzf4H|ntzBa@7arB?Z65AWULjN( z!xxb$;hkKIiUa3(tzH+Coo9PJ%r2D_NNk;ScXKmjki$COJ?K~M;IfFS>S8X@N41?f zY2Eg6*O*yp0b8nFYEqNZ9H|wV&Rc)}V{%dN)*_WE!#sW%)Ra>WBqZbk9qCy^P0|)( zTLP}4`6Yhr=-AZ6*xdPnnt(17^#d*?3&-A33e4mHedlObs^ONRF@kE1Ozb2x<%k1%QJ8 z>A|b8uYT5iUx$Xo9@B}YHMKqZv}oRY6c+Bo~}3*K2ulOzv2$id+unV~73*gxy9Y+j)18?6zd+M9_R3M$3< zz;nBOH?jcCkLEBwGV(6gr}Rpzts`3)jY2ZH7p>QlUw)L6i`Ox2A^iFjQwE80%2iF= zN~F4&w@7#sahS%N%FqZt)8BY{ymLPHqhV) z^%!kYiPckQozsrt)}`-Cl}>)& z+i2q)x9cB!^O&u?*w;^=IPGTAdaU3hBkFMViMz=y!n;`I%$M=!5iJxypDetzA5O=* zQj!i8ot-bgw8A&fckh)j+G?}JV%I*x6B~%K^UgC0p9@T{-9jr*72I-9SD35i-^O;% z1@ZCzep>6=NolY3XCG?J>$`5gB=3i0u%aL#Ys)qm|j*z#B?@MI^B8D)AfyGoOzmC&~~H!lkiJ9u`0|~ z@h=9wkQa#fwR%MAG^xS;ZXBnhyh5$Aw$?jXK0rCN@%j7|Frv=7;&fnV{b7E?fBtT| zA!wSx%|^dBBRxt*ucNpq-#S}F=QuhEemWE9%F~5$TvJx{E$KYr*r&oiYY9UO4?zYC zwuzor(vkEHGRzRjAB+0~6bjW;TR;hXkAra9CN=2xIcRccZ@2oiUcY=-Mx|%}VclH# z)i~Vd(>0CSn(hQKFL$+k^7b9OkJeSIUQwkp|K)?z?V$g_)IALxD2F`XDGK%(;f=-3 z7Ry1@o+*Z7r!s8(UA8EvY%h6COJTz2-s)SX{3IWU|zguu#{fUa7LcbrDHN4wa4L@TL!LfHDd zJZ)(*lpwvOmAvxy?`RYl+T=esFP^JKO~wfWzeEy(%ZfBt{HwHa5k0q|`dqwBhWMsq zEgH-gSjDyfLLArU>N05M`_hk?Pq5D1Uy+|>y{LaB2y}n<<}$U~hhiUyLPNIlJ~1A1 zrSeFl{&_pRxV61v$yAzau2}`9`3rOohr~t<5D3)bknc(fS>hJ7$}o|X^E8q}s+(c{ zcg%GKZ&g%pdjpH<7FZ83o<0bN35)0F2SCKDfa2C>W?-^mYNI$Sy~AT!hTYUp_3h5#^24=}g}NM=t!H1Oyt0_!LC;tu5$L{P2Sd9& z&y{hdabM&9^1DYp{oT|i^6&j!T;m+~h_3v-{j?wa&XPwmP&9(5gZMV?BI8RRxTEya zaxE3p=PP103V+DVkw_0RwLuK7Y)#;(wY9`2AtUVZzVEB3>AE-KCsAvR8{ny*thjm6 zxEtFI$t^!rv~)+jJ+weReJ`M@pS;hu?xN_AU@pT={P{Lp;e1}de*`<2Lb}xAkBmaY zng3f%Sk<7l_u9&a)Y!}{EM2>KdRFOwGwZ+#>gewnI$fjJH zsks^4b)3yq`!SknRhMHQf9!Owo-QgazJ;kM^S*p%R6|OQYX3-5ctrQtSSyES`Ti?! zF$dNNQ5cx>tSv|~>IA54`Rq+q@Y}D4aL;n<5DWMr`yhPZ<_lzVg^5uDZI+Y`uA`Bdql0_=e9XL9FO1RnHvyF7m z^xHe&rMGQf$%>C0{3F(ydfvoP-75qYMA>Fj3yz6>dTmglc~VAV$o6c#YmNNT-q9GZ z)7|APYtB}O!Q&asR&&DisE`wsks8O|E4v}?MG0BPau4l0W)KNJ85N@xu%im)LV{%b z-Svk14}_jCd7n+)O&%aU&@6L@t@u1$_9*5Zy!jNJ>QM~Tp7U|m< zaBf`H!!lca$O^CAg+56Qcs*)!Xr`64664dFi~KB+~+kjV;CfNKx!vnVl5%irH>`G_KdMW#O`vtN~wLUY9OXL^keg z-$KWCsO%17V)0fLK3ig%QzXVQ%7e&~7JPWVom2H%nedgY`}L9vJRGEv9lze>6kHPH zKydTr91Cs(i9ji#nz~Cj5$VUV328Hh2CUb-CLt^wjJ_=H%m~)BYgE4o{b-XHaYVd5?&>6-=Eyt zq+m1g=5am;glrFFm2;w58f(plyT-Gq(W2BuPSUaIw7?f=dh6i{_YRtgueCvb7YuLfc1)T)ztH8ISWZ_$igQicLgO^z5xw|@TOCrv%@aT>tTEMnDck5}e_dh1&$2}6pC0lX_Vx_Z=J(`G`6~vogeo$3wBWO>P@#BhyugHS zlEl7DoplqsEM#@$V$VhKr~fw#WHX56{*I{Kn|{+AIM4=w0aAVCzHhH^jyLI0p(2T# z;d9;Nwn_HVn(oPuKKN)ZTDiQ++Hhv0sggXT6 zCI$pYt=yl+@Bca$e1A+Da(>GCJZ1@O{E$=-A_C#}4~VG(UA;d^;`OUFZ9)!JN{^yvC>NR20mxpLyezRcIe(IE6(Z%XO#= zrF72|y6h-=*8Hs6%y9J;4Ftxszh^8b@_$H0;K0A5)`o1W#x5&qLJR|Mo!*}FQa9^P zNkAC$1nGh#Z^s({4)_)SG$mKU)Lbr2XpEtAm5xW1_CZ(ox@yTJlSHUw4dUM54aTF$140FzRic)EN5u@oz`I3m2gxUHtR z!8{Jc&Gs!QEaMX*(8j=00uDL5HJbS&RtS zET`pPLP$Og{8mE41Q{4_PEbPbN+L(q6Q# zGqzZ~meddPf1fcT7}UaO-?JD3u=ihj)0H1%UXzEhn*{qS?`A%(q^Z?7(CE3Jp`QOO z&~D75@b17(@wzZ$m5pekY61+TJT571c1Sn;AH6&K=&8Eh^=K@uJU*F_>VjC(=XiuP zXTa&963F4qSaCJUjtk*Lag4#p+_h7z^80F_v)vUicc9d~n!K0Mx6p9U@@#II-+imt zM`z$H5OpS6A{BncpjG`~;JvZ3G%=U4z)n2+BC^RpIk-tG?SuHq5mFe>_H3n3wRhL> z9$>8TIE792-6~)JzDqLgA_QGr;O@KeoKmtEWsAYyTz;t{rsojMKNe5KnzW!Ri;-0z z;W{Rk(XmL8-hBK&n=26`(~Upl3jp#%)`i*hf1`5Xjaa=JAM1(K-6%6T7{x%ZgxCOQ zzAk@YXW2joNI3{_&mgmDc(s=IrNo>};WJ6Z0z>pZhjGfn+#x8DUDL9+2Ysypa_SV$GbC@!>BHKPpgYTZqpz(p62i zT|s?GG++S@SCrxT+vpqhjn&~Wusc@P;xy~6I|B4Zd%>tl*UXSuqNb|DtDR5Z|AfyQ zKM2Dd?5aplSY*EY?Lo>2XuHm)ZVr5n13?d6xTqv4$hP!M)CWqh(LYtT7q2~*NhmuQ zBr79G`BPw4&;~D;tzbz$8M3|S4vC-$V=ysS0|l;ENDL-6{4PQhe*?hnPcQ^kS|vZg z78#q3+2EPWkU`1yL;`?!EGIH3qtMhF1_Q`1dtFVeK^zk;} zIv>%%cr)~se{=PRFog(XDz^~yq!;NY-eCmF!y6g=PKo%%h0nI}pY(%^b_u{8@JdAk zzD4||Qdixn$f2t9u*nn=5NP@3DCHXbNC8Yq&@|)vd*y8GD~Zt|@y@v}%d{poamz6qP`(b$#DbkPc#0^lB$!#b7R0sUhVI zDC&90Tt;mw+SPQrqVs6OC(~(uJVlj15O~cEaB-y>jD_CR^`3V&aB*oxF*DLf@Ti!0 zM4WMy6)m-^3*!0Am_*`xG0?>&UYqk9rcX2w6wOX+x>a}UW&=|NQHw;<=P6AaceYDM z1B;%@okf_Q8r;bN`H92nk%1; zqWM6e8)!$2%Rk&;_+uCV-gI;fBZAN>p}XIs?JdlOV`59lYNV}n@|BP52lh$;(o}Ip zT;2kYt5%QvA@JnGGT9)j^mY*jw>Gh9`P&_($>xIjZ)$JiQ{zUu+)A zFB;(QU4fcC?B6Q=J6_MAnijw{Zwz(Z-S3N>g94>{H}KXJ0|I8KgqM!JGIHe0DfX2D z0P-g>RtO@La{n;6U1H{Bm8oVq%V@E82XsX>&Rj)dHk+x7zwWeNEyD2Ju|b!LXMPf) zad&(kHfQRw&Q9Pmyz**tz9f;H!y{n?g$8DhmLLs$?Y^yG^(k(+u7|~F=)IRjXBWWl z9~Y$6H->cfNTslJO6Ca+>TgXg8o$bhA2^B#&2^-JG8#@fT9x}TF75x}rw*2oY{d5K zlIW1KE#<;EXy`Kv8*7`h?z?ibfua2}U~{`ee@h%@ODGPDUTsFS><+7B7jybRKq=eX z*4CEH;|Hi)zo|?-JXgJ&Cq0d)_lAS9!fPrwkB^UO@=Ii3?_hk{RB9-{pprMVIoSHq`W^fX3>)FD{WYUzSR&UR zDL$VyKfo#PPj+ecSGDZ~&Uwjo%l9>%bhv0jh11e)wuGj}JB2ps`y(sf0T<`xuRmv@ zD@8fw(x)rsMDP?8_xtPaDF7uGyUHBwE-;k3^t>D2yopVW!6`&dLLTJuDvv)cK!~C( zf5{_%aT2)t39s4T(56=`AP!*O6H61aJQ0|%vb@^kp}3z3vQ1REO>GHQhdRVI#)-uu z+)f#JpH`>N_TEeSItWMgu+0&jF+R2FP6vst zzIig7_G1|N6Q->{xdafQBVfoXrciVXUY4v8U3FMHp17_jZ`dPn9=trM9w24PB9KAG zhx+3YkSK-&-S-21upJ}BL5 zWj(yZT05?;@qTLvmnLvoUm2{eJedP%#I<|Z!^3_=zS>$@ZSC>;-=S88<@17kOde_( zdM0Up^n`E0gL5N!$G{YDe00>k1s?o0tzZ;c1x%adttmtu1@r$}<(E7d_b4x3XuOP{ zdQ?u2&yPWUzx-G_qK=+Yx3Ua46B9G@%QNY^(tkX&hb@deeZvp>ME+F=Z|6 zK+G4^bjKQoylM4`NP0NX8^Bjv3iCiKl)|g>1MdyH0AzE&@{h+ur_DrhDS(BZZU!%< z6-9g#{LMZN4+P@}74P2lAq$YW$a&U^XF-jFos>a31}^xe{0l6rq7yv`prA9A=K6 zjoaGUSUXF2LA;c*Hip5#p66>j{G@2IumtNY!hFog^av%XUg5OO_J;baT_{^21n^{^;Mv<^!cs=7~!)g-Z3;O7UJ zQnXPeixf}%rB(IS0iX;^putcxKqj91)v~d8b%PWcXaEU2^0q-jZ%7zn#%e0n_E2Cr zzhcGx=Xq9ZDNa;m@CQE_M3uOQ3EvR$QOnfn zMY#V*EoqFdcZxpIW0u$p?AA|G(AlvGcxFRN^DPA3jE?oHHjFuO%Ll=EV(0uJ3;?4p zjnXbo*KKP_mH(()wi+IlZI?_tZas8Vb!KK8+W*~XRN@XT*t%vNhMI*71EV|^bar~| zv2$c*G1TXMFE&Sj@sG;$7@@z@sGQ1v?Riw0PpA+{O7|apmKS1^j#*2giVVlVbCR77 zE6N1a>^lVYV@>ld%IT|}3dEHA5k-w9*At_0Prkgi!n&r+hEu>AU|S(^X53r$fm{6z zM>3Cfkr}H+a~}57SQ{E6hT^v-JGC+M6H$`MMJ#jbUG=GeAWi+ZWC5ZHO=PK5I>fb? z%{Np&IvB*UR$Y`hZZM9oYSpS_ED?a6!ygiDqB*JNL9;XV5{1c62DC-PRD7n0`$j-G zSJ4)2&=qav+2rarcFO<9vVb4Pq^Ks(#rYTfhL7v*#wB1RMKwgvE{^+A0Y0;g2h8@^ z8?a5K6-0LS^GLf+4|_?4Mxj8PzV$dP-Zta{vl3|AP{qALh1Lbo!s@WT%6fdckxn@T zp17EjZ%F=lh>Sb`*6PLP23??!gB=BWo|R*+YHb>omPZWRYs8bqe`l!1Ih{DQcE$QT zQfj0`njeD(AJq>8g9aK_8{Yk{6Wg&25BboBaRTZNw59AQ9*qs3qO7F%LX94I>}ScXkR7w-B!Ef;((wS6(mxC)3-Ty%g4yMA-}gMJ zzQA3egbRD4Z^l+d4j_!@R_3LRfDDlQYst%AVivOm?42FthytsA@0og*zHp{*|N5|V zYysT!hzt|Tu~r1Xm}_qUic0hzw=(9-`DmFo(4PQBNIw$%x}TO%Ax);A?Z$Lnp$>($ zIEBNv4r@`;HNl37RF_z7Mc#9Hd5-l6|JTncpnP`UmPH@EoACiyC=B~Z7m9))>JohM zYYd3tbu7Iy>+0pIr2}8Nm-$L0BP8ynGC&>@OUt7__vCfhIK>6!}2BdDccl4hX8n;O5OPG z05~PwU)H3>b}BWD0arrd_de(FqH#eEx_9vWn#_c#xnd8;Uc%-Jr4Clv@*+l-!wdS07={|+VGrGhBv4K2cNFT8*d z!LFX~d=QcLO8CmFSudbLX;fPME5d2IWkh#!L7BVP(>0|SI_h^2@%-z#1i5iIVecI7 zc{F>t@(mF>d02JSvcw!ukeKWIgC%0Hl893LbviPyb-K3L3aqecV`{#z=1wGv;FOpX zyi&fvAB;<*FLA{y7stA!=WcY+GC+aH{-T3;>hM?Zd4X28Gvb-vXSB=o_`tYgtB!dv zr@ERIfe1HYFwm9&EXIz-v^$TZklRh1kh>#)82%6j_}>pYN-83bPvY`ASmtu5bUc0W zbwnB%CFx(^=lVI^KJY*b?MRnaLkrZsKk#spc#)1yOavF6oiW>D?UVpZ{tYnpJjFI+ zH1=I!_`6VrVL}kHP_z+0bz1X5o5|@zct2w6%Y9_tv$OZM<1s%Ux&@b26rj}@D$k7G zRzETiII10oPh*dX2dTXt{5g2lJbct$8RTFrWvWv&Vh5mkk_th2%S=vGwA-crp6@x zE);H4hppA}%RVWjZqG>Vq=dR9eK)vqy}LJX{27wD--Mp0y!g+H{;jh3z?_ub;a)+? z)2{Dea@?I)tvDd1J@SmqnjU>8QwB?R58|4@7VtKG+UhNSKGoMS`IP3Uvm`?Se##KVHx*U0xt62L)t~x8`T4Mw2y;>LrHf^Q%NO23?_yT z1~Vc_ZR1~E<25lAeEnA8L%p+m7|{QfJ&@}wfJBLYO^;)ZHCu^)11=HYW=1t0?Cj_~ z^4a6$EXC6OqnVG7kBJGM3Q6k(GkX(`Z-aQb)`C!b%~eT>_1Kjh5SaqL&z-qaTJgON ze;~Lpwm(A{KPA%7Oz~N@j){7G7?{VB>CmSlawC^x6Q)RVSUU^>lupFxhKXS?99~g+ zX=T&9-9bIt?jpE%=SuUES7_3^C)$B$fSD+b2Th8J^OC(Gj9}%2P$8!vv9u$ z*Sf&{=aG-!*v~WK>8ZR_YaFGs)tqH^X|vG>>-t`?d|Fv{V681_*4`UrehsVLNb}+~ z9w~&HU(@ZRX^&p|3<;hL+VFV^8jpnOtjU<(RUH!n)EWNvye;af<;n!UctVD=Z}*Qa zuQYpff6Nkag8^lPBTs9I?*030UGJ%`Dz^j`+Ngg%mgWgyrSJxkVb;gn88Qh?X7WfR zfM3EL_12FHJ&6w(s9h~GqQRMoElXQN+Y{H-PL5Vif{%dwYv+GB7Mu2h$G#N1^XDX7 zeZN3pt}~lcTU**t@_C;18|(S)+P1~XD~`NG`beNs3ktq5a2D}A1cE3pq#F=+E8~sf z_uUP%=#$W81~&S~cP94}#GSmXt5LvHo+2>J7Mvd;^qvY>a8F)I2XV$PH2Qyw#;8s{ zbS|Buj^`;dmUU@qZ}W3)1Fe5&4`Xr*Z=>eWUd(07n)e|IH7V+Qg%2$bQ15^U$aa`c z?^i0(2;nq+pauipLWaQ!*2aWxP)dX$`*<$+6-2VW!Jc8^c=oX52!ER9ltZS!A8S!< z+`OH@h!O$>?*d`q_vA0vMN;2BJ|fqb4%mne_}P_RH(mfjqrU$N zUxQ3!C|=Q%QA;AtYCQ5cIe(<87O6O;B@7pm!h#Y7&1V?yq22Z-&1*y|B6%u&tnpPuu`UzGT*p zPS2Hv0C2rK{8wZ>*@TH!&kk93{sKNDtQV=xMnXWUJ@|@akT`2STtYQndC#C?Cy{%A zM~(;^|8LkH{ro}zzF$g4k)9fDGP*DW=su{i(;I4MXMaChu!0T5M4pu{)>pPSh4){3 zo`AfamcD93zRtm^>J3FOM`fAL1pzO2ZKZ}t_qh^%VOT|)ob*fBiBAwrNk2XD4n=z& zLMe&m0MkgvX0Z@a6F(7h!h&5@XwjA~{|#Ee?4%Q>l*-ouo!)sJZ2m+_wtM(G4u3La zW%_i%j*A%YQ(Ap#4P|Shai&=(OKwTLi}0gpxdXC8K?H7(PXt=Dz&P`Yh=~10VTQPusM)^mD!`+hhwX0yfMF%H(KyvffAt<0t$;H@@7UtR%MSpN1=1itH!~SpuRI-q!mU63kni1aTFnd>e~nJrvM?>|Bvj)t^u}o1S~8Hj^W_te zi(p$Ix!T4M@;UuWpQVNf_)MsLd0or%rxy&dE)w72qJ};IBTA6~CQEi=t5ji?KB!qFI-#Xu58MSY4Z9e}vR>5O1K`@-0^vE^sn{l!)z z#0HlY-Ej%3rgksbdo7=q(~7+TH&ricspDX+Q=AdzpD02ToRDa=hFk9+a06V+j(mDD zo{o+S^}yTJGh(i5IAC8>{~F^PH=V%Ds%pRLGWm4XVf5&AC)(@F>^R7@K-+asv*TpP zlTA1hz;JW{*t#hW<=?_JxXRkwze`lxQ0Ku;dHe> zFo~ox{`Q$5IO>iprq5Wa^rn3Y#7ZDeZ=2oCU)bv1pp|EYf1^S#09}qzSn!2 zjt|9k$s&|vfM(9V`G4uJveV-D=2a$S zWD72Nml4yfl+)2E5zJCc-i@5roEXMpUDtv4l25hKb|07=V^GtC*)MTJA2nTwQ!#TQ z%}Ax&yeURPR7LrCVcb@)s>`E_|l zTkNx*n~EDQ>E^m>3*SwZ^IC1W&ZXYbZCG%QlD~ldeq*|Q%u~y@fImOCL_kS_r#9@a z{@uaSiybO&kV`Ynuh|s^UEDp*%XWmZkt`%f5A#kAv+`c}R?@N$A;T!J#w)&zAf$tO` zN`U{J;8pY1YQXRT!EoZau5-@Q?>zL}*QYLj8a|(yy(QC(hIE zxs+jEP+Wnhl4rapb4?5-Ui+3YUiHo9b(04O@;pDo>%h@L6Q^DkdFyKWNmWyDZLrzbL{d)C@pM0SHWN8 zbADm;+Vh}hkMWvF;Ay>>g^+9!b)4TThB(t7dk_rW+!#M|?+PfgtQvTx3k zDLQ zdh}w=lzOBbAp+Z(BqumXICuc@_w9+RcgJS?!IHij`iHh*l-OgVlO@J`**06UjUrj0 z!1%bNfu+?LbSsSQevhs@>mrmSJEm9|JOuK$WOx_yVzgA@b@72!v;F_V)nl)M1+ZCo?R+ zyMs*P(5>T(2H|YlqeH;pn#Ff8T1p+eu#GJ^l7iefk9{DWC|?2&%=aZ?uKp|W)>%FE z;!`^`cwGJxX`@L>WOLAbgY_GLJGD)Uj|2SHw@w9#jvf}vvbZ0N)~un**D<c0@$%QTJ~&SRQOmq6B(cOghqYf> zjw_=sjIgT17nCPKG9W}Pl9^E+zS;~7F2Kzz$#-FQR!Y>@!44K`qs~IIKw{O z1m>{A*apD6qXDW@ZgnRAyM>(K>C$?H`d} zjLSqSAq_7HlwaxxxJT48>_vtCIX*qp$$WF~g&DXbis2CY(tZ-ylmg6)VV6gRfuC?O zWMeG>>Ca_YPTcG1kn|oM4LU4*x2r)05G1r+sPg0eh1WRjq$eN`3}q$gO_25MA)VJ2 zO3l<3T4Qa?6`*XQVLFR(No&qT>)L8!i9N(WvKI=L(Knf;mmTG1`V5jC*aILdnH-%e z!Cv*(>$m*QLtelJhrQk14OZA`6}SW8>DvKOp8x?Ks8M=>Q455labs6NXu}$Wkzsj~ zEb3Pk5GD@cOjGe7UIn`&c_rL;0tzV1w!mCurKcBv?1HR#fUM~$83Z|i1yHq{cG(F z%ynYLOnE8Ck&53}>^J4#^)o@?wm*Za+jNgufOn`U&VHkq#t-OXUEEJlGg}zAo4f`K z8bZqS{iVV*r_)b*7?3GHvWo(uL?8QyrQe$Dt;Zi5wZ9O3Avc}rv%3RD`AO$T{&bN> zcEo^3RzDyDa$n)7mq`72xj z#)L0A04kv=^UC~N8Kdu<}oBXZZ54C#CA|AiJ8XeD{)_hbap2c2Q{lnuEC2W1m+1#C;6H-rNAYrpjQ{Y^@; zsh@xiML*C7$(td<-`WA!#^y8h1}I`rP6O65t2F!io66|$K`@hT;RMNK*$G0v8YCR3 zb3o(`7n5=e|H*7F8jQvT<@c%s50kxs8s+L(kL9BsIfu7+3X!?Z161l8wu`fMzYR~Z zO}EGfsQzx0k7IOlZsCm=sxvT}qXNOv$Lwv6k0ZJm=yzjvo(xH>uxNBm-{-t>l(d|Y z+&ZCb)N?gwL1TsajL!8~OX3I-2MZo)(%y%jFaEN`vJ-HuE~-FO$70xLzkbwOA&R7i zO>gw!yb7L?`l;bdpx<~={GeX}VA`=$4qq_za{?(>K*(={0OO~PQf5dM^EHeeNY(?} z8DL4WNz-lljjaaJi+5^%f?rAxU#&CO4_FlHQF4H_&t`^tcT zTnV*3pr&&XZ?M(?$^F|~Rr~G@_Lt^s*TzATI@9nPv|~pR4+coQ*?K^6Fj`xZH1FaVjK$opS?c9Pg*_4TlvHVUA^{iaEhgu}{ZCIP_Utlz%LcTQd%mV31 zk$F^1*huBnZq?R5TG`rQYX?z>$9959=|FQVg*pnjoa;~rPCE+_9kbQ~#Y9AB*kMVc zroH5OcUKjZ>ui+8b59eK28@i}#HbXLKZ8xGxr%bC3DlYZOS5)14XT>o<%ZhZF=J#N z2vogmH_UU)R8|^#!~~2u`NEW5OHZ~@b|e(oil091w>w)xCCwRuHT{~Etg)ne8S!*_ z#@DbBJq=G@W5vl^17^o8dQ9yL-<%J?qzdQ=_65q1wUHh4Uz+b5Q>`d@acSYH$606qv>R(>|dq>K+KW?n#``d{drP^(Dz^8zHs*Ecn#y z5)kbMbHCLY(E#?%H=mLofd;b$M8QsiG>1ILXWka#b6FVKMQydqJ5HcxRV~^Z0efUj z-R=kqc5$O%v1yp9vIs_((AA;cgI#I+fMt_`&(SG$H~Y^3;(-=E*qaaXZq24#4OUn_lWxHMe^?EKL$jp|2-J#;vqy zz9v@_XUJjXkbe$=akQ4{@RNwy51S;h2orl@JZ^$IRxXxySIiov;qs_BB=1RGo7R+- z#X5$QN`IkXV$rJf?d>E;U(oD3QLQL?E$bePg5r^(U=;@xIgs{JzBq; zS=Bl`7lZ?oIZ;w%RlXYw9n6-~qHKsUE4}K947o8M>qE!JSYQQ|(S_3BleCSum2~_? z%_m+Z^i#~x2t^M%-7L4-(dDs2@6DcenKgU6ENs=$Oq*<~lNOGFfYN75qUd|;@tJn2 z!cOVaOvT?_IhB7PjnaSV>Dg#Zb>G;hB#D~cF3 zE)p?5!GJsqdrRY2%mzLaCjVGf)?-OlZE4yB6K7&uaYDIf78>whJy3BfMQLOlLcfSK zdx?m66-20`*%B{)bmiFd9XX*`ezf}hHDXnyZGS$}$2Ge#>~Q|SjZVC@Id!)qk#DM1 z)Pu|pkA!|Iu~|&+OoZ_+wON94Pc*TC&`e6lu&Vzxxri5XPJ|8)#Sdv+d?1*)13)io z2R~74$E&M&Zb!`C!)Nu?c#p=E7-NvF3kP<*OAJ}eO0T{BFR_F7TQV)!rbr`=T2h8?kZxo^ zhLDn!MnJl|L}BP|=}zfRk?!v96p&8iJ$`@B`=`wKnVEf^v(MgZt-THdMC=VCJyK*3 z7q9il{)^}4$A?>j@Gqa%ZBV(m|9T*w{uH{uN7HW3s}Juipd;c9c?P40%TPT6-|vFP zsY|uEP9M;^7Esu)tZ3DV56@Gk7)`w;O`UEE*)5jX_b{V^z!wPKj*atBFN;N6`tLC; zzI~ss!|T(Y`my+`e1Q_Q_1V5A&_4lW9-x@t%zn#6TtJk}G<-KQGKTE_{hjpx&_&3` zW}Js_<=)A(^r=4lm}N+f$E}Z;Pj=}pY2D{BP&?e9KJoWg8eF2>=EwLSPoM|+95VtV zf1ahm)=c*IXn7_ch!E7rF8C*>G6|f+j)%G4|37|!murdleHJHx@9OYMf@E~^D!c6l zjZKrg)z=0Fx$<|ezPbh;JQubi#Vh$@e0kJ6TtC6MypE43BzZ3ob|0K(A}cz-CZBuFnkeO zQSu)3tlxbTltbg^Y^5pdWRZtb^cr(ZNjxP0wCZ7&DNyw}eAI1M>Gd zt`H-VcWm$!AunKOVUpy-uVaM=ErRAUA;l1O<|Iv{+75}T+UN;;wfTL#SI4f6`ev*d zb|&|wuf;xZG~cM^g<3f=(O4T9jqy+IK9p%`FiQdWSckxUAAASG(8t3*O^BO%dh|K(TC@C zbhFLPo~*MJDFT0?7R2gk48z!8MCMjx$Jqy=IGce!F>E$DMn2PnTsjF_@93L~?tE|h zNqI|(5c5+@d+S~jL*ov+=iF(U)M^=vy|!VcG+#6xr|6_tn3K-yH7CHpC!+jMKxa6O ztOU}V=72tvA*iAXSSd9cX;XX-a|VZF#rJmg?dGk+4JGyOZn5Jtl{vtpc(20EN<)oD zIV5SNvzM~6q1{{lzfS@GZHBWFL4O)GXlRD{;Qy?bMEEXfRjt}3{@;P+|M_!A7y1s_ zSUVo&S8Sz>By&=MH#+1UOl4Rbc>7;GJF?^piuW!}83dC0sYdsDIi=DCS4DDDS~=sL z*iyHKe|;!l|IzzK2XCM$ZRBkCV@>>N@CSDBGNXNyZ`X{4hwW>7*8;{tNhmqEez{fE z;qam&+q8z=b)*NHV1-nXNFj%4xy@1S5=~+{!>fk%Zl(*7>lQ=}*$Cple|AElo7+87 z*iR577(88v)6Yd)LOztEYJApZ$4ukwdp$HC(M|-O757MI7+KV(xLvrH$gNbq@ZZXU z&nrhgAJd+jZ^rZ{-{ zG0pfa4M#M^wR`!wp7Fd+ ze|_YLTgVZyNkX{HI$CGFDL<^=a}%@uI{#Ih5_KUf2ASPqCnb~Q6oZJ%E_LRW7maWW z?cNh0JOl+Oqw=E}9_PTHogmg0u-bX|VQyBvw1>8ziWz&49;H_OBG`k6kaI7&=C06< zRq`l`tm{_@$L)EVlQuz0iEiM0FeDUD=Wh`BmD<936n2NK?zF`mJy3X&&GtTde1FK( zONhdfxvxc96@E^N1)VsvgW|jAfj29g_94b+BJkJ`?3W9Bu8OzV+a5^u|3|xPGm20? zmWPC;_*P0Lz`y)1)mrNR7)X#SZe;tPzIExhbg9URnwdJtCX?aqe`;!;%QAuHN_C4D z!Iq1g5Rx;k76^t2|Qd`oi zpeZ@~@R8=Zy70s%=jzR;pA?Zz7a=m-+LY$?%&%^-QPD1}giATT*&f6%)6UL`|5sS~ zExm=cMg{_`pf(;&*U4VdrU-c&%(8uuu;<+x@dPTK%qu!_)s51!ry{ zjbG14s_VN^wJGNdHl$ac&32{rgRG(m&t5t91dnrC?RxNXQ2{&799N}5^|1g0s(srr z%J2L~+S28!NixU(8R8fE2>1hrHL1UBw{LBF z!CHba$Lm|Zd;MWphgihvGuouVEuHk!7$^=F*7fyJimdyF#c{WP)ZJcTSQW`)FZ*N( z-(#2v72C|ZoQ^FHF=Sq)=`kuS&G7L3*56O~=n4(%+f6^<8=(VwyEcoPrw;Ur4?ZUj z@3Sm?Ic;aD+hLEk^oKThK0})W^6aUzEw=vJ&%T#JX^&1Sb!_7DTqe&<#`IbC#}{09 ztm{jq?Hl6uQ6=G&8~BmUe^Ob3Ufh#$mn9kJq3~oMXHwQ@C1*1;t)rTCQs8n{Q+z7!85u&%9l*Sk!f{*hGEQ~Wl z?>%+>)TPE5^;`jp&E6_+@fdO^8I0U7kZ0&6F-^UY zJPtG$bsgG)imF6%#~gXcajitCADZ`1rJ5|%SMom|@|ZLs9lyWb6pY?Xb2%p0OI4wu zQ6aT^ZBd6Fw??6Bl9*1-{>>;)cDgsVwX1~hA0CgDS%0-otowL#_T?#?q_9XxWHAFQ z`(8j}t%L2<((T5L6&W#-Zf3Wgzo%AT?-s_NPM6mP79fvBhl-+wXi%z zDy3-}9m^XDl~S|1M*Mg#tkOv{E7+$Bkr=DIBy1U_$_Tlj?S67j0Ef!8d%n1m@oR`% zGYDR?NN)VWc9q14w7SlCJt#iHk}oc$wA6i19%BG#m|AlD87ef8)J7*Wt3Xy%oNx4S90-$;6f;KdA z-@EIg+o68?eOy#uPchGA91EE@pQ<$~Xi^dp!|ijAJwOpVUFv-=nMcszz5V4ai)p32 zhw9weHC=h)OCM^`01`joW9HjQ6c$07DO?1tT<2M?KWS;B=VDuTieUg?sq!(!K`9%2sJa=--j&)y&{#KR)uWWZ=5~h%Rtz>p5GkQ%j?v ze$dL-Y9g-jL^ut{80fHliVTaoE!tXTw^i7}6A42y-U!{jif30*n<>KfaVsb`G(lj< z`Z7rQK?L3y#~mdWP(dY^6ep?`*mZ1M6&#vU+GST?{0c&$pg|d{C4MfJ7^=NphG=55 zS;44kP*UpxXv$OwSOv*j@%Wvf*0V`E@4GX@j6z^YLr6xK9DDewe>>>ASa~T8U-3X- z){uv%ID~}o@iY6*S`_`F7A1p08=ki^U~hkKl-;!@y^lN^{F(!TPpRkBP<6koz(+s` zhcb-m?&lh z_q0E`k7%jfF#|iz%%?|;(;0zzG)aGg}l2QArvYh?Tv0B@8lfl{{^wA=slHmk;iO`ZnWmJ=T~DQ>-J_7;ksWxd6F|C{Ctok zVWqjfhfA`l@ZL2VDfK1GKBg~8As3Cp#sCFf#Z8v4Pahv|$dqAA8}Ok;d4Nn2TVoN| z-lO){xz|END=MU&zT&H+ctrO@vP%n8j9zRssSv>TJ`HVRbl5+IH}?I&(RJ*I_cu}q z1~N%CvqGIxLliX4>mw8pL2g!`Di?wkzp=Z-P>gIdEz%W|Pu>Fv2#P{(&~IAa2+;MJ zlg#(zw?aRnQtls~EbUwYN2maobo^&x*S>212Aipo$Xv-CQ8=H*QIvwmTbPLQ(@llk zKr0T`HMc2o=9*K5spKK1MYXA-(&Ua`|2~p}Ss)KTR)J9ocKOLh!e?b%!tD?2tmqaK z<~?YtmPEPvIU@E{2E0{R#JHz?DkU&-dPz1pyrUq?V!Fq1>SbhEZd1?a6id7$=31Zx z)^0x=EydRn4wSaiT${)XZK-IC!+8z=qBCO)ja7v?b+^Lpg`EinwF;oj<2=oM7uTT4 z7AnY?>JgeGAnX*?Z#Xc@OcV50n_`So$;Hw<63aE+mNCVwdZdQI8$qqZx*&Zl$kb0H zf{HH*!A=iX=hu?fg?K%nMdPOknY<7dj9YM2~F>){)=bp7p7;|$bJxhtrnp2;;`Heh6;SD$|9l{{`@{jZ{R9kXN8%(9z~_1XHgXm4 zhMz+c@H;(pDkudn8k@hhMe!PL?)n!pCyQ<1$JBLPQ}eEoPtvI}+`1$ELhf*;S>2gj zzeGW~d|u&qONUm2#iep!!2IP4hn7QpQqz-z1AHe96j^f0e%#I&30kCM_CE5SrthaJ z{$YNjz%NlaSv%SkvAZNnewoLJD&$P?=3zCJYt27YF7|Gh2?;7Bt7mD%GD%Uj4O>je zexg}LO%=hMMAgj=%t<5^;4t#3%640BZn4Quy{ahm(~=N3;1S}jQ=A8Nq@Rw`f`F}i z^o-6EnZKLFH>poWc7va(#7X$ZjmYa`pra`KIa1wgY{F`=KV6Rj@MdZ{>E#xf^kqZl zX0a^DQS*5?aOx6LUMQ~Y|Bp?L~*Us?P^?zt1;AZD5C8oZ69 z=eW~Au|zkki>RMv?0WL?rjRX5!!hRNx5Pr|Tb8Ep#5~Ti^|6?lVsL|!SQnB)6)S2P z1C}VI7qF6#{g?eh80;+w^z6uxX}Z~ZA%kR#gR@@=r6-~|-2u@2))u>0i-%Y|GE7Sb5`2l;SEC9GmiOVCs(gb0 z#;)5yHCuH6s59<2%T!VVukm+L`^$=lh~QFnCQIlYoji_D)zj=H>l8!^-nq-+b%yT< zD;wxLdNQ_Ozpa8@vQHEQPcgD?(uy`>o)C5+mJ*!K;1n!-Y>o_)~JFP@T;n(`& z!sQ}byd9fEZzW>cB~G?wa@%_5tej?*TN3wIx{!fO^v;GLIWJ^d!2~W22*TohqQ%7u zm2p~-S==>*_tLr!o1$A#V=c0p?R{KqJ}zZ8Dm!i%+VpgTzvHQenIDvzlI@9?A{Gzt z<*{i)X*D_wIpM$PK%E_enEBikH@C_XA)_V5F2>x+!PWI%a^4y9ra4gKUXroBIy~(F zzmTis&u(I(&VVKav^x}bn~XN^BdlT0B59mX*%gUZESYWgrkL3_-@8brH&JRa22+x$ zbRcdRbG~P39mc|`B}Hv}f|v6zyuS|+6}=$QrqH$Ysm)h7 zeCx3ZaGF~_RS#ILNXCuulF7Y&w?c09#L}EyH97r|UhnehJC-qpch7|uxTLs<0~t-A zCP4L=v9zpY;OYn9G0w7I`IxH=dyaf4DojV1TxkO0r%MZbn?z&$)FUQ4MqT-0!<|}| z#GNIHC-Iw!^|k}Te?xP#8a=P6mAv&vStjxV?EeIE4vpZF|65ybh_3IC96OujsAIZQ zP_LadUhH&B#;r<4M`bDMy-_&j(Q`q9g$2sA=>q&;8WOo^Lo(hicVv+YUnHYGsk=X+ z4?Kn20%FKcWXT6A}`M8=d$#xRML0QyMW`BYu0sG`J1zw*zxJCm?_C*2$m zl2Rt02ZfM9EeJ$GTiRBYj;)GKF_>;oo3Hq4%w3-N7wzbdH8pcUeJYeeN)3RbNe?cH z_4cAEoF3&ZWpj5K`roby|AY4@i%E#U!D<`CgppYu(meIcIRSvkSt$!Xv()pwm!7UH zH)*?CB1u`d062^!Z)f<0n!Aq69C|%!+Z@Ux|5m4?l+bKLZis5nn?&kYnpIFtN!Gx- zuL1N??u|(@xvfRgt5UD<&$^6P4e`4UMC;F5z8Y`43v7;lqP~#r2{<)%c|b%mx$=hv z^u>ZnV|(YKZ9@{A>1hOnHm?oz%zIO}QbmjaN1PmY12at19I;ks0AMW;B~o92nTW4t zYE1$43z@KIgWjNcOb3O$z#GV4$$a?y($b#eX>~z%SAVZRbg%&Ixt?Cm(L?}pv0EN4 zt$5z5-D2NLv|7~G6#jL1XZ0@bSA@sEIzIJhC_+n38+dmJnc})>{s-Sd?~@&xA2Gj^ z_<3p+SJ)tEkY4q@tyQkpMB7T!U^kV>Lbd(WO4F|Wb{2U0Ak|hq7RK3qj8`}BM}Lg{ zNHJk=>5=~X{gv7qzewD*axr+oKZbw@Qz>ekmR{+*ux_KEuH;&MZCcA;;OF6m^!~pn z91U{$`^&x_NpiG2ki0@Saw^mFHvz-8_R_|g=SyTfjH*}Ti3rIn1aYRf5E zH~}>*J`A5k-Pc9dtJiCqG5SxVeRb1Nx5wxIA=RqHK~`jfCs)k_+vrizdGEr!lk6}L zk_2(}$u}usFQPlR{uP0U2yH3RzgM4W@0ESv0-T(cbH{0s#f3-t6A*-S z=i?J&qr;0pF1U;c9PR?PJ$Zl!lE+ruQ%GY>adF!Mp3gY{(e`SUd-ywKY<%0HH&WBq zj7j^7xDWUNh<=Ap(9N2E*R>=SQq++XidG+f$1<98B4|-gMlKb6zBP!8@ z#k&J4i)(JIHC%SdHptaOs3>6@`==CmojH@F{`sqZSEe#QiVg)=mwm6_i!W|U;3Qdx zO&T0smDtZe)&fXv-_%#3{L8bM`m233@vBw?W)Vt18OR!sYtsswO>{LEHs|9gsJXQz zzk!dkcb^!1N6=Jd9)RcI*WSM_XhCC80KAcyXRf@17i<;rMRVp4@GcW4ZrL&eV6)D{P`^_j)kiC9&uf;3SG`!KT;HO;_ z+1+JGUe4}|lTeOmF-vE_R<;fYJ2{ZvgM-E2tXx2{2tHaVly7rXZnfJyBiNhsfrwBw$Ss;d+MJquK``=6$&SbiiQ9*uK1S- ze4gzCT89d>^Ut5pbFbCUia7*#;+55<7I%p?Lwxh;W~V*4+TM%A>uqXyf)z_PRl2fCqHWAo<)Um`=|Ee}(ki%5tZh z8NXTffpn?1-%sqXOLQ73y#MWQRJufdc>{rhf;xr^1uL%`$kG~QRU7QwSARdH9q@pU zMu2R(F_dy$i?>m*Ed^Qg3OCfq1~ZFgwO0$~X`>~7f)LWS3Dsc>&{mshMx^8aee|VB zXGaf%!LJEefEcYp?$b>udbV!3H5hqWdS;4@ev?V1#;-1#tseo%^i{|2Ge6?n_Ur%u zH=K!e0-@)vAN>MN;s4c8b{E$FSs9n?o=D)PRlI;AOHzS8(@}fmcRq+3(o#-O!1vo-30@cf8#0vI{IQ{T^_=R;XT|Y=o z%5Z@5H(bE)0G|{!vU%6GhUjiod%F;7#(p{R(si2e?2A#pG+SCVr*mn>F%Krk=*o`c zF7+CKsd=!{rl%ny8e{rPW#nNmK%1rk;jcmWO=HGZEcAPo=ZcjJm?wSm8aub{8 z&2>EZN7^|s%k4Zs5K}Zq$3NQf|62DxcEf(+<*f^!aU8ZzwKJm_=c}I6*>>R&)AVM} z1~^UAMYYk{Kr8iu922hIO53a}gY%oV4_YzD9_qsf@=;d%!YSn#j?WaAw>+FS$@liZ z76S&8W0gmU3#Qv5-pkK5#KEisyO%OKu@6h1&6@uxU&CBf}ewDSG%@H zsfNZY=wdAvS#48!M73Aoy-%2ETelTo@cFm9llYYdj16hq!Iai=ksYz-4Pv)O4aw%QN0ft&h|9R)YGE$D4RNC^$dg-^AThR(b z4i~YtNWJ%x%yp4k0z!Zb_S9jU!U0apj5*zpOIA@ zmUHZeLp&*lw5qR=f#2H7dyIJNVsOxnxb|kf_sNpmhPGzA1Fp6{vqdoa?$xi>nE^N`quu zfG|(l*@a*~c|gB-tX3WX%EQLFhm+~M3~gz!l_P0aRD#(V;txO*7@1JzpS&;}8 z)iZ@e?>==Bm;CC#x>5Y5{y)uQF;R^%J{CmPtja$;#svb~2Yl-LIg)L63RMom`v7KK zcuI??Ee&%ArzW9!_Bka?p~9HtVm82%bY7_*OU1}HTMl5jy;=8|Bf5PfXDU1^@h@1n z9!^KjxjFVFuRM@ovYvkEH~l+u7`EZzx&U{k1yROHM~HP;gX5gV*D_dy!r-VaF*sEi zg?w_pwrGw^?9-N6iaXsMBx@Arslb9>DXO)yn3sg2A#wL~**6_~a$>v96(sivTa@lz z7u3}BP>yL$i%_X_+I@cpg^w1gh-C#tC#z*K!=AQvan}pzl9+9ZtS&35YZt-6NvQ)! z^GUcVS%=ytcnG@56_~S}m6-wtrp9LRB&=Q8b|%+~H(Gj5mWlyz?V|ew{6pMMV4KDt`5^5bGTB8?W-xLpsk&5M*3sXcrkNAs zhoLH_v8GSz0P|&==5gN89o=>3UQ7O!1J`ycH+3Ta>3+Hm@WQkq5=%${+=*Q!7;0^K z89$`%n(<%yvpxkLr_VFW=6C^I@FGXR;%zP%dZFjwpuy>|6_^1*71hwirZF|1aX6}$ z*GsRc4(e7tf7RVm!B7Ep(@6Cl+({{l_W+7hkQDBSX8fj3PA3XSXBjF&K(^kV1D->@ zI`BVO5O-px-$Ly?bTAw{&*F!#U0R6h+g_Fp{^+35j6YfnK1d+R1D1%x*9n_IGcG^$ zLm~0uK%0EG6;A;I3;kN2{Iw^L(;=k-d-kPk8Ylm!!UU49KRKa->syBS8RM#JNJWL zSm_gGw(JyZtE*6^M!#h+e?%gmE1SR{U2hmQ-!zyFM;bIbKggQH4qsyMK9k9_42a(S z5X2%xuPBFetWpN`JAoBT|6@}GN~AE=S&&qY{n8KeTN@{1SD8(In$5+FG~wz~mmyjG zEr3MwCPi0{>9LqB{?x+Cz`QED(BHLe^{!h_U~h;@js-OH9v_`?jLv%Z7s|jel<}Ev zq%zIZnDb6MPJ!Ff>JdiP!YWBeTc%u~p9E}IVUEqB(Yya)Gp%zwaeK{g^vF9r=u&E- zDWzDO@%SlX@sjpNX(2VEv7h?LSsR@p%l9Vm;R9!5qHs0itiX2cBWSHBPXs zey-a#(mmOv+}KyjjFzb|KCrspToPnz{}?KSj2^^$Fpj2ssx8pT^nw|%Pm0F?2TQnTO&R@o{BOCfJmnj-y9XZ7!Jm#)S4Q+s(?oThIfFh76;F@y@FIYZ@I(!_8k^AmG9jA3->>?iO>2 zemo}Bwo+K8@s>vdy1!qyg#$GKHLr<_Pmcl)dHb&OWK&DD1>Na4Vt4JUBt)XG3*A$m z5kI7aXD<>%CBO%1{STyDb;8L{wY*O(BnKGSd#i{N2?)rn*op2|$&D8zO#b}$uCt$y z+4<{R9GUEs(m~q`|-^$#K}6;S*_e0yw77W5sB+=<~iv+ulpv71tqeaF)s zhUp<(xIkFcIa&VkFP6x8G$rMvB#o^&u+GW-dUSGYu_(6dsRxPRDF7^at@b}!U(nPY ziDmIOmdktcw>(gZ0EGgdzJqyb2HUc5&zoO^=4ww0IZhpgQvfEmexA+{QRWRsc!1Od zZS@1}sc6?^v4`Wms~>LxV*X1hJuEMzl&&9-#zAL>_Mfln@ z8&#FzKdc#i!w>S#x`W4$HqFj60!uugrrZV4A&_Xdg$vL&9 zi(ZPhThITAKD3YXWtw0xZ^fTqdewA`Ub&p?-`jYIT$I=x#A8Wor76 zh35`ajJ$u}++WuIgPix8voc0J+(Uq1XZ_=vmnz#+d1$}WK8P2wJFt_E?_0@ez)hg9 zOZm)7jGqa{^ETznY6Qi3ie)1|4FDC8$~$2LNu#`8%JUz(K|NhQu*_jK&%j&Lfmy~p z5!EYAuc@LOi13ASU$@V!Bh%XsC5VI;NeQ9KSR^ysTM-5jvq^0M=_tw~Y8dl^LM|5W zvu%&?47MEuxlpv(e-aXqa0|XhDy#FY`j6NnZ1A78<^Rz)VoI}qbaEY>E;M#EaDlgB zw*DxDa%1cUG-HGMR&EckUU{qQwKSqUSNPw7t#KqKIfniGvx4SVu&w*Gg%x2*azoma z!EdvG+-0+S?>mc9GFs+$kNaQ^7ZN-;A%6tdrBGs3zfdUiN>PSf@n(@W#2CS+Rq(C& z!ulv|F%T!j@yl7Q%-rz1IqKkxQ#>F(IoojcSe|_85ZjWM32cvwMWCnlcy-jVVc|vs z)d5hcN42ia(t>#BLSIZUVZn&SBYPA=1cj>pj}KJF-#0tX5*U#zuIcbWZuP&7k0A>* z7V!D`iI$zsA6xIOK7=4S%kp-f;V`|%e#@Wqy-(A;=h%Ovq4*|#GAS@`n+ZwP790;S zKw2hD=6{L@2I?^2YAQccw<+DD3b81kcS$){*`-LS^CNra3wug|(=(=h#1G1d#qZQO zG0vdB4b(vM2~;Z1K5*aPc&geLU+VoYs=x^S=YbNF?YC4t!LfUclSadbI($t>%vByP z?6d#akcVWv%!(%dK#9eBX@lTg*I0qeyyWs4sK@Wu#t_ZW4u~#@`0HOxq~9M+GY)sb zgfSK!NcU^86IN&EK|5_<)?0PMwe;p9Jz;X)RTqK*rV656I&ze>qM1Nvv%vM`|3YhV za`g@Kkv})Bu^^?yR1qsIuW9Zf@)f3D{=gyV@)p3xbFL^@+}pl<49Ido#7ZTMU`MVm{@&|N75E^|QXnqnWX6#uWI=Wz=Pv`%X}RZ;(=jP6Nl=||707Ot z9t;fHJQZ?Gx9o^}k4u0DhhD|xFDaRm2&zOFzen!_f{OX6*RwIC_eG2L514m6V5MNm z8#EsyD?@oiD9M5H1zI3yXYrrOmoX>@ZE-<%&Dr^@wP2%~1B>P$T1&D8hTvmG>L-A7 z<~f%qwSYannoDb#cjY}bnXLi$BGUG#Tth6HQ3_cXNCTK-4BJsqYVCQD(WBba?uISs z=l5;Ta|ntmEaEpFwLgH|&x?H*2{RF(MOs>rng&l|`)Vtfi7@KBXiU-|U$@@Y&faX0 z{f}&@g>HIWYGxYL^AnlEge#i8iY(o?{=}V4JI1yal%5j6?Hl%NJeT4Q-fD+$10YL< zv!P}63)h&W>Y!W`U=~xHb5jbwwP8Kl0rGI(6I1;e|TDKc+m3P6{VW@>8g ziKH!-gBa8xqv^ND9UpjuBtsH-ZB!=CaygJx;k(tx47^xJf@_L=2n0l8d-9H~NO$n! zOd0_>)4O(G$pA&-pkZj~wV)7KJ&fpb4Q%Q`&ogZ0FpU0wU;ov=*S?%7^MaWJ1cC@i zAY4zSG}QITmPatF4uqtyr*0LzfG5|jz3@}vcnM58ZPbi^CB>6(AOktfa5)hF+ynG3 zIKS@7r;M+RF8yeC?ZG?VO1`>IBfO=pPSH859k&a^Hu|HY>x5=YY6??xrp)&L+2iox zZ4wH}XDCJsei1t1$24=GppmKzC2g}AJD5-@vLHaJ=zCE~ORhyuy(=n$`q_322)ua4 z=~J9`;?OF*I_In&-F=Tctn){+G#`7yo1kex?{6tCZEcVDrM3@rTrR35esl-O~ zf3^UqX%VOqJe0Oc!FgmfGBOuvu6#f=tjJGL{zP9>pJ0~on+55k3wbg#%E3PyqqtJlL0`H1)YPI(Cm?;qOO1%Y;J?4!QPfm4Y|WVAyk z^W1`wE-5cXo}GdnCFnhoSd=|$ZHu$Vfy^?~y7!Cd<0*c*@m83?99FyR{01yMhsHcp z^B0-Sy7I1uhLxK4)tq1e2;5hS{eJ3rh7tK7n$wr+Ko|y#Rf55!CM%YG60dW6^c(|_ z(dQ?{uFoxR8O=i5;FPdwH6Cy0+fxRx^u#zvfdW&T%cbhqgik#5aAEA74>At++%mPX z<*BHU(-w+SMr%ro^H{`p{AA4++f(6<*Y9(Va|2G!3dqG#2wn(V6 zSS*iK49BDq1}~-K{FdytIR%2F&d|H>m~|t zet{B6K@-*Cekf)mVzzg9#nEFmI2hBnVP~}6&g0dK!-|lFJ>n?2yc}mtSqZga)v6HB z?UdhY0bk5GN`X+w)~)H$Bgu)~+W?@a5jA&8tBOlJt1JSUO?lkt1-xO*p;j3NdwOXEk_oLj@xW~fv7P%zC>FX5X^BlkiTdXjZ{1mazEm9cdX+KOnIeWBl z-GHPB#RBHYljEjtHI3i&EFb)H;O0Mp=XLUQ(<#uhiLC+!GL_(Fn<7Pq8CVBxsqqA`VKr7}L^39&Ppkp`6=h z#?|%pg^3Pg3}-B;=oI*JjSrVuKP-F$HrL5%1u9*2#I7Eq!z@KzbZhEUj3c0I=}Q$F zqJ;GAGGcCk%BB*9RHVK@l1cf-kR_^ zkTAKq_jZoN1cz!vWZ%{UigrL_0B+6(?R=1gGd|}f)hz42L>CmEexU?LcbIsC++Jjd zh2BGA-T;-U6=}cr`jr!i%)C6nK~^x zTaR!AEaAZfmEr`5vVE8jk}@@S7Z+sfIErjPj52vn@S^k54<9#xr&-d)Z+uLc^g%wk zpmy~A7Ii>FzJc^~eUbeq<${qECn(nO;>Z2dCj?__GOCCdu;BQrvs~A^LT`bpKYBp0 z$L#){*tAcwa(TFNxHuZ>*~Ps;T*mZ8;H$|LBi8z)4a52yRSNz0T|X@_Ql#~Wtm7vU z=DxU*3h$fkS8R6naDh>VS;8@FfUwR6BZBnm?OR9;`YeS2hZ}+tRG5SW!y|6b&30*Q z6d7N@r073!4rS;JszUVqShf^zmEq27TAm97l0Wk+A0MK2ZExq z(Yb>0CkVQkuVC0-RNt+D?R1jwjKIJ=upRnVPhMEbVnY70EZ0vA?8J9Xi17_NZk^6^ zE0U%WJ>pKY%J9X2N`rCq0bf_?SMzK@n?;TPJ*-b&&T0MmIWBygp(qKE!{PZ2caROf zq#pp&Se))>DLy;Jg^w8@T+9VU%+#nm?{Lym$ORZu!rVGw<9GJU?)Zp+UgeN68n8TV zmFv4Ux*0eu7lk~#8E}0W;EiiHtI9qaRZI>b64z)i6zWbVl}!T1CU84l9>!EfF((cbQF&m3W-$|eFk8?#;H@xmtDqh0c}zOdRW zUn=uo_ObQ)jJ~ktA3lh=WrgL9f&o3bQ4OitIh zSrXuv)EZeOc{o07VH&_GF~c&1%)l#UrP;yW0T(xUgPz%j1~L76r@Wg+2ZB3eJEpUx zkdE@(qdF&6)RN%r=q#tXfRQ($X*60FmPbUfN~Taor!bPe@dI2)s`=BD@s|+6=T*?0FKV;( zS8@vn+%JFcbPFm@+dr$?BW3{WSEc${xc`$vi+vuS;%PqD3^*gBu)dm|U%b>VFyG2! zi_}DkFtTUJlOqOQD4U+wrxI{^My&BF%I>tDT}WmlbTetjYYYI-p7*Wsf1m)?6`#k| zWHbHdr7nMyM0wCs;i#!Xxr_U!4@VtiL(sBJMbmH9}y)q3?(rAO07>ReLwerUer=zH* zHH~p7Ho=1(YlpT9UZLojL`e}svZr4+SmYcH>u!fF)`SO!o;mH5vPzP@C7@ntQ_Ir* zm#zcjxHGhEr-Q~KLuxcK{I3FqSyUipu4!LPcF&t-%)bq$w)o=>E#*(I15M?*l`vHh zP>%tfV|Bi;Kygm7>ftUPxX&Ei%WA?Z+SL)^6;&6JmVccC9BX#rYc`{EJxvoQcst+E zR(G-C=2d6&8ddJ!c^`6$Hv)QF`Tl%@j_SOqpSdFi0E7l*@+aE6!?P-UU)@`iW8^X+i^fJPFDK&l zGZWOlXF2pG%C16~XmhdOLvcx;~Z|88W+3Bn5 z=JY?Qhcs&b(420C09!lR!em5VO z*ViYNf{=7vg1wZ53EK(QU22~Ed6E@I`$<-VmehIsYdqBUSt{* zIqFbus(LDOzM=|sMPqLdZ^*Td^K0W}l~zh~Gd~w`+v{=w@W7}JF_+y)M~cw9QF1iA z0A+S*1xRt77V%w9d6&~~<`A{af3rbf$^_-=j?Ty!J78QPLMhTd)Ld?jy@$^wP&{mG z*dh+CNM9d!nBT!9BpcG4Ro_C42#M4CAH-Ws?v)iNjlg}m;V37Jb7?_IG7Ip1g7{BZ z6&Cy5X0VT-Ck-O|M)g$P*3R#!U_vybA=@V9twzHJ*g2hINA_X)BKR~X;U5=}OP>fX zAPT!*gxmWsL0c4kKEftfXNrRlU-&l>bfLKuH=<|a{mDW;SGgle`VTW?Ffz1I2fc)u zQoshx8}8|j793THJIel4w%hZ~jAJXJI83@NTgF{nouK=LC6W~mWf`RF((qQuN`7RL_Bu8-;3Eb9|I`xMq7tzzYSv1A#e zpcmE!gpNmmjjD1N+b9my_+Quh z{RV-BPHrBLlz=&$K2^O?Ex-dF?H?#{STMzWBSh!Kpv?9$Tt1@v6JL{`hT(Szg==<{ zxOXN%3CPd7l_@AU6gxt77klv%@+* zSX-nbXBzNL-1P-9;EXiPB}c3qgfs3=-sZ(;%6jsx`IvAJEWcg?-rX8Zax1tJ z3lfcq3OMY-UyxNi-{8-3&D@t~w>*idevp#%8UuuuUtA-}E82gzMnnziw0Pzpmb?NU zI+#X2#y=+65Pc&Xb*9|e@H7h-P?0;d4DkYO8%_?}b?mi1 z2LYwLU$ZEDk0u0F{#oRVTbAqTi$C`?S%U%4S?_EY+MOvaN{VC-8Pz&?@MR%!U3SV@ zf`){>3`BhvvF{n5A}p-2M!6en*32ZH{m~hDdYNw$|5)rBPHQii5tqw_Q#99T^oNJK z{S8Y)ye)mfkm4kmiXlz6eQ7$qiYniu@y z&IZ9SoJ|#70^-dK224aAOpTFFm&77&$)xfF?CrttavW9|SdXZT zGE;lC*+%fY40`FY|3aM>>6E@2rD^@0vwD7e0Y8u%kT5W4_uXe>lSF%+>W_X0TvsXv z2D{SQJT!lbe$6%;8fM!;H${Tw1Trmk-BUqPA!)&6Bft(`LUOOsh>m<7S(P7c6_3-1 ztb6%R>hdelm|4VCAYKt-WGhJ(>=f90{$&lAQ~;A0Y8c0ML}WptxzA=8kc2Z4+(hn0 zd5`>~CDl@E6+|u*Z@mJrQ$^YaPW%QO#jodInFc8DunetFWAab-kxXqt8abqkXg9*% zq-||_O7ZvuEl zyS__QQbuD?D$!A9bMR^&#TcS#63@&EtIr_zaZ{v^sf+FS2jHLeXe~-iOp`OR&v4_R zk>>C=U1!%TRxRHzDRM0%8H!{9@Dn#+d`II0jw6&BD=}+KX>gZ+uqWy+B)F{*6y-YFzg2-@~Rx0w{PI z^>X^ptpv@FLulbSJeX(g;A3103o0S`J$XqEV2zm>=d&jCD-X&$4+`=S{T;z`q&rXk zR_aWDSJ~-B;>&Gc=1;$GTGA`$`lp=r*@ge4QMvK-d_A>hifQ!P=_ZpxmCs*##Q_8ej7^&EicBC!fa;+$tum5)6 zAUI2T2)|{aiiX2yTr7d!;2RaMP~f73T9wOJ(|_{c|F5RA4ruayySPD2x*KG)D4io^ zv`BYKN_T@O;7BP+LAp~KBqgOox?8$IB=x=L_xC>k;f7;R+;Lr>bB?&2y+^>gWlE8z z9C%f?1IT9-C4Jk6OX%h6J>?+dk4d9=po@sz>_o0wQI2uNuOl}XQtAa?<4N~}`z;gf znM_6~y6cwmek=;dXw2Le2hYD|B7hr+pY@KPzCN7e!D_rgOA+=ZPOHg?KMAjiDt=#$ zvNvG0Ju5`@=DFnu#;K1V_|YaOY9Vpy0)WGB^Gv+U)0~%uSYo!)wu}@A=_SS!!81Y! z`EE=L{Va1GD`DKMv$ET6pY$eVI@_%0LbcN46c~}z{r42V2^yO~K6lwq4*i%W`kt@G zV=Oer`?-06O9ZI{))?0bZCTE4<`?OhFAbmLGTkSbMX>pLQTK4QCwU*Kzq+TxhRPQX zXu~Y0GQ%qK!{LEoY35UNtfvW~!(Y;|Z|E%cb)*DMJ18kjj*MG33oWfq#g6nFU$a}X zGj?KK)B5yvrxp$v1xtT|n*8?{1J_uDveG3?=bp}3Dbj0$)+AhWtm`2= zQSm%5?*P4a!^^?HVXI@|aP@=9ghoV{j{=X?FE& zIL8?=Xx*zxgx2zxa!(n6m0t|tv&=~Ze~MrEpyURP{<7lUI();j@C&{~W8-r5n?~ht|pioHlF&Am5vO5c4FyZzI@AhlS+Fuv~{_==ycww6e91U177qBz7 zzXZ!hg5vi~Pj9CVB76IvXf@0GKS0Fr_+^*B%gk!2Z_(G%R4{V!=rMp0yRi+gp)gqji&@MlJkYDo*E^Q&0{$#E zni5Wc$Q@R^B8)D;c@@KSYdm~#sa`>q=QF>Y&mOk25HHTGOQR_t5?U+@=i`s$20}6E zqK$ZUxF){!Q#SGE8%)$b3$zv$FM&Fu+&*?n8W>vO07&}w}*SYM-vWMSqXdA!c7sL%HQ6^_Xf~un3 zN56LUfE$v%|LF&Z62N^1dOlb~DmT{63N&s$cXyb{{|^Dy(+yq`mQ_qdr5S`Zbb~7A)tb(ipC>3 zaMP+Cw>LTa&cYj6=jkTh&G2lV!mjcVFH6X(mJj8kAhLD`5sM^6rSdiZ{D3mbw(k&* zE&=bM+Q>qC+kCdL)Vg>Ti_}IzgMk_M=AyF{KTAuF{MSdVtnsxk>v%(P6IUPpC;pR- zObOch<2Ayfn6bUPJ9|V^W@21mvQlWWB2}n{p?&blcvob2#<_gHft|RP5STduGn<*Q zx%>AsIHzCvm00>!q*k%mMWioQQ8VAtjT=UHDc2sK29)ar5j?JD+?ad7J?&nd_P1|H`UrK^cSwRzca3UK=-#M216UAgv5o zHg){S+6_{%_d|g5PWfBbhP+`=&pG&v1h~?~cH!m(jKM?76yiWf`&o@jT0i25Ba!maYx-e<5`C%Fq z?e%^P&~tZr1TYeFh}IEO_CJKm^N?Pb&XMP<=-;PRh4dkOI_eQ19y`-p>QMPN(^!a& zJ%y+%On1EPqaCaO2T2WD8-MHkMPYW*+@Z!a=#dI$AXwDvSUgtobSZEcU!|>ZJsv-pBHp@FQ*Cf*W^$u^C2x2Mp@#BiF0~ z?SR@J-J%_v!>wn^Hr@5*=ICq!rc|vX z(auUQP~rrIv({h7MaVyYs^%iKuNawFQX}`b?VgeqKBYP<0aAli5g)#(RMb$Gl|vc@ zx5QnpRS17!d!FQ#qtyY3Eud74n5?Gc(dX{jombSS=xdOoC}0;t!c71|v{vvU-?Nsp ztsEfl;M!XL{=o)cjFK5GScob81wW~>A(a@22l+N)uT*!80m?u-_V%9H>L2}lx`Xii z6`sr&s>BtyGss*06GwBurUCz4y;v-zr!B` zE2VVx6{ca#RNm=DU^u``Gb$?BZ9FRxAhBsXPrk;Mxffgf=h65xk(THsytfbU6Hz^r zLn>yS`I|N=j|N4SZt%tYjD*@JR8z9391o*&Uwws{o=mM%u~$W<*sKPf!`b{@6Qr`k zvn-6oJq=lyX;Q$F%9_0CGz?=myU5Y9l^))o z73XK{f#q0g$))R9U=HDs0X2@;iaF-rLM-!Ii>daq@MmHSs*s-`)BZmx5WD!@8sU}( zkOGY}YaL*%^Jrqh1&2BY=${P%3_W$U@%75(FW|L!AeYk`vYJML9sxG(u(Jh zMIX}|3NWyZuIJ+?D$N0t5<044dJt%;$aLq9(*sbuy9jXhQFs7cE{upyF_{NNAtAA% zYQg!xD~Ui=d7wV%AMELm$0`(kY@(J9wAT?oWA|hqKx>?F<$p!?Z*p^E?(q4BWdK_4 zajicdsKi#D__jsLKA-mw0jN$Sc*N)&k9T>$>V8+cd2{Gunn{*i3#6jWMIRC?DKjd6 z^m1unKNSq3RMy~knFrh$D9eEDq>Rv@5XJ0+gcnJfUP^WM*aU)-%W+h!RtmiyLm&Se z0$K^54!F5p4C-x%!P%x;m8)VH5t+VL6g4p}o6iExdzkbg5}#-Rk_pzFJHz77B(;EG z*UIzw2bRJdNBsJrDqK*x7}$SsnKbcYrCruvLGh3CQ_wsHvEadPKkmG?joM~A-Q8C6 z!nflGa%1fyb8AZ2hylBI5b7K~0?#Ej3`si3sW9A~Te!?E1i`m~{m;Wrza>xR5!a9p zPlQ=tKonCiZWh)Jln5K_5B`u*)5~54jTwb!sX!#v0M`gq&{XBm-FiM>4QsqX#n3o( zT6I%!tk2@Jm@%k6N?ldU=!m8G2fIX9PoYOrT^jGW}=T^O4c`Gzbp>rChHlORcx1`idF80hnuz9jFIZ`&NGq z$9xGwG9}%7a4S{>J#Z4;O+NsY~4i>KDLVY;J}8;&CX*U^qAXQJR{%Y$)&*SqNL zYg!7~jxjB;6hVUov|em+yMRqZG|yRbQ7?fT9^77{?wCd_FL?9*vM}f!@MQFlaRQNg z%)W{;FvqwOgC_vTo9P+gBZg0@J@eGMx_V;&Hq15E&HLFGrh4o6ti2x_Fxx0(#qAoi zd+okA=<9>wvzOJWG+p#?I1FV2ATy2{7c}MbfSzztdM)R?>4gDJ_b#|m1crrdX4B+a zF4x~y6s;*5BnIwszHZ{czq(8pfse@eMxu51cvIj>gCsIrR^W_cQ)Yu?;)FVA^|6n7 zCJ8+$VCh91lz3h9V&5ijf6^M2rw1hfgO+46S%1iX=r1H}KnjN+&Dz@`BQ zV#&h%tO3(K2+}~~@cFr*Sx-bk5YqLAz!nJA4i@k3nnUs%A2$XcF$}eyk-{17z_1oK zxVVBQPsI5L9$M((yOeITnrF5PGkmFqyeQ;}tpM)644U9gBPKA40j@+|xxg=huK3X? z{Owz4tOh6sdUZpG7t~T{pz?Dv7mutLIM^QcL!j0aj2Z$aGV%EE%?^YLDV;%Jf3>?0>HqWWb9>c$z=8)BT~pmxe-4r8 zEj|hF)RP%jC&BqVh~&u@hUa!zQ{$XpxAJtMyre~(io$~{4ni@|bY(t24g@=fq$8p3 zN_Sp?S;J9&7rEg#Y?m(pFU*>ua^!N{@5Mv>Phn9zy~RGJuqLOF4k*g3Ai|-Q#U>XY zKO~@r-~qc0YmIWIVQeBtkF9auexy0mfvM6b%6pE{gb9LApuA4Jy52J!vGb-la7dQ2 z^Aip3B8S`E+&HOm!B1K7fL~_9b6l5vIso; zu7D%h2{AxOWad;|H*{5>09?50bfFe%_RG$lEl^Oe5ZEe(+Wfh$sPyqbQZ2&Xp#*Tz z5Lihdu>8r2Dhi4GdpG*2K*ale->gJbpvpxWt}{1dfuAr?s{RZ=*E$}Ic6v|)@0Hi1 zJNq+V;nLpx7u_9c)36aVic-;UBLc%*EBT4g3F$vCg@aOLB;>ktMLRQ4(g$$*yi5n%NRJGtC= z>je$QxGQ8QGyb%~66}Ms{&Q*2cKn-jo8N46Xy^42ueEqb=k0Y+O*{W{U=?=z!n#wf z+qsQLCLK@h3la69icn!|Tu!KCC7JAX2t<|S(tgHy( zh3zL7oeM8aG{Pa!)5_ZMNuZIei4@)VGJ>K;la*<$@rIKNs0st^PFP^dm*f+`vKqN& zL?6(3)jxePl8x(obM33zr=I$Avdq>lgfDWp9QJMa&v07qT5)6DI#<|Sye!MRSiyJE zc!?&90c7^{=w3twTUlU4JUV_^LS0aUjh_(BAo24lB8I)6bfP|o$z9H-Y3&FfgISOQ zBBW)(|NfB(yeHCQ3sx5eP<4wJkT{~Hm{jbKc>O`yJaA+lAaQw z?@AM5J8hxpegLjS6vpe`vXg2B_o)9`C5VF9#CFa(Aft}KcQ{b##VdncTSg}YDUIRw! z3h3+Dc;v)~<$<474}P@@caF)p@go(!gdFIQ)eMUR*%3mUhW(r*AABHZ&*estoR?K+~3A6yJcxc%Gd+^;gc2HX73IauO|B=W7s?Ua) zx|rX%a?u9Jgv#dpQq1B|o1I!DO;o#DanCvSg3v+tcXBQYVtBzw zAsuw=v|(riZ*e(;0M>74iYhNrJLXUf6QS~UoF&yKE+M36SAjaF!}YogaBP=jz($!n zkzx(zXBxdlEhWvCM(;XFhhte55p2z=L$X4c-rq-<>DP7#mJeez0~E(kNUz+f&E9Z} z9wlHzGX{`!0JDq2DSU$B=;h%$Z%PE-aE7hZ3p)cSk?I?zy8~I+x+0ZD#HBGyy){us z-Rvqim0$oyzh3)`FL?0zsr_gm$wDa8(X%$d!EMH_^M?p$!Nj^1TM>}irQQgjnmOY~ z*uYha^4fkzgk-|ZV0?b2NR&`t*7dz&MASz~y}Hr0yo8!JyD3`pOtAWR0r$UC{N8>M z=c+lnz#r>aj^kr%b8a@AjGVgMY>nSGSD`cjlBX3hS)bH`C;T`XZ!$YbpDrZTr%&|+ z^s%CthIFX){; zu;@el=&I_Pmi(cCf_;WgFY55tJf--SCUggSH7_FEficdq_P$V#wK49;xTl6yHxQGM zBsAQQQP2V2rfo?7F~-hVZA^7Yr~wqA{w>C3H!HA;iMf}!!%yt@Yc+jWm}0^-BfB=_ z07s15fF=|R#JUH{VVfIlS2pEJ*H6!PZF5}s7&4?_ayvi)F}E_9}!In z`iN0;viYHi^B<^^HE}^izN*|XT6-g#&_tBuY0nUlcR+Tv3Tz377M!9kt;BFFAi#5N zu3yv=AT3P8Tea5-fNTILQTuoh25U{>#vrj}O~hhkBKivh+XKjCV;^Bjx5|$E_Zi`y=&4!NBumml3u-v`&0A_N)w)iKWI6xmX z%YMp}D(XRy7QfQoEUaRIFCIZ{u2SXNtTCHUYP>ko_lzWfAJKsmF(^X>GS=XnMubC$ zdp^8;wZf*$Av97}r8jd>a7$q8f>2R47hxOYu1VrYLy*94+63!Y%yCxRbnQfB z=l~nbvm|zaU4R#eq!>%4)#*$c zYZY6*CkuvQX1y}MZgy`6n)nob1aN@+XIM}v?mPtBfq%-hfxgkyfh~6URmFGzjyVV1 z#&b3X*E;Ns3#$}7_~FlIa2KZjy_GzeJ6E33SYehj2ScwyqZKq>#^yA5@KLS{%pxn3 zZ{~=dS^pAI&~9&7&)l4&KIO06z*4wJwUu#jqcXFv^!zz3DajtLs_?|w;Wgcc@|-il z5msO~Fk2RUXHPpGl&U&+nqVoD7T?Z2l;$_=`GQ>()WVV78l=;n6M#TsYDr7rEaf(- zO+xp4zJuZ$PpJ|pxiP-KJtMSvnrb_n+IlWHQn=Aoy^oUdv6WYoEJ12559muheZRYa zcyp(*i7Oi4#bNi~5l*4hk?E}gYr(_U9(i<7+RIGiEkC32(d2L9vb`CNBoa=k7v9Tj zdd)Fik>`c~Vf~>s6;W5kw&xH6vEmNim)6NxLh}tLD_R!4^vKat&KgQ{Dzm<5Qvypq0KeIKxCNh2fs^dqt2@H z3c>|uHrof}kcpo=VT%YrODJ->_xtHG0i*e#f4xPdQAtc3E;x0)tOmQZweMExULBcr zZccD4db}x39~ub!xX&&5{Mab_-6hViT}e&*@xh?WUeVcErh;JKb5@#SAv;psnHnom)c zafWSvOv+v%1r&PJsV1vaJ-*=|cr|L>j#`Yoev;QKB>S(VbryOfKo2yL5=T9UE}ruK za~7rc4eO&F%x1xL3le*r5 zq@4+6L<-)6KUo4r76wGA%(A^9K!3kWCS27sa4C~JRB?_I-Z(%0G*%SJG|ztWM1e#8 zjtMYmNiGMt(Uznlg*N;ivc!F!g`Y>~c1{OTX*f6T923FifuFI!^MHL@X%@ zS-W%t54N-p#STcH>@zJKx#~|3!o+?tHbK>Op)X_VyXFQ`qSTW$e;tG?syr>R`^3Mj zm}EoEsFW36B>Z<22ogrcHy)Yetp5y{s$-s;)8xD#{4bVph!Swr=Lzdqi;+(5UOL3mUX`y;%RfBM4VMCez?xTM>Gf)@SxwX@fUkpW* zW%g+ae2T=A2pi}a$&=D>BU-2woQ2e~s`&3t@=5cHB75){gJ=!H!aEM(Uu*O|L?F#3 zfQ$&Q10OgSd3k0v_LJOi_YC`yDkgEv|2+5ruA=SuPC4T|U$ud(T?8KJNmOR&CvdhK z_FMUw#&{x8hu(`)1AX4qS{!K||uRpH;?%#&!sS*_ZeNqb(@*9)pJp27cw= z#g`-z$M5}56-xV(WiOB2*BiIBSr;0TR?1NS%cObu9`u)!()$TPi+x(f1KzU0IOTJV zrq^VCs4Q{ZiVRP>K}4m$onc?S-|ib(?-s@v=!uU9H%~G|&XAC$C&*6Y(v<8N$i*nS z5#X2KGc|e?9ny9InXK)iXcrDS&HqySqkQ=FBm|i#09it9E_8V1vnyQKG=W7P|D2LQB$G*7$hVj zz0rK5J6IW!!&ot?%J*NS8B@DxVsCXN4P-!Cy(b!{4e-|hDXi+qw0bnKxkCncfS~)~ z001e|k^=pXuIa)1=gP~^t4hRZo$%2~iLy2F+%5UedO0AU=m8YOOX1-) ze~3B22Nj>c|0+H^;1dkpKCg62sQPWicPit2PxzPW0Iq>~i}mFBP;nCXT1#0Ihe++N z>f5+~JE@l(_E)`JwO@^N8n5_2Er7h?}Q%Zod=-KMmnfugCTAde(USViQXKO{uTC z4Y=g77kb+&D|Q}ygi_;YMn#-2IpthRm6?RyHh(Y=MrpC7`N^1!%w+vN?`J7J%9RX= z$qiqgWvo_)7|ZvTk^XuN!}cvNOF9+ymAa7u^g)%RxW@}Wu9^qnp%Yj7lSch?0!EC_ z{oeO+v%|)M&$_p&ML|ua?b@*vC25SIP|dsEwzjtKXXUe5p&|`m$tCXY-Ycg0QJ01= z(0Uc%#~QQ`@2;$({5UM-D4G9u(r+A+LgPC%1FbOx9vkYo;wB-xmzJ8m9}_=VzF@@*MaMWRgvP-7@0sIu1df$D8trSvz;S$49&n98=v=>ps2vN)wm-;5m$ zXBR}-tY%09&WR>e`N_G~JNrKU@kb#&S_Yvz1gJ$EO_;};is(y2oV+2>ns9Gd^Q6fw zqe{s#j?L=nHy>V>n1=*5!_2L+M?4laj z2@!ogEgcqo>1NA$U1^mtJQ2IufZw@<$=wGLsrY+c(YqxrW_i)rvA1sSu>nzJvo;Y^ z)6#vqHPwo{jQo>I^c!Jeb^O=F)l)=t?S1>BS06_*G-FlUQI4Q0asYuK1bM|mt`sBwJhv5e zS?L%vwi-(O(AnBsT+Klq@ECNedI`N$_+cF%Bz3dak}xG%dT~%Uq{Kvs%q78ntuiUi zE=;6y;@j73kkt!f&L~4I-IZQ%1+!{IEvaMDLS0Nk@fK8q=twa&nOu|#vsHAE12%F9 zmIljbq6v4cdKYbD(nruYlXVWaIjI$300BpHLZQ{+pP46<*J`c*hJ?WqhYZq>`rW(t zL3kKAyd2wXX;yJPkkbtEIAXqpK>v-cV02vy93ikp4oKh#{7nMf-;uv^nW;KI#Up%K zgl@}6OhE^I$!2{OX|MKUH1Nq}nW*>V0oE&xdI8)aBlNkZ8>30L3S}0@T~oAFkKfoE z(>R!H*4yUMvSl_quR{XaBwe)wfG1rNUpDPR0)yPMdvC6bs4biy z5neaey>E$7ko4ssl!|L1QwBbpGjX0=pLywAa&d)gGQY&NpxjT7^RtaTI68}xJY$B>+r8G)(>tW%{Jxe@OSJ9qjDEA`9#YT z`VPOjlvC$fK}+eO@oV{q_NK~4Kb~FF^7U8~Pe;8GUzqbnmMXQ)Fk#(r>Sk^r; z_AOVQ4+{5su~MZ<9Ecey4m+;syj=1lR^Gi4^VslGy3k|bU9mW}#2jPpS@Pq~jq9 zw=m1!f0&vY+&gRWxFPCnBi;lbP3=RZlO3NC!cgQSWnU#m_Zim84DYRC**qOqZYzY@ zWka8dC?nWxJcfLR=kUaNHp`gsX~AkAY?drJ#2Z{4SxDiXmo5hr4a|i!@Z{cpO&-XJ z*f{!n%Ot~Na2S8v{Ul+PdAKOaRSJ_Umq_5h7BD+;vbO9xI#|ENbMRv^s=<56;MdV^@=;m*RH_C|O7;lMZu=^1-=1C2 zgs&DQ?5l9{>Vh|k?S5TVcJRK1Rlmi?N|unr5q8rXB_z5hLWG^gF_3TcXEgZTLcf0 zzN#*p@8{{H`&*)*H$}NRM&@N(FyDO{^yse1SBN0IZzOs^PyBJZ6T& z056{eaH8>+ac0x08~4vK^CL1hxkRmWUXNl9nwC8dXK_0`yXnd zpzG+?&rp4G-(aI*V{S(qt@q5lY&)+b_$i+oxgJobOWJ*4OX8)UN zR}_`%V!th3!pF5u<&Y%Wy${*=x5sp^A5$k2#r>lg%Owel)eQMwa-n zQaXY`gx&G(yKo%%Iun>4{B^CDDbyI>Ng+vVU1yyaRRdyw#1eFeq;NVh5@LeC~azipZmRfqr&Ji$E*7 zh9Hv%58a-*&%|ZCQyZ`%Zf!3rweesn#hriD=33>5T}3Jx_?fcHhvV4fEqz?x@!y7h z3J)te7gTlM=W|PgCE{evjZ`}$?5u@CToSyqm;-e0GJV={f>e#usXj{j&SXLR682LB zS=Bko_X$;Sl1IV@ofw>TUi-81B;WrkB;vBcn1A6vLZLm{6?N+bhZWZkc3*mcTNkx_ zYlao%GlX{h6#4LM?k-OBall zpQ%EdeI_U$9cQyR?d@>s8w$a6Tt&JhdFOv7eTVSFBdI0OTssI zxv&YUhy?>>U3bVdp1^Ka~?l<)ZMGxShW>oGj7e1r&t&lALImL6J!)Dlv)v2K3f!#F8&>B<1 zZHK*S{0X^J0LL3Ex9P-jiQeWW4f&T|4?}g3) literal 0 HcmV?d00001 diff --git a/docs/conf.py b/docs/conf.py index dc6d9e8..53b3dad 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -20,7 +20,7 @@ # -- Project information ----------------------------------------------------- project = u'BlobPorter' -#copyright = u'2018, Jesus Aguilar' +copyright = u'2018, BlobPorter Contributors' author = u'BlobPorter Contributors' # The short X.Y version @@ -74,7 +74,7 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'alabaster' +html_theme = 'sphinx_rtd_theme' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the diff --git a/docs/examples.rst b/docs/examples.rst new file mode 100644 index 0000000..186dab3 --- /dev/null +++ b/docs/examples.rst @@ -0,0 +1,118 @@ +======== +Examples +======== + +Upload to Azure Block Blob Storage +------------------------------------------- + +Single file upload: + +``./blobporter -f /datadrive/myfile.tar -c mycontainer -n myfile.tar`` + + **Note:** If the container does not exist, it will be created. + +Upload all files that match the pattern: + +``./blobporter -f "/datadrive/*.tar" -c mycontainer`` + +You can also specify a list of files or patterns explicitly: + +``./blobporter -f "/datadrive/*.tar" -f "/datadrive/readme.md" -f "/datadrive/log" -c mycontainer`` + +If you want to rename the target file name, you can use the -n option: + +``./blobporter -f /datadrive/f1.tar -n newname.tar -c mycontainer`` + +Upload to Azure Page Blob Storage +-------------------------------------- + +Same as uploading to block blob storage, but with the transfer definiton (-t option) set to ``file-pageblob``. + +For example, a single file upload to page blob: + +``./blobporter -f /datadrive/mydisk.vhd -c mycontainer -n mydisk.vhd -t file-pageblob`` + + **Note:** The file size and block size must be a multiple of 512 (bytes). The maximum block size is 4MB. + +Transfer Data from an S3 endpoint to Azure Storage +-------------------------------------------------- + +You can upload data from an S3 compatible endpoint. + +First you must specify the access and secret keys via environment variables. + +:: + + export S3_ACCESS_KEY= + export S3_SECRET_KEY= + +Then you can specify an S3 URI, with the following format: + +``[HOST]/[BUCKET]/[PREFIX]`` + +For example: + +``./blobporter -f s3://mys3api.com/mybucket/mydata -c froms3 -t s3-blockblob`` + + **Note:** For better performance, consider running this tranfer from a high-bandwidth VM running in the same region as source or the target. Data is uploaded as it is downloaded from the source, therefore the transfer is bound to the bandwidth of the VM for performance. + Synchronously Copy data between Azure Blob Storage targets and sources + + +Transfer Data Between Azure Storage Accounts, Containers and Blob Types +----------------------------------------------------------------------- + +First, you must set the account key of the source storage account. + +``export SOURCE_ACCOUNT_KEY=`` + +Then you can specify the URI of the source. The source could be a page, block or append blob. Prefixes are supported. + +``./blobporter -f "https://mysourceaccount.blob.core.windows.net/container/myblob" -c mycontainer -t blob-blockblob`` + + **Note:** For better performance, consider running this tranfer from a high-bandwidth VM running in the same region as source or the target. Data is uploaded as it is downloaded from the source, therefore the transfer is bound to the bandwidth of the VM for performance. + Synchronously Copy data between Azure Blob Storage targets and sources + + +Transfer from an HTTP/HTTPS source to Azure Blob Storage +-------------------------------------------------------- + +To block blob storage: + +``./blobporter -f "http://mysource/file.bam" -c mycontainer -n file.bam -t http-blockblob`` + +To page blob storage: + +``./blobporter -f "http://mysource/my.vhd" -c mycontainer -n my.vhd -t http-pageblob`` + + **Note:** For better performance, consider running this tranfer from a high-bandwidth VM running in the same region as source or the target. Data is uploaded as it is downloaded from the source, therefore the transfer is bound to the bandwidth of the VM for performance. + Synchronously Copy data between Azure Blob Storage targets and sources + +Download from Azure Blob Storage +-------------------------- + +For download scenarios, the source can be a page, append or block blob: + +``./blobporter -c mycontainer -n file.bam -t blob-file`` + +You can use the -n option to specify a prefix. All blobs that match the prefix will be downloaded. + +The following will download all blobs in the container that start with f: + +``./blobporter -c mycontainer -n f -t blob-file`` + +Without the -n option all files in the container will be downloaded. + +``./blobporter -c mycontainer -t blob-file`` + +By default files are downloaded keeping the same directory structure as the remote source. + +If you want download to the same directory where you are running blobporter, set -i option. + +``./blobporter -p -c mycontainer -t blob-file -i`` + +Download a file from a HTTP source +---------------------------------- + +``./blobporter -f "http://mysource/file.bam" -n /datadrive/file.bam -t http-file`` + + **Note:** The ACCOUNT_NAME and ACCOUNT_KEY environment variables are not required in this scenario. diff --git a/docs/gettingstarted.rst b/docs/gettingstarted.rst new file mode 100644 index 0000000..f672d03 --- /dev/null +++ b/docs/gettingstarted.rst @@ -0,0 +1,38 @@ +=============== +Getting Started +=============== + +Linux +----- + +Download, extract and set permissions + +:: + + wget -O bp_linux.tar.gz https://github.com/Azure/blobporter/releases/download/v0.6.09/bp_linux.tar.gz + tar -xvf bp_linux.tar.gz linux_amd64/blobporter + chmod +x ~/linux_amd64/blobporter + cd ~/linux_amd64 + +Set environment variables: :: + + export ACCOUNT_NAME= + export ACCOUNT_KEY= + +**Note:** You can also set these values via `options `__ + +Windows +------- + +Download `BlobPorter.exe `_ + +Set environment variables (if using the command prompt): :: + + set ACCOUNT_NAME= + set ACCOUNT_KEY= + +Set environment variables (if using PowerShell): :: + + $env:ACCOUNT_NAME="" + $env:ACCOUNT_KEY="" + diff --git a/docs/index.rst b/docs/index.rst index 123ef7b..f9d3ee7 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -3,18 +3,19 @@ You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -Welcome to BlobPorter's documentation! +BlobPorter ====================================== +BlobPorter is a data transfer tool for Azure Blob Storage that maximizes throughput through concurrent reads and writes that can scale up and down independently. + +.. image :: bptransfer.png + .. toctree:: :maxdepth: 2 :caption: Contents: - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` + gettingstarted + examples + performance/perfmode + resumable_transfers + options diff --git a/docs/options.rst b/docs/options.rst new file mode 100644 index 0000000..6fe4fd0 --- /dev/null +++ b/docs/options.rst @@ -0,0 +1,52 @@ +=============== +Command Options +=============== + + -f, --source_file (string) URL, Azure Blob or S3 Endpoint, file or files (e.g. /data/\*.gz) to upload. + -c, --container_name (string) Container name (e.g. mycontainer). + -n, --blob_name (string) Blob name (e.g. myblob.txt) or prefix for download scenarios. + -g, --concurrent_workers (int) Number of go-routines for parallel upload. + -r, --concurrent_readers (int) Number of go-routines for parallel reading of the input. + -b, --block_size (string) Desired size of each blob block. + Can be specified as an integer byte count or integer suffixed with B, KB or MB (default /"4MB/", maximum /"100MB/"). + The block size could have a significant memory impact. + If you are using large blocks reduce the number of readers and workers (-r and -g options) to reduce the memory pressure during the transfer. + For files larger than 200GB, this parameter must be set to a value higher than 4MB. + The minimum block size is defined by the following formula: + Minimum Block Size = File Size / 50000 + The maximum block size is 100MB + + -a, --account_name (string) Storage account name (e.g. mystorage). + + Can also be specified via the ACCOUNT_NAME environment variable. + + -k, --account_key (string) Storage account key string. + + Can also be specified via the ACCOUNT_KEY environment variable. + -s, --http_timeout (int) HTTP client timeout in seconds. Default value is 600s. + -d, --dup_check_level (string) Desired level of effort to detect duplicate data blocks to minimize upload size. + + Must be one of None, ZeroOnly, Full (default "None") + -t, --transfer_type (string) Defines the source and target of the transfer. + + Must be one of file-blockblob, file-pageblob, http-blockblob, http-pageblob, blob-file, pageblock-file (alias of blob-file), blockblob-file (alias of blob-file), http-file, blob-pageblob, blob-blockblob, s3-pageblob and s3-blockblob. + -m, --compute_blockmd5 (bool) If set, block level MD5 has will be computed and included as a header when the block is sent to blob storage. + + Default is false. + -q, --quiet_mode (bool) If set, the progress indicator is not displayed. + + The files to transfer, errors, warnings and transfer completion summary is still displayed. + -x, --files_per_transfer (int) Number of files in a batch transfer. Default is 500. + -h, --handles_per_file (int) Number of open handles for concurrent reads and writes per file. Default is 2. + -i, --remove_directories (bool) If set blobs are downloaded or uploaded without keeping the directory structure of the source. + + Not applicable when the source is a HTTP endpoint. + -o, --read_token_exp (int) Expiration in minutes of the read-only access token that will be generated to read from S3 or Azure Blob sources. + + Default value: 360. + -l, --transfer_status (string) Transfer status file location. + If set, blobporter will use this file to track the status of the transfer. + + In case of failure and the same file is referrenced, the source files that were transferred will be skipped. + + If the transfer is successful a summary will be appended. \ No newline at end of file diff --git a/docs/perfmode.rst b/docs/performance/perfmode.rst similarity index 65% rename from docs/perfmode.rst rename to docs/performance/perfmode.rst index 4d6d3fd..7d37088 100644 --- a/docs/perfmode.rst +++ b/docs/performance/perfmode.rst @@ -1,33 +1,28 @@ +================ Performance Mode -====================================== +================ + BlobPorter has a performance mode that uploads random data generated in memory and measures the performance of the operation without the impact of disk i/o. -The performance mode for uploads could help you identify the potential upper limit of throughput that the network and the target storage account can provide. +The performance mode for uploads could help you identify the potential upper limit of the data-transfer throughput that your environment can provide. For example, the following command will upload 10 x 10GB files to a storage account. -``` -blobporter -f "1GB:10" -c perft -t perf-blockblob -``` +``blobporter -f "1GB:10" -c perft -t perf-blockblob`` You can also use this mode to see if increasing (or decreasing) the number of workers/writers (-g option) will have a potential impact. -``` -blobporter -f "1GB:10" -c perft -t perf-blockblob -g 20 -``` +``blobporter -f "1GB:10" -c perft -t perf-blockblob -g 20`` Similarly, for downloads, you can simulate downloading data from a storage account without writing to disk. This mode could also help you fine-tune the number of readers (-r option) and get an idea of the maximum download throughput. The following command downloads the data previously uploaded. -``` -export SRC_ACCOUNT_KEY=$ACCOUNT_KEY -blobporter -f "https://myaccount.blob.core.windows.net/perft" -t blob-perf -``` +``export SRC_ACCOUNT_KEY=$ACCOUNT_KEY`` + +``blobporter -f "https://myaccount.blob.core.windows.net/perft" -t blob-perf`` -Then you can download to disk. +Then you can download the file to disk. -``` -blobporter -c perft -t blob-file -``` +``blobporter -c perft -t blob-file`` The performance difference will you a measurement of the impact of disk i/o. \ No newline at end of file diff --git a/docs/resumable_transfers.rst b/docs/resumable_transfers.rst index 9c63324..1ed6faf 100644 --- a/docs/resumable_transfers.rst +++ b/docs/resumable_transfers.rst @@ -1,58 +1,48 @@ Resumable Transfers ====================================== -BlobPorter supports resumable transfers. To enable this feature you need to set the -l option with a path to the transfer status file. +BlobPorter supports resumable transfers. To enable this feature you need to set the -l option with a path to the transfer status file. In case of failure, you can reference the same status file and BlobPorter will skip files that were already transferred. -`` -blobporter -f "manyfiles/*" -c "many" -l mylog -`` +``blobporter -f "manyfiles/*" -c many -l mylog`` -The status transfer file contains entries for when a file is queued and when it was succesfully tranferred. +For each file in the transfer two entries will be created in the status file. One when file is queued (Started) and another when the file is successfully transferred (Completed). The log entries are created with the following tab-delimited format: -`` -[Timestamp] [Filename] [Status (1:Started,2:Completed,3:Ignored)] [Size] [Transfer ID ] -`` - -The following output from a transfer status file shows that three files were included in the transfer (file10, file11 and file15). -However, only two were successfully transferred: file10 and file11. - -`` -2018-03-05T03:31:13.034245807Z file10 1 104857600 938520246_mylog -2018-03-05T03:31:13.034390509Z file11 1 104857600 938520246_mylog -2018-03-05T03:31:13.034437109Z file15 1 104857600 938520246_mylog -2018-03-05T03:31:25.232572306Z file10 2 104857600 938520246_mylog -2018-03-05T03:31:25.591239355Z file11 2 104857600 938520246_mylog -`` - -In case of failure, you can reference the same status file and BlobPorter will skip files that were already transferred. - -Consider the previous scenario. After executing the transfer again, the status file has entries only for the missing file (file15). - -`` -2018-03-05T03:31:13.034245807Z file10 1 104857600 938520246_mylog -2018-03-05T03:31:13.034390509Z file11 1 104857600 938520246_mylog -2018-03-05T03:31:13.034437109Z file15 1 104857600 938520246_mylog -2018-03-05T03:31:25.232572306Z file10 2 104857600 938520246_mylog -2018-03-05T03:31:25.591239355Z file11 2 104857600 938520246_mylog -2018-03-05T03:54:33.660161772Z file15 1 104857600 495675852_mylog -2018-03-05T03:54:34.579295059Z file15 2 104857600 495675852_mylog -`` - -When the transfer is sucessful, a summary is created at the end of the transfer status file. - -`` ----------------------------------------------------------- -Transfer Completed---------------------------------------- -Start Summary--------------------------------------------- -Last Transfer ID:495675852_mylog -Date:Mon Mar 5 03:54:34 UTC 2018 -File:file15 Size:104857600 TID:495675852_mylog -File:file10 Size:104857600 TID:938520246_mylog -File:file11 Size:104857600 TID:938520246_mylog -Transferred Files:3 Total Size:314572800 -End Summary----------------------------------------------- -`` +``[Timestamp] [Filename] [Status (1:Started,2:Completed,3:Ignored)] [Size] [Transfer ID ]`` + +The following output from a transfer status file shows that three files were included in the transfer: **file10** , **file11** and **file15** . +However, only **file10** and **file11** were successfully transferred. For **file15** the output indicates that it was queued but the lack of a second entry confirming completion (status = 2), indicates that the transfer process was interrupted. :: + + 2018-03-05T03:31:13.034245807Z file10 1 104857600 938520246_mylog + 2018-03-05T03:31:13.034390509Z file11 1 104857600 938520246_mylog + 2018-03-05T03:31:13.034437109Z file15 1 104857600 938520246_mylog + 2018-03-05T03:31:25.232572306Z file10 2 104857600 938520246_mylog + 2018-03-05T03:31:25.591239355Z file11 2 104857600 938520246_mylog + +Consider the previous scenario and assume that the transfer was executed again. +In this case, the status file shows two new entries for **file15** in a new transfer (the transfer ID is different) which is an indication that this time the file was transferred successfully. :: + + 2018-03-05T03:31:13.034245807Z file10 1 104857600 938520246_mylog + 2018-03-05T03:31:13.034390509Z file11 1 104857600 938520246_mylog + 2018-03-05T03:31:13.034437109Z file15 1 104857600 938520246_mylog + 2018-03-05T03:31:25.232572306Z file10 2 104857600 938520246_mylog + 2018-03-05T03:31:25.591239355Z file11 2 104857600 938520246_mylog + 2018-03-05T03:54:33.660161772Z file15 1 104857600 495675852_mylog + 2018-03-05T03:54:34.579295059Z file15 2 104857600 495675852_mylog + +Finally, since the process completed successfully, a summary is appended to the transfer status file. :: + + ---------------------------------------------------------- + Transfer Completed---------------------------------------- + Start Summary--------------------------------------------- + Last Transfer ID:495675852_mylog + Date:Mon Mar 5 03:54:34 UTC 2018 + File:file15 Size:104857600 TID:495675852_mylog + File:file10 Size:104857600 TID:938520246_mylog + File:file11 Size:104857600 TID:938520246_mylog + Transferred Files:3 Total Size:314572800 + End Summary----------------------------------------------- + From 598818d6bc0b6587b3ea64b67721a17545b9c36b Mon Sep 17 00:00:00 2001 From: Jesus Aguilar <3589801+giventocode@users.noreply.github.com> Date: Sat, 10 Mar 2018 10:04:24 -0500 Subject: [PATCH 6/7] - fix use exact name not working - documentation updates --- README.md | 4 +- args.go | 8 +- docs/conf.py | 4 +- docs/examples.rst | 127 +++++++++++++----- docs/gettingstarted.rst | 51 ++++++- docs/index.rst | 10 +- docs/options.rst | 52 ------- docs/perfmode.rst | 72 ++++++++++ docs/performance/perfmode.rst | 28 ---- ...mable_transfers.rst => resumabletrans.rst} | 15 ++- internal/const.go | 2 +- 11 files changed, 243 insertions(+), 130 deletions(-) delete mode 100644 docs/options.rst create mode 100644 docs/perfmode.rst delete mode 100644 docs/performance/perfmode.rst rename docs/{resumable_transfers.rst => resumabletrans.rst} (85%) diff --git a/README.md b/README.md index 9a221e1..f05b061 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Sources and targets are decoupled, this design enables the composition of variou Download, extract and set permissions: ```bash -wget -O bp_linux.tar.gz https://github.com/Azure/blobporter/releases/download/v0.6.10/bp_linux.tar.gz +wget -O bp_linux.tar.gz https://github.com/Azure/blobporter/releases/download/v0.6.12/bp_linux.tar.gz tar -xvf bp_linux.tar.gz linux_amd64/blobporter chmod +x ~/linux_amd64/blobporter cd ~/linux_amd64 @@ -46,7 +46,7 @@ export ACCOUNT_KEY= ### Windows -Download [BlobPorter.exe](https://github.com/Azure/blobporter/releases/download/v0.6.10/bp_windows.zip) +Download [BlobPorter.exe](https://github.com/Azure/blobporter/releases/download/v0.6.12/bp_windows.zip) Set environment variables (if using the command prompt): diff --git a/args.go b/args.go index 530d76e..d790497 100644 --- a/args.go +++ b/args.go @@ -174,7 +174,8 @@ func (p *paramParserValidator) parseAndValidate() error { p.pvgDupCheck, p.pvgParseBlockSize, p.pvgQuietMode, - p.pvgKeepDirectoryStructure) + p.pvgKeepDirectoryStructure, + p.pvgUseExactMatch) if err != nil { return err @@ -257,6 +258,11 @@ func (p *paramParserValidator) getSourceRules() ([]parseAndValidationRule, error //************************** //Global rules.... +func (p *paramParserValidator) pvgUseExactMatch() error { + p.params.useExactMatch = p.args.exactNameMatch + return nil +} + func (p *paramParserValidator) pvgTransferStatusPathIsPresent() error { if p.args.transferStatusPath != "" { diff --git a/docs/conf.py b/docs/conf.py index 53b3dad..c1b6301 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -15,7 +15,7 @@ # import os # import sys # sys.path.insert(0, os.path.abspath('.')) - +import sphinx_bootstrap_theme # -- Project information ----------------------------------------------------- @@ -75,6 +75,8 @@ # a list of builtin themes. # html_theme = 'sphinx_rtd_theme' +#html_theme = 'bootstrap' +#html_theme_path = sphinx_bootstrap_theme.get_html_theme_path() # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the diff --git a/docs/examples.rst b/docs/examples.rst index 186dab3..ef15ce2 100644 --- a/docs/examples.rst +++ b/docs/examples.rst @@ -1,43 +1,52 @@ -======== Examples ======== + Upload to Azure Block Blob Storage -------------------------------------------- +----------------------------------- Single file upload: -``./blobporter -f /datadrive/myfile.tar -c mycontainer -n myfile.tar`` +:: + + ./blobporter -f /datadrive/myfile.tar -c mycontainer -n myfile.tar - **Note:** If the container does not exist, it will be created. +.. note:: BlobPorter will create the container if it doesn't exist. Upload all files that match the pattern: -``./blobporter -f "/datadrive/*.tar" -c mycontainer`` +:: + + ./blobporter -f "/datadrive/*.tar" -c mycontainer You can also specify a list of files or patterns explicitly: -``./blobporter -f "/datadrive/*.tar" -f "/datadrive/readme.md" -f "/datadrive/log" -c mycontainer`` +:: + + ./blobporter -f "/datadrive/*.tar" -f "/datadrive/readme.md" -f "/datadrive/log" -c mycontainer If you want to rename the target file name, you can use the -n option: -``./blobporter -f /datadrive/f1.tar -n newname.tar -c mycontainer`` +:: + + ./blobporter -f /datadrive/f1.tar -n newname.tar -c mycontainer Upload to Azure Page Blob Storage --------------------------------------- +---------------------------------- -Same as uploading to block blob storage, but with the transfer definiton (-t option) set to ``file-pageblob``. +Same as uploading to block blob storage, but with the transfer definiton (-t option) set to ``file-pageblob``. For example, a single file upload to page blob: -For example, a single file upload to page blob: +:: -``./blobporter -f /datadrive/mydisk.vhd -c mycontainer -n mydisk.vhd -t file-pageblob`` + ./blobporter -f /datadrive/mydisk.vhd -c mycontainer -n mydisk.vhd -t file-pageblob - **Note:** The file size and block size must be a multiple of 512 (bytes). The maximum block size is 4MB. -Transfer Data from an S3 endpoint to Azure Storage --------------------------------------------------- +.. note:: The file size and block size must be a multiple of 512 (bytes). The maximum block size is 4MB. -You can upload data from an S3 compatible endpoint. +Transfer data from S3 to Azure Storage +--------------------------------------- + +You can transfer data from an S3 compatible endpoint. First you must specify the access and secret keys via environment variables. @@ -48,71 +57,117 @@ First you must specify the access and secret keys via environment variables. Then you can specify an S3 URI, with the following format: -``[HOST]/[BUCKET]/[PREFIX]`` +:: + + [HOST]/[BUCKET]/[PREFIX] For example: -``./blobporter -f s3://mys3api.com/mybucket/mydata -c froms3 -t s3-blockblob`` +:: - **Note:** For better performance, consider running this tranfer from a high-bandwidth VM running in the same region as source or the target. Data is uploaded as it is downloaded from the source, therefore the transfer is bound to the bandwidth of the VM for performance. - Synchronously Copy data between Azure Blob Storage targets and sources + ./blobporter -f s3://mys3api.com/mybucket/mydata -c froms3 -t s3-blockblob +.. note:: -Transfer Data Between Azure Storage Accounts, Containers and Blob Types + BlobPorter will upload the data as it downloads it from the source. + The performance of the transfer will be constraint by the bandwidth of the host running BlobPorter. Consider running this type of transfer from a Virtual Machine running in the same Azure region as the target or the source. + +Transfer data between Azure Storage accounts, containers and blob types ----------------------------------------------------------------------- First, you must set the account key of the source storage account. -``export SOURCE_ACCOUNT_KEY=`` +:: + + export SOURCE_ACCOUNT_KEY= + Then you can specify the URI of the source. The source could be a page, block or append blob. Prefixes are supported. -``./blobporter -f "https://mysourceaccount.blob.core.windows.net/container/myblob" -c mycontainer -t blob-blockblob`` +:: + + ./blobporter -f "https://mysourceaccount.blob.core.windows.net/container/myblob" -c mycontainer -t blob-blockblob - **Note:** For better performance, consider running this tranfer from a high-bandwidth VM running in the same region as source or the target. Data is uploaded as it is downloaded from the source, therefore the transfer is bound to the bandwidth of the VM for performance. - Synchronously Copy data between Azure Blob Storage targets and sources +.. note:: + BlobPorter will upload the data as it downloads it from the source. + The performance of the transfer will be constraint by the bandwidth of the host running BlobPorter. Consider running this type of transfer from a Virtual Machine running in the same Azure region as the target or the source. Transfer from an HTTP/HTTPS source to Azure Blob Storage -------------------------------------------------------- To block blob storage: -``./blobporter -f "http://mysource/file.bam" -c mycontainer -n file.bam -t http-blockblob`` +:: + + ./blobporter -f "http://mysource/file.bam" -c mycontainer -n file.bam -t http-blockblob To page blob storage: -``./blobporter -f "http://mysource/my.vhd" -c mycontainer -n my.vhd -t http-pageblob`` +:: + + ./blobporter -f "http://mysource/my.vhd" -c mycontainer -n my.vhd -t http-pageblob - **Note:** For better performance, consider running this tranfer from a high-bandwidth VM running in the same region as source or the target. Data is uploaded as it is downloaded from the source, therefore the transfer is bound to the bandwidth of the VM for performance. - Synchronously Copy data between Azure Blob Storage targets and sources +.. note:: + + BlobPorter will upload the data as it downloads it from the source. + The performance of the transfer will be constraint by the bandwidth of the host running BlobPorter. Consider running this type of transfer from a Virtual Machine running in the same Azure region as the target or the source. Download from Azure Blob Storage --------------------------- +-------------------------------- For download scenarios, the source can be a page, append or block blob: -``./blobporter -c mycontainer -n file.bam -t blob-file`` +:: + + ./blobporter -c mycontainer -n file.bam -t blob-file You can use the -n option to specify a prefix. All blobs that match the prefix will be downloaded. The following will download all blobs in the container that start with f: -``./blobporter -c mycontainer -n f -t blob-file`` +:: + + ./blobporter -c mycontainer -n f -t blob-file + Without the -n option all files in the container will be downloaded. -``./blobporter -c mycontainer -t blob-file`` +:: + + ./blobporter -c mycontainer -t blob-file + By default files are downloaded keeping the same directory structure as the remote source. -If you want download to the same directory where you are running blobporter, set -i option. +If you want download to the same directory where you are running blobporter set -i option. + +:: + + ./blobporter -c mycontainer -t blob-file -i + + +For scenarios where blob endpoint is from a soverign cloud (e.g. China and Germany), Azure Gov or Azure Stack, you can specify the fully qualified domain name: + +:: + + ./blobporter -f "https://[ACCOUNT_NAME].[BASE_URL]/[CONTAINER_NAME]/[PREFIX]" -t blob-file + +And the source account key, must be set via an environment variable. + +:: + + export SOURCE_ACCOUNT_KEY= + -``./blobporter -p -c mycontainer -t blob-file -i`` Download a file from a HTTP source ---------------------------------- -``./blobporter -f "http://mysource/file.bam" -n /datadrive/file.bam -t http-file`` +:: + + ./blobporter -f "http://mysource/file.bam" -n /datadrive/file.bam -t http-file + +.. note:: - **Note:** The ACCOUNT_NAME and ACCOUNT_KEY environment variables are not required in this scenario. + The ACCOUNT_NAME and ACCOUNT_KEY environment variables are not required. diff --git a/docs/gettingstarted.rst b/docs/gettingstarted.rst index f672d03..95448cd 100644 --- a/docs/gettingstarted.rst +++ b/docs/gettingstarted.rst @@ -9,7 +9,7 @@ Download, extract and set permissions :: - wget -O bp_linux.tar.gz https://github.com/Azure/blobporter/releases/download/v0.6.09/bp_linux.tar.gz + wget -O bp_linux.tar.gz https://github.com/Azure/blobporter/releases/download/v0.6.12/bp_linux.tar.gz tar -xvf bp_linux.tar.gz linux_amd64/blobporter chmod +x ~/linux_amd64/blobporter cd ~/linux_amd64 @@ -24,7 +24,7 @@ Set environment variables: :: Windows ------- -Download `BlobPorter.exe `_ +Download `BlobPorter.exe `_ Set environment variables (if using the command prompt): :: @@ -36,3 +36,50 @@ Set environment variables (if using PowerShell): :: $env:ACCOUNT_NAME="" $env:ACCOUNT_KEY="" + +Command Options +--------------- + + -f, --source_file (string) URL, Azure Blob or S3 Endpoint, file or files (e.g. /data/\*.gz) to upload. + -c, --container_name (string) Container name (e.g. mycontainer). + -n, --blob_name (string) Blob name (e.g. myblob.txt) or prefix for download scenarios. + -g, --concurrent_workers (int) Number of go-routines for parallel upload. + -r, --concurrent_readers (int) Number of go-routines for parallel reading of the input. + -b, --block_size (string) Desired size of each blob block. + + Can be specified as an integer byte count or integer suffixed with B, KB or MB. + + -a, --account_name (string) Storage account name (e.g. mystorage). + + Can also be specified via the ACCOUNT_NAME environment variable. + + -k, --account_key (string) Storage account key string. + + Can also be specified via the ACCOUNT_KEY environment variable. + -s, --http_timeout (int) HTTP client timeout in seconds. Default value is 600s. + -d, --dup_check_level (string) Desired level of effort to detect duplicate data blocks to minimize upload size. + + Must be one of None, ZeroOnly, Full (default "None") + -t, --transfer_type (string) Defines the source and target of the transfer. + + Must be one of file-blockblob, file-pageblob, http-blockblob, http-pageblob, blob-file, pageblock-file (alias of blob-file), blockblob-file (alias of blob-file), http-file, blob-pageblob, blob-blockblob, s3-pageblob and s3-blockblob. + -m, --compute_blockmd5 (bool) If set, block level MD5 has will be computed and included as a header when the block is sent to blob storage. + + Default is false. + -q, --quiet_mode (bool) If set, the progress indicator is not displayed. + + The files to transfer, errors, warnings and transfer completion summary is still displayed. + -x, --files_per_transfer (int) Number of files in a batch transfer. Default is 500. + -h, --handles_per_file (int) Number of open handles for concurrent reads and writes per file. Default is 2. + -i, --remove_directories (bool) If set blobs are downloaded or uploaded without keeping the directory structure of the source. + + Not applicable when the source is a HTTP endpoint. + -o, --read_token_exp (int) Expiration in minutes of the read-only access token that will be generated to read from S3 or Azure Blob sources. + + Default value: 360. + -l, --transfer_status (string) Transfer status file location. + If set, blobporter will use this file to track the status of the transfer. + + In case of failure and the same file is referrenced, the source files that were transferred will be skipped. + + If the transfer is successful a summary will be appended. diff --git a/docs/index.rst b/docs/index.rst index f9d3ee7..f3fd7f3 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -8,6 +8,9 @@ BlobPorter BlobPorter is a data transfer tool for Azure Blob Storage that maximizes throughput through concurrent reads and writes that can scale up and down independently. +:: + + .. image :: bptransfer.png .. toctree:: @@ -16,6 +19,7 @@ BlobPorter is a data transfer tool for Azure Blob Storage that maximizes through gettingstarted examples - performance/perfmode - resumable_transfers - options + resumabletrans + perfmode + + diff --git a/docs/options.rst b/docs/options.rst deleted file mode 100644 index 6fe4fd0..0000000 --- a/docs/options.rst +++ /dev/null @@ -1,52 +0,0 @@ -=============== -Command Options -=============== - - -f, --source_file (string) URL, Azure Blob or S3 Endpoint, file or files (e.g. /data/\*.gz) to upload. - -c, --container_name (string) Container name (e.g. mycontainer). - -n, --blob_name (string) Blob name (e.g. myblob.txt) or prefix for download scenarios. - -g, --concurrent_workers (int) Number of go-routines for parallel upload. - -r, --concurrent_readers (int) Number of go-routines for parallel reading of the input. - -b, --block_size (string) Desired size of each blob block. - Can be specified as an integer byte count or integer suffixed with B, KB or MB (default /"4MB/", maximum /"100MB/"). - The block size could have a significant memory impact. - If you are using large blocks reduce the number of readers and workers (-r and -g options) to reduce the memory pressure during the transfer. - For files larger than 200GB, this parameter must be set to a value higher than 4MB. - The minimum block size is defined by the following formula: - Minimum Block Size = File Size / 50000 - The maximum block size is 100MB - - -a, --account_name (string) Storage account name (e.g. mystorage). - - Can also be specified via the ACCOUNT_NAME environment variable. - - -k, --account_key (string) Storage account key string. - - Can also be specified via the ACCOUNT_KEY environment variable. - -s, --http_timeout (int) HTTP client timeout in seconds. Default value is 600s. - -d, --dup_check_level (string) Desired level of effort to detect duplicate data blocks to minimize upload size. - - Must be one of None, ZeroOnly, Full (default "None") - -t, --transfer_type (string) Defines the source and target of the transfer. - - Must be one of file-blockblob, file-pageblob, http-blockblob, http-pageblob, blob-file, pageblock-file (alias of blob-file), blockblob-file (alias of blob-file), http-file, blob-pageblob, blob-blockblob, s3-pageblob and s3-blockblob. - -m, --compute_blockmd5 (bool) If set, block level MD5 has will be computed and included as a header when the block is sent to blob storage. - - Default is false. - -q, --quiet_mode (bool) If set, the progress indicator is not displayed. - - The files to transfer, errors, warnings and transfer completion summary is still displayed. - -x, --files_per_transfer (int) Number of files in a batch transfer. Default is 500. - -h, --handles_per_file (int) Number of open handles for concurrent reads and writes per file. Default is 2. - -i, --remove_directories (bool) If set blobs are downloaded or uploaded without keeping the directory structure of the source. - - Not applicable when the source is a HTTP endpoint. - -o, --read_token_exp (int) Expiration in minutes of the read-only access token that will be generated to read from S3 or Azure Blob sources. - - Default value: 360. - -l, --transfer_status (string) Transfer status file location. - If set, blobporter will use this file to track the status of the transfer. - - In case of failure and the same file is referrenced, the source files that were transferred will be skipped. - - If the transfer is successful a summary will be appended. \ No newline at end of file diff --git a/docs/perfmode.rst b/docs/perfmode.rst new file mode 100644 index 0000000..a66e09e --- /dev/null +++ b/docs/perfmode.rst @@ -0,0 +1,72 @@ +Performance Consideration +========================= + +Best Practices +-------------- + + +- By default, BlobPorter creates 5 readers and 8 workers for each core on the computer. You can overwrite these values by using the options -r (number of readers) and -g (number of workers). When overriding these options there are few considerations: + + - If during the transfer the buffer level is constant at 000%, workers could be waiting for data. Consider increasing the number of readers. If the level is 100% the opposite applies; increasing the number of workers could help. + + - Each reader or worker correlates to one goroutine. Goroutines are lightweight and a Go program can create a high number of goroutines, however, there's a point where the overhead of context switching impacts overall performance. Increase these values in small increments, e.g. 5. + +- For transfers from fast disks (SSD) or HTTP sources reducing the number readers or workers could provide better performance than the default values. Reduce these values if you want to minimize resource utilization. Lowering these numbers reduces contention and the likelihood of experiencing throttling conditions. + +- Transfers can be batched. Each batch transfer will concurrently read and transfer up to 500 files (default value) from the source. The batch size can be modified using the -x option. + +- Blobs smaller than the block size are transferred in a single operation. With relatively small files (<32MB) performance may be better if you set a block size equal to the size of the files. Setting the number of workers and readers to the number of files could yield performance gains. + +- The block size can have a significant memory impact if set to a large value (e.g. 100MB). For large files, use a block size that is close to the minimum required for the transfer and reduce the number of workers (g option). + + The following table list the maximum file size for typical block sizes. + + =============== =================== + Block Size (MB) Max File Size (GB) + =============== =================== + 8 400 + 16 800 + 32 1600 + 64 3200 + =============== =================== + +Performance Measurement Mode +---------------------------- + +BlobPorter has a performance measurement mode that uploads random data generated in memory and measures the performance of the operation without the impact of disk i/o. +The performance mode for uploads could help you identify the potential upper limit of the data throughput that your environment can provide. + +For example, the following command will upload 10 x 10GB files to a storage account. + +:: + + blobporter -f "1GB:10" -c perft -t perf-blockblob + +You can also use this mode to see if increasing (or decreasing) the number of workers/writers (-g option) will have a potential impact. + +:: + + blobporter -f "1GB:10" -c perft -t perf-blockblob -g 20 + +Similarly, for downloads, you can simulate downloading data from a storage account without writing to disk. This mode could also help you fine-tune the number of readers (-r option) and get an idea of the maximum download throughput. + +The following command downloads the data previously uploaded. + +:: + + export SRC_ACCOUNT_KEY=$ACCOUNT_KEY + +.. + +:: + + blobporter -f "https://myaccount.blob.core.windows.net/perft" -t blob-perf`` + + +Then you can download the file to disk. + +:: + + blobporter -c perft -t blob-file + +The performance difference will provide with a base-line of the impact of disk i/o. \ No newline at end of file diff --git a/docs/performance/perfmode.rst b/docs/performance/perfmode.rst deleted file mode 100644 index 7d37088..0000000 --- a/docs/performance/perfmode.rst +++ /dev/null @@ -1,28 +0,0 @@ -================ -Performance Mode -================ - -BlobPorter has a performance mode that uploads random data generated in memory and measures the performance of the operation without the impact of disk i/o. -The performance mode for uploads could help you identify the potential upper limit of the data-transfer throughput that your environment can provide. - -For example, the following command will upload 10 x 10GB files to a storage account. - -``blobporter -f "1GB:10" -c perft -t perf-blockblob`` - -You can also use this mode to see if increasing (or decreasing) the number of workers/writers (-g option) will have a potential impact. - -``blobporter -f "1GB:10" -c perft -t perf-blockblob -g 20`` - -Similarly, for downloads, you can simulate downloading data from a storage account without writing to disk. This mode could also help you fine-tune the number of readers (-r option) and get an idea of the maximum download throughput. - -The following command downloads the data previously uploaded. - -``export SRC_ACCOUNT_KEY=$ACCOUNT_KEY`` - -``blobporter -f "https://myaccount.blob.core.windows.net/perft" -t blob-perf`` - -Then you can download the file to disk. - -``blobporter -c perft -t blob-file`` - -The performance difference will you a measurement of the impact of disk i/o. \ No newline at end of file diff --git a/docs/resumable_transfers.rst b/docs/resumabletrans.rst similarity index 85% rename from docs/resumable_transfers.rst rename to docs/resumabletrans.rst index 1ed6faf..0fcbb83 100644 --- a/docs/resumable_transfers.rst +++ b/docs/resumabletrans.rst @@ -1,14 +1,21 @@ Resumable Transfers -====================================== -BlobPorter supports resumable transfers. To enable this feature you need to set the -l option with a path to the transfer status file. In case of failure, you can reference the same status file and BlobPorter will skip files that were already transferred. +=================== -``blobporter -f "manyfiles/*" -c many -l mylog`` +BlobPorter supports resumable transfers. This feature is enabled when the -l option is set. This option must specify the path where the transfer status file will be created. +In case of failure, if the same status file is specified, BlobPorter will skip files that were already transferred. + +:: + + blobporter -f "manyfiles/*" -c many -l mylog For each file in the transfer two entries will be created in the status file. One when file is queued (Started) and another when the file is successfully transferred (Completed). The log entries are created with the following tab-delimited format: -``[Timestamp] [Filename] [Status (1:Started,2:Completed,3:Ignored)] [Size] [Transfer ID ]`` +:: + + [Timestamp] [Filename] [Status (1:Started,2:Completed,3:Ignored)] [Size] [Transfer ID ] + The following output from a transfer status file shows that three files were included in the transfer: **file10** , **file11** and **file15** . However, only **file10** and **file11** were successfully transferred. For **file15** the output indicates that it was queued but the lack of a second entry confirming completion (status = 2), indicates that the transfer process was interrupted. :: diff --git a/internal/const.go b/internal/const.go index 1c137a2..7530994 100644 --- a/internal/const.go +++ b/internal/const.go @@ -6,7 +6,7 @@ import ( ) //ProgramVersion blobporter version -const ProgramVersion = "0.6.11" +const ProgramVersion = "0.6.12" //HTTPClientTimeout HTTP client timeout when reading from HTTP sources and try timeout for blob storage operations. var HTTPClientTimeout = 90 From b067032fd540f945919a25a728fc1f3bc8193371 Mon Sep 17 00:00:00 2001 From: Jesus Aguilar <3589801+giventocode@users.noreply.github.com> Date: Sat, 10 Mar 2018 13:55:55 -0500 Subject: [PATCH 7/7] - doc updates --- docs/gettingstarted.rst | 23 ++++++++++++++++------- docs/resumabletrans.rst | 2 +- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/docs/gettingstarted.rst b/docs/gettingstarted.rst index 95448cd..2e1709c 100644 --- a/docs/gettingstarted.rst +++ b/docs/gettingstarted.rst @@ -19,7 +19,9 @@ Set environment variables: :: export ACCOUNT_NAME= export ACCOUNT_KEY= -**Note:** You can also set these values via `options `__ +.. note:: + + You can also set these values via options Windows ------- @@ -40,7 +42,9 @@ Set environment variables (if using PowerShell): :: Command Options --------------- - -f, --source_file (string) URL, Azure Blob or S3 Endpoint, file or files (e.g. /data/\*.gz) to upload. + -f, --source_file (string) URL, Azure Blob or S3 Endpoint, + file or files (e.g. /data/\*.gz) to upload. + -c, --container_name (string) Container name (e.g. mycontainer). -n, --blob_name (string) Blob name (e.g. myblob.txt) or prefix for download scenarios. -g, --concurrent_workers (int) Number of go-routines for parallel upload. @@ -57,13 +61,18 @@ Command Options Can also be specified via the ACCOUNT_KEY environment variable. -s, --http_timeout (int) HTTP client timeout in seconds. Default value is 600s. - -d, --dup_check_level (string) Desired level of effort to detect duplicate data blocks to minimize upload size. - - Must be one of None, ZeroOnly, Full (default "None") -t, --transfer_type (string) Defines the source and target of the transfer. - Must be one of file-blockblob, file-pageblob, http-blockblob, http-pageblob, blob-file, pageblock-file (alias of blob-file), blockblob-file (alias of blob-file), http-file, blob-pageblob, blob-blockblob, s3-pageblob and s3-blockblob. - -m, --compute_blockmd5 (bool) If set, block level MD5 has will be computed and included as a header when the block is sent to blob storage. + Must be one of :: + + file-blockblob, file-pageblob, http-blockblob, + http-pageblob, blob-file, pageblock-file (alias of blob-file), + blockblob-file (alias of blob-file), http-file, + blob-pageblob, blob-blockblob, s3-pageblob and s3-blockblob. + + + -m, --compute_blockmd5 (bool) If set, block level MD5 has will be computed and included + as a header when the block is sent to blob storage. Default is false. -q, --quiet_mode (bool) If set, the progress indicator is not displayed. diff --git a/docs/resumabletrans.rst b/docs/resumabletrans.rst index 0fcbb83..846078d 100644 --- a/docs/resumabletrans.rst +++ b/docs/resumabletrans.rst @@ -1,7 +1,7 @@ Resumable Transfers =================== -BlobPorter supports resumable transfers. This feature is enabled when the -l option is set. This option must specify the path where the transfer status file will be created. +BlobPorter supports resumable transfers. This feature is enabled when the -l option is set with the path where the transfer status file will be created. In case of failure, if the same status file is specified, BlobPorter will skip files that were already transferred. ::