Skip to content

Commit

Permalink
DataMovement Share Destination Checkpoint Data Update (#39764)
Browse files Browse the repository at this point in the history 
* new file checkpointer data

* updated tests and constants

* remove accidental using

* Trigger Build
  • Loading branch information
@jaschrep-msft
jaschrep-msft authored Nov 13, 2023
1 parent 371c357 commit 801dd7e
Show file tree
Hide file tree
Showing 6 changed files with 254 additions and 60 deletions.
61 changes: 46 additions & 15 deletions sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/DataMovementShareConstants.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,23 +24,54 @@ internal class DestinationCheckpointData
{
internal const int SchemaVersion = 1;

private const int VersionEncodedSize = IntSizeInBytes;
private const int FileAttributesEncodedSize = OneByte + IntSizeInBytes;
private const int FilePermissionKeyOffsetEncodedSize = IntSizeInBytes;
private const int FilePermissionKeyLengthEncodedSize = IntSizeInBytes;
private const int FileCreatedOnEncodedSize = OneByte + LongSizeInBytes + LongSizeInBytes;
private const int FileLastWrittenOnEncodedSize = OneByte + LongSizeInBytes + LongSizeInBytes;
private const int FileChangedOnEncodedSize = OneByte + LongSizeInBytes + LongSizeInBytes;
private const int ContentTypeOffsetEncodedSize = IntSizeInBytes;
private const int ContentTypeLengthEncodedSize = IntSizeInBytes;
private const int ContentEncodingOffsetEncodedSize = IntSizeInBytes;
private const int ContentEncodingLengthEncodedSize = IntSizeInBytes;
private const int ContentLanguageOffsetEncodedSize = IntSizeInBytes;
private const int ContentLanguageLengthEncodedSize = IntSizeInBytes;
private const int ContentDispositionOffsetEncodedSize = IntSizeInBytes;
private const int ContentDispositionLengthEncodedSize = IntSizeInBytes;
private const int CacheControlOffsetEncodedSize = IntSizeInBytes;
private const int CacheControlLengthEncodedSize = IntSizeInBytes;
private const int FileMetadataOffsetEncodedSize = IntSizeInBytes;
private const int FileMetadataLengthEncodedSize = IntSizeInBytes;
private const int DirectoryMetadataOffsetEncodedSize = IntSizeInBytes;
private const int DirectoryMetadataLengthEncodedSize = IntSizeInBytes;

internal const int VersionIndex = 0;

internal const int ContentTypeOffsetIndex = VersionIndex + IntSizeInBytes;
internal const int ContentTypeLengthIndex = ContentTypeOffsetIndex + IntSizeInBytes;
internal const int ContentEncodingOffsetIndex = ContentTypeLengthIndex + IntSizeInBytes;
internal const int ContentEncodingLengthIndex = ContentEncodingOffsetIndex + IntSizeInBytes;
internal const int ContentLanguageOffsetIndex = ContentEncodingLengthIndex + IntSizeInBytes;
internal const int ContentLanguageLengthIndex = ContentLanguageOffsetIndex + IntSizeInBytes;
internal const int ContentDispositionOffsetIndex = ContentLanguageLengthIndex + IntSizeInBytes;
internal const int ContentDispositionLengthIndex = ContentDispositionOffsetIndex + IntSizeInBytes;
internal const int CacheControlOffsetIndex = ContentDispositionLengthIndex + IntSizeInBytes;
internal const int CacheControlLengthIndex = CacheControlOffsetIndex + IntSizeInBytes;

internal const int MetadataOffsetIndex = CacheControlLengthIndex + IntSizeInBytes;
internal const int MetadataLengthIndex = MetadataOffsetIndex + IntSizeInBytes;

internal const int VariableLengthStartIndex = MetadataLengthIndex + IntSizeInBytes;
internal const int FileAttributesIndex = VersionIndex + VersionEncodedSize;
internal const int FilePermissionKeyOffsetIndex = FileAttributesIndex + FileAttributesEncodedSize;
internal const int FilePermissionKeyLengthIndex = FilePermissionKeyOffsetIndex + FilePermissionKeyOffsetEncodedSize;
internal const int FileCreatedOnIndex = FilePermissionKeyLengthIndex + FilePermissionKeyLengthEncodedSize;
internal const int FileLastWrittenOnIndex = FileCreatedOnIndex + FileCreatedOnEncodedSize;
internal const int FileChangedOnIndex = FileLastWrittenOnIndex + FileLastWrittenOnEncodedSize;

internal const int ContentTypeOffsetIndex = FileChangedOnIndex + FileChangedOnEncodedSize;
internal const int ContentTypeLengthIndex = ContentTypeOffsetIndex + ContentTypeOffsetEncodedSize;
internal const int ContentEncodingOffsetIndex = ContentTypeLengthIndex + ContentTypeLengthEncodedSize;
internal const int ContentEncodingLengthIndex = ContentEncodingOffsetIndex + ContentEncodingOffsetEncodedSize;
internal const int ContentLanguageOffsetIndex = ContentEncodingLengthIndex + ContentEncodingLengthEncodedSize;
internal const int ContentLanguageLengthIndex = ContentLanguageOffsetIndex + ContentLanguageOffsetEncodedSize;
internal const int ContentDispositionOffsetIndex = ContentLanguageLengthIndex + ContentLanguageLengthEncodedSize;
internal const int ContentDispositionLengthIndex = ContentDispositionOffsetIndex + ContentDispositionOffsetEncodedSize;
internal const int CacheControlOffsetIndex = ContentDispositionLengthIndex + ContentDispositionLengthEncodedSize;
internal const int CacheControlLengthIndex = CacheControlOffsetIndex + CacheControlOffsetEncodedSize;

internal const int FileMetadataOffsetIndex = CacheControlLengthIndex + CacheControlLengthEncodedSize;
internal const int FileMetadataLengthIndex = FileMetadataOffsetIndex + FileMetadataOffsetEncodedSize;
internal const int DirectoryMetadataOffsetIndex = FileMetadataLengthIndex + FileMetadataLengthEncodedSize;
internal const int DirectoryMetadataLengthIndex = DirectoryMetadataOffsetIndex + DirectoryMetadataOffsetEncodedSize;

internal const int VariableLengthStartIndex = DirectoryMetadataLengthIndex + DirectoryMetadataLengthEncodedSize;
}
}
}
2 changes: 1 addition & 1 deletion ...age/Azure.Storage.DataMovement.Files.Shares/src/ShareDirectoryStorageResourceContainer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ protected override StorageResourceCheckpointData GetSourceCheckpointData()

protected override StorageResourceCheckpointData GetDestinationCheckpointData()
{
return new ShareFileDestinationCheckpointData(null, null);
return new ShareFileDestinationCheckpointData(null, null, null, null);
}

protected override async Task CreateIfNotExistsAsync(CancellationToken cancellationToken = default)
Expand Down
119 changes: 83 additions & 36 deletions ...storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileDestinationCheckpointData.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,16 +30,27 @@ internal class ShareFileDestinationCheckpointData : StorageResourceCheckpointDat
private byte[] _cacheControlBytes;

/// <summary>
/// The metadata for the destination blob.
/// Metadata for destination files.
/// </summary>
public Metadata Metadata;
private byte[] _metadataBytes;
public Metadata FileMetadata;
private byte[] _fileMetadataBytes;

/// <summary>
/// Metadata for destination directories.
/// </summary>
public Metadata DirectoryMetadata;
private byte[] _directoryMetadataBytes;

public FileSmbProperties SmbProperties;
private byte[] _filePermissionKeyBytes;

public override int Length => CalculateLength();

public ShareFileDestinationCheckpointData(
ShareFileHttpHeaders contentHeaders,
Metadata metadata)
Metadata fileMetadata,
Metadata directoryMetadata,
FileSmbProperties fileSmbProperties)
{
Version = DataMovementShareConstants.DestinationCheckpointData.SchemaVersion;
ContentHeaders = contentHeaders;
Expand All @@ -48,8 +59,12 @@ public ShareFileDestinationCheckpointData(
_contentLanguageBytes = ContentHeaders?.ContentLanguage != default ? Encoding.UTF8.GetBytes(string.Join(HeaderDelimiter.ToString(), ContentHeaders.ContentLanguage)) : Array.Empty<byte>();
_contentDispositionBytes = ContentHeaders?.ContentDisposition != default ? Encoding.UTF8.GetBytes(ContentHeaders.ContentDisposition) : Array.Empty<byte>();
_cacheControlBytes = ContentHeaders?.CacheControl != default ? Encoding.UTF8.GetBytes(ContentHeaders.CacheControl) : Array.Empty<byte>();
Metadata = metadata;
_metadataBytes = Metadata != default ? Encoding.UTF8.GetBytes(Metadata.DictionaryToString()) : Array.Empty<byte>();
FileMetadata = fileMetadata;
_fileMetadataBytes = FileMetadata != default ? Encoding.UTF8.GetBytes(FileMetadata.DictionaryToString()) : Array.Empty<byte>();
DirectoryMetadata = directoryMetadata;
_directoryMetadataBytes = DirectoryMetadata != default ? Encoding.UTF8.GetBytes(DirectoryMetadata.DictionaryToString()) : Array.Empty<byte>();
SmbProperties = fileSmbProperties;
_filePermissionKeyBytes = SmbProperties?.FilePermissionKey != default ? Encoding.UTF8.GetBytes(SmbProperties.FilePermissionKey) : Array.Empty<byte>();
}

internal void SerializeInternal(Stream stream) => Serialize(stream);
Expand All @@ -64,21 +79,33 @@ protected override void Serialize(Stream stream)
// Version
writer.Write(Version);

// Fixed position offset/lengths for variable length info
// SMB properties
writer.Write((int?)SmbProperties?.FileAttributes);
writer.WriteVariableLengthFieldInfo(_filePermissionKeyBytes.Length, ref currentVariableLengthIndex);
writer.Write(SmbProperties?.FileCreatedOn);
writer.Write(SmbProperties?.FileLastWrittenOn);
writer.Write(SmbProperties?.FileChangedOn);

// HttpHeaders
writer.WriteVariableLengthFieldInfo(_contentTypeBytes.Length, ref currentVariableLengthIndex);
writer.WriteVariableLengthFieldInfo(_contentEncodingBytes.Length, ref currentVariableLengthIndex);
writer.WriteVariableLengthFieldInfo(_contentLanguageBytes.Length, ref currentVariableLengthIndex);
writer.WriteVariableLengthFieldInfo(_contentDispositionBytes.Length, ref currentVariableLengthIndex);
writer.WriteVariableLengthFieldInfo(_cacheControlBytes.Length, ref currentVariableLengthIndex);
writer.WriteVariableLengthFieldInfo(_metadataBytes.Length, ref currentVariableLengthIndex);

// Metadata
writer.WriteVariableLengthFieldInfo(_fileMetadataBytes.Length, ref currentVariableLengthIndex);
writer.WriteVariableLengthFieldInfo(_directoryMetadataBytes.Length, ref currentVariableLengthIndex);

// Variable length info
writer.Write(_filePermissionKeyBytes);
writer.Write(_contentTypeBytes);
writer.Write(_contentEncodingBytes);
writer.Write(_contentLanguageBytes);
writer.Write(_contentDispositionBytes);
writer.Write(_cacheControlBytes);
writer.Write(_metadataBytes);
writer.Write(_fileMetadataBytes);
writer.Write(_directoryMetadataBytes);
}

internal static ShareFileDestinationCheckpointData Deserialize(Stream stream)
Expand All @@ -94,29 +121,23 @@ internal static ShareFileDestinationCheckpointData Deserialize(Stream stream)
throw Storage.Errors.UnsupportedJobSchemaVersionHeader(version.ToString());
}

// ContentType offset/length
int contentTypeOffset = reader.ReadInt32();
int contentTypeLength = reader.ReadInt32();

// ContentEncoding offset/length
int contentEncodingOffset = reader.ReadInt32();
int contentEncodingLength = reader.ReadInt32();

// ContentLanguage offset/length
int contentLanguageOffset = reader.ReadInt32();
int contentLanguageLength = reader.ReadInt32();
// SMB properties
NtfsFileAttributes? ntfsFileAttributes = (NtfsFileAttributes?)reader.ReadNullableInt32();
(int filePermissionKeyOffset, int filePermissionKeyLength) = reader.ReadVariableLengthFieldInfo();
DateTimeOffset? fileCreatedOn = reader.ReadNullableDateTimeOffset();
DateTimeOffset? fileLastWrittenOn = reader.ReadNullableDateTimeOffset();
DateTimeOffset? fileChangedOn = reader.ReadNullableDateTimeOffset();

// ContentDisposition offset/length
int contentDispositionOffset = reader.ReadInt32();
int contentDispositionLength = reader.ReadInt32();
// HttpHeaders
(int contentTypeOffset, int contentTypeLength) = reader.ReadVariableLengthFieldInfo();
(int contentEncodingOffset, int contentEncodingLength) = reader.ReadVariableLengthFieldInfo();
(int contentLanguageOffset, int contentLanguageLength) = reader.ReadVariableLengthFieldInfo();
(int contentDispositionOffset, int contentDispositionLength) = reader.ReadVariableLengthFieldInfo();
(int cacheControlOffset, int cacheControlLength) = reader.ReadVariableLengthFieldInfo();

// CacheControl offset/length
int cacheControlOffset = reader.ReadInt32();
int cacheControlLength = reader.ReadInt32();

// Metadata offset/length
int metadataOffset = reader.ReadInt32();
int metadataLength = reader.ReadInt32();
// Metadata
(int fileMetadataOffset, int fileMetadataLength) = reader.ReadVariableLengthFieldInfo();
(int directoryMetadataOffset, int directoryMetadataLength) = reader.ReadVariableLengthFieldInfo();

// ContentType
string contentType = null;
Expand All @@ -126,6 +147,14 @@ internal static ShareFileDestinationCheckpointData Deserialize(Stream stream)
contentType = Encoding.UTF8.GetString(reader.ReadBytes(contentTypeLength));
}

// ContentType
string filePermissionKey = null;
if (contentTypeOffset > 0)
{
reader.BaseStream.Position = filePermissionKeyOffset;
filePermissionKey = Encoding.UTF8.GetString(reader.ReadBytes(filePermissionKeyLength));
}

// ContentEncoding
string contentEncoding = null;
if (contentEncodingOffset > 0)
Expand Down Expand Up @@ -159,11 +188,17 @@ internal static ShareFileDestinationCheckpointData Deserialize(Stream stream)
}

// Metadata
string metadataString = string.Empty;
if (metadataOffset > 0)
string fileMetadataString = string.Empty;
if (fileMetadataOffset > 0)
{
reader.BaseStream.Position = metadataOffset;
metadataString = Encoding.UTF8.GetString(reader.ReadBytes(metadataLength));
reader.BaseStream.Position = fileMetadataOffset;
fileMetadataString = Encoding.UTF8.GetString(reader.ReadBytes(fileMetadataLength));
}
string directoryMetadataString = string.Empty;
if (directoryMetadataOffset > 0)
{
reader.BaseStream.Position = directoryMetadataOffset;
directoryMetadataString = Encoding.UTF8.GetString(reader.ReadBytes(directoryMetadataLength));
}

ShareFileHttpHeaders contentHeaders = new()
Expand All @@ -175,9 +210,20 @@ internal static ShareFileDestinationCheckpointData Deserialize(Stream stream)
CacheControl = cacheControl,
};

FileSmbProperties smbProperties = new()
{
FileAttributes = ntfsFileAttributes,
FilePermissionKey = filePermissionKey,
FileCreatedOn = fileCreatedOn,
FileLastWrittenOn = fileLastWrittenOn,
FileChangedOn = fileChangedOn,
};

return new(
contentHeaders,
metadataString.ToDictionary(nameof(metadataString)));
fileMetadataString.ToDictionary(nameof(fileMetadataString)),
directoryMetadataString.ToDictionary(nameof(directoryMetadataString)),
smbProperties);
}

private int CalculateLength()
Expand All @@ -189,7 +235,8 @@ private int CalculateLength()
length += _contentLanguageBytes.Length;
length += _contentDispositionBytes.Length;
length += _cacheControlBytes.Length;
length += _metadataBytes.Length;
length += _fileMetadataBytes.Length;
length += _directoryMetadataBytes.Length;
return length;
}
}
Expand Down
2 changes: 1 addition & 1 deletion sdk/storage/Azure.Storage.DataMovement.Files.Shares/src/ShareFileStorageResource.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,7 @@ protected override StorageResourceCheckpointData GetSourceCheckpointData()

protected override StorageResourceCheckpointData GetDestinationCheckpointData()
{
return new ShareFileDestinationCheckpointData(null, null);
return new ShareFileDestinationCheckpointData(null, null, null, null);
}
}

Expand Down
Loading

0 comments on commit 801dd7e

Please sign in to comment.