Skip to content

Commit

Permalink
Implement Tar Global Extended Attributes API changes (#70869)
Browse files Browse the repository at this point in the history
* ref: Global Extended Attributes API changes

* src: Global Extended Attributes API changes

* tests: Verify Global Extended Attributes API changes

* Address suggestions

* Address debug suggestion.

* Improve the decision of reducing the size of the GEA path.

* Use Path.GetTempPath for the GEA path.

* Rename field tracking GEA entry number.

Co-authored-by: carlossanlop <carlossanlop@users.noreply.github.com>
  • Loading branch information
carlossanlop and carlossanlop authored Jun 22, 2022
1 parent b1839d3 commit d3f7e01
Show file tree
Hide file tree
Showing 23 changed files with 986 additions and 626 deletions.
8 changes: 6 additions & 2 deletions src/libraries/System.Formats.Tar/ref/System.Formats.Tar.cs
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,11 @@ public GnuTarEntry(System.Formats.Tar.TarEntryType entryType, string entryName)
public System.DateTimeOffset AccessTime { get { throw null; } set { } }
public System.DateTimeOffset ChangeTime { get { throw null; } set { } }
}
public sealed partial class PaxGlobalExtendedAttributesTarEntry : System.Formats.Tar.PosixTarEntry
{
public PaxGlobalExtendedAttributesTarEntry(System.Collections.Generic.IEnumerable<System.Collections.Generic.KeyValuePair<string, string>> globalExtendedAttributes) { }
public System.Collections.Generic.IReadOnlyDictionary<string, string> GlobalExtendedAttributes { get { throw null; } }
}
public sealed partial class PaxTarEntry : System.Formats.Tar.PosixTarEntry
{
public PaxTarEntry(System.Formats.Tar.TarEntry other) { }
Expand Down Expand Up @@ -101,14 +106,13 @@ public enum TarFileMode
public sealed partial class TarReader : System.IDisposable
{
public TarReader(System.IO.Stream archiveStream, bool leaveOpen = false) { }
public System.Collections.Generic.IReadOnlyDictionary<string, string>? GlobalExtendedAttributes { get { throw null; } }
public void Dispose() { }
public System.Formats.Tar.TarEntry? GetNextEntry(bool copyData = false) { throw null; }
}
public sealed partial class TarWriter : System.IDisposable
{
public TarWriter(System.IO.Stream archiveStream) { }
public TarWriter(System.IO.Stream archiveStream, System.Collections.Generic.IEnumerable<System.Collections.Generic.KeyValuePair<string, string>>? globalExtendedAttributes = null, bool leaveOpen = false) { }
public TarWriter(System.IO.Stream archiveStream, bool leaveOpen = false) { }
public TarWriter(System.IO.Stream archiveStream, System.Formats.Tar.TarEntryFormat format = System.Formats.Tar.TarEntryFormat.Pax, bool leaveOpen = false) { }
public System.Formats.Tar.TarEntryFormat Format { get { throw null; } }
public void Dispose() { }
Expand Down
9 changes: 3 additions & 6 deletions src/libraries/System.Formats.Tar/src/Resources/Strings.resx
Original file line number Diff line number Diff line change
Expand Up @@ -186,12 +186,12 @@
<data name="SetLengthRequiresSeekingAndWriting" xml:space="preserve">
<value>SetLength requires a stream that supports seeking and writing.</value>
</data>
<data name="TarCannotConvertPaxGlobalExtendedAttributesEntry" xml:space="preserve">
<value>Cannot convert a PaxGlobalExtendedAttributesEntry into another format.</value>
</data>
<data name="TarDuplicateExtendedAttribute" xml:space="preserve">
<value>The entry '{0}' has a duplicate extended attribute.</value>
</data>
<data name="TarEntriesInDifferentFormats" xml:space="preserve">
<value>An entry in '{0}' format was found in an archive where other entries of format '{1}' have been found.</value>
</data>
<data name="TarEntryBlockOrCharacterExpected" xml:space="preserve">
<value>Cannot set the 'DeviceMajor' or 'DeviceMinor' fields on an entry that does not represent a block or character device.</value>
</data>
Expand Down Expand Up @@ -240,9 +240,6 @@
<data name="TarSymbolicLinkTargetNotExists" xml:space="preserve">
<value>Cannot create the symbolic link '{0}' because the specified target '{1}' does not exist.</value>
</data>
<data name="TarTooManyGlobalExtendedAttributesEntries" xml:space="preserve">
<value>The archive has more than one global extended attributes entry.</value>
</data>
<data name="TarUnexpectedMetadataEntry" xml:space="preserve">
<value>A metadata entry of type '{0}' was unexpectedly found after a metadata entry of type '{1}'.</value>
</data>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
<Compile Include="System\Formats\Tar\TarEntryFormat.cs" />
<Compile Include="System\Formats\Tar\UstarTarEntry.cs" />
<Compile Include="System\Formats\Tar\GnuTarEntry.cs" />
<Compile Include="System\Formats\Tar\PaxGlobalExtendedAttributesTarEntry.cs" />
<Compile Include="System\Formats\Tar\PaxTarEntry.cs" />
<Compile Include="System\Formats\Tar\TarEntryType.cs" />
<Compile Include="System\Formats\Tar\TarFile.cs" />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ internal GnuTarEntry(TarHeader header, TarReader readerOfOrigin)
/// </list>
/// </remarks>
public GnuTarEntry(TarEntryType entryType, string entryName)
: base(entryType, entryName, TarEntryFormat.Gnu)
: base(entryType, entryName, TarEntryFormat.Gnu, isGea: false)
{
_header._aTime = _header._mTime; // mtime was set in base constructor
_header._cTime = _header._mTime;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System.Collections.Generic;
using System.Collections.ObjectModel;
using System.Diagnostics;

namespace System.Formats.Tar
{
/// <summary>
/// Represents a Global Extended Attributes TAR entry from an archive of the PAX format.
/// </summary>
public sealed class PaxGlobalExtendedAttributesTarEntry : PosixTarEntry
{
private ReadOnlyDictionary<string, string>? _readOnlyGlobalExtendedAttributes;

// Constructor used when reading an existing archive.
internal PaxGlobalExtendedAttributesTarEntry(TarHeader header, TarReader readerOfOrigin)
: base(header, readerOfOrigin, TarEntryFormat.Pax)
{
}

/// <summary>
/// Initializes a new <see cref="PaxGlobalExtendedAttributesTarEntry"/> instance with the specified Global Extended Attributes enumeration.
/// </summary>
/// <param name="globalExtendedAttributes">An enumeration of string key-value pairs that represents the metadata to include as Global Extended Attributes.</param>
/// <exception cref="ArgumentNullException"><paramref name="globalExtendedAttributes"/> is <see langword="null"/>.</exception>
public PaxGlobalExtendedAttributesTarEntry(IEnumerable<KeyValuePair<string, string>> globalExtendedAttributes)
: base(TarEntryType.GlobalExtendedAttributes, TarHeader.GlobalHeadFormatPrefix, TarEntryFormat.Pax, isGea: true)
{
ArgumentNullException.ThrowIfNull(globalExtendedAttributes);
_header._extendedAttributes = new Dictionary<string, string>(globalExtendedAttributes);
}

/// <summary>
/// Returns the global extended attributes stored in this entry.
/// </summary>
public IReadOnlyDictionary<string, string> GlobalExtendedAttributes
{
get
{
_header._extendedAttributes ??= new Dictionary<string, string>();
return _readOnlyGlobalExtendedAttributes ??= _header._extendedAttributes.AsReadOnly();
}
}

// Determines if the current instance's entry type supports setting a data stream.
internal override bool IsDataStreamSetterSupported() => false;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ internal PaxTarEntry(TarHeader header, TarReader readerOfOrigin)
/// </list>
/// </remarks>
public PaxTarEntry(TarEntryType entryType, string entryName)
: base(entryType, entryName, TarEntryFormat.Pax)
: base(entryType, entryName, TarEntryFormat.Pax, isGea: false)
{
_header._prefix = string.Empty;
_header._extendedAttributes = new Dictionary<string, string>();
Expand Down Expand Up @@ -87,7 +87,7 @@ public PaxTarEntry(TarEntryType entryType, string entryName)
/// </list>
/// </remarks>
public PaxTarEntry(TarEntryType entryType, string entryName, IEnumerable<KeyValuePair<string, string>> extendedAttributes)
: base(entryType, entryName, TarEntryFormat.Pax)
: base(entryType, entryName, TarEntryFormat.Pax, isGea: false)
{
ArgumentNullException.ThrowIfNull(extendedAttributes);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ internal PosixTarEntry(TarHeader header, TarReader readerOfOrigin, TarEntryForma
}

// Constructor called when the user creates a TarEntry instance from scratch.
internal PosixTarEntry(TarEntryType entryType, string entryName, TarEntryFormat format)
: base(entryType, entryName, format)
internal PosixTarEntry(TarEntryType entryType, string entryName, TarEntryFormat format, bool isGea)
: base(entryType, entryName, format, isGea)
{
_header._uName = string.Empty;
_header._gName = string.Empty;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,16 @@ internal TarEntry(TarHeader header, TarReader readerOfOrigin, TarEntryFormat for
}

// Constructor called when the user creates a TarEntry instance from scratch.
internal TarEntry(TarEntryType entryType, string entryName, TarEntryFormat format)
internal TarEntry(TarEntryType entryType, string entryName, TarEntryFormat format, bool isGea)
{
ArgumentException.ThrowIfNullOrEmpty(entryName);
TarHelpers.ThrowIfEntryTypeNotSupported(entryType, format);

Debug.Assert(!isGea || entryType is TarEntryType.GlobalExtendedAttributes);

if (!isGea)
{
TarHelpers.ThrowIfEntryTypeNotSupported(entryType, format);
}

_header = default;
_header._format = format;
Expand All @@ -48,6 +54,11 @@ internal TarEntry(TarEntryType entryType, string entryName, TarEntryFormat forma
// Constructor called when converting an entry to the selected format.
internal TarEntry(TarEntry other, TarEntryFormat format)
{
if (other is PaxGlobalExtendedAttributesTarEntry)
{
throw new InvalidOperationException(SR.TarCannotConvertPaxGlobalExtendedAttributesEntry);
}

TarEntryType compatibleEntryType;
if (other.Format is TarEntryFormat.V7 && other.EntryType is TarEntryType.V7RegularFile && format is TarEntryFormat.Ustar or TarEntryFormat.Pax or TarEntryFormat.Gnu)
{
Expand Down Expand Up @@ -208,7 +219,7 @@ public int Uid
/// <exception cref="UnauthorizedAccessException">Operation not permitted due to insufficient permissions.</exception>
public void ExtractToFile(string destinationFileName, bool overwrite)
{
if (EntryType is TarEntryType.SymbolicLink or TarEntryType.HardLink)
if (EntryType is TarEntryType.SymbolicLink or TarEntryType.HardLink or TarEntryType.GlobalExtendedAttributes)
{
throw new InvalidOperationException(string.Format(SR.TarEntryTypeNotSupportedForExtracting, EntryType));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -255,7 +255,10 @@ private static void ExtractToDirectoryInternal(Stream source, string destination
TarEntry? entry;
while ((entry = reader.GetNextEntry()) != null)
{
entry.ExtractRelativeToDirectory(destinationDirectoryPath, overwriteFiles);
if (entry is not PaxGlobalExtendedAttributesTarEntry)
{
entry.ExtractRelativeToDirectory(destinationDirectoryPath, overwriteFiles);
}
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,28 +22,9 @@ internal partial struct TarHeader
// "{dirName}/PaxHeaders.{processId}/{fileName}{trailingSeparator}"
private const string PaxHeadersFormat = "{0}/PaxHeaders.{1}/{2}{3}";

// Global Extended Attribute entries have a special format in the Name field:
// "{tmpFolder}/GlobalHead.{processId}.1"
private const string GlobalHeadFormat = "{0}/GlobalHead.{1}.1";

// Predefined text for the Name field of a GNU long metadata entry. Applies for both LongPath ('L') and LongLink ('K').
private const string GnuLongMetadataName = "././@LongLink";

// Creates a PAX Global Extended Attributes header and writes it into the specified archive stream.
internal static void WriteGlobalExtendedAttributesHeader(Stream archiveStream, Span<byte> buffer, IEnumerable<KeyValuePair<string, string>> globalExtendedAttributes)
{
TarHeader geaHeader = default;
geaHeader._name = GenerateGlobalExtendedAttributeName();
geaHeader._mode = (int)TarHelpers.DefaultMode;
geaHeader._typeFlag = TarEntryType.GlobalExtendedAttributes;
geaHeader._linkName = string.Empty;
geaHeader._magic = string.Empty;
geaHeader._version = string.Empty;
geaHeader._gName = string.Empty;
geaHeader._uName = string.Empty;
geaHeader.WriteAsPaxExtendedAttributes(archiveStream, buffer, globalExtendedAttributes, isGea: true);
}

// Writes the current header as a V7 entry into the archive stream.
internal void WriteAsV7(Stream archiveStream, Span<byte> buffer)
{
Expand Down Expand Up @@ -82,14 +63,27 @@ internal void WriteAsUstar(Stream archiveStream, Span<byte> buffer)
}
}

// Writes the current header as a PAX Global Extended Attributes entry into the archive stream.
internal void WriteAsPaxGlobalExtendedAttributes(Stream archiveStream, Span<byte> buffer, int globalExtendedAttributesEntryNumber)
{
Debug.Assert(_typeFlag is TarEntryType.GlobalExtendedAttributes);

_name = GenerateGlobalExtendedAttributeName(globalExtendedAttributesEntryNumber);
_extendedAttributes ??= new Dictionary<string, string>();
WriteAsPaxExtendedAttributes(archiveStream, buffer, _extendedAttributes, isGea: true);
}

// Writes the current header as a PAX entry into the archive stream.
// Makes sure to add the preceding exteded attributes entry before the actual entry.
// Makes sure to add the preceding extended attributes entry before the actual entry.
internal void WriteAsPax(Stream archiveStream, Span<byte> buffer)
{
Debug.Assert(_typeFlag is not TarEntryType.GlobalExtendedAttributes);

// First, we write the preceding extended attributes header
TarHeader extendedAttributesHeader = default;
// Fill the current header's dict
CollectExtendedAttributesFromStandardFieldsIfNeeded();
// And pass the attributes to the preceding extended attributes header for writing
Debug.Assert(_extendedAttributes != null);
extendedAttributesHeader.WriteAsPaxExtendedAttributes(archiveStream, buffer, _extendedAttributes, isGea: false);

Expand Down Expand Up @@ -611,30 +605,30 @@ private string GenerateExtendedAttributeName()
}

// Gets the special name for the 'name' field in a global extended attribute entry.
// Format: "%d/GlobalHead.%p/%f"
// Format: "%d/GlobalHead.%p/%n"
// - %d: The path of the $TMPDIR variable, if found. Otherwise, the value is '/tmp'.
// - %p: The current process ID.
// - %n: The sequence number of the global extended header record of the archive, starting at 1. In our case, since we only generate one, the value is always 1.
// If the path of $TMPDIR makes the final string too long to fit in the 'name' field,
// then the TMPDIR='/tmp' is used.
private static string GenerateGlobalExtendedAttributeName()
private static string GenerateGlobalExtendedAttributeName(int globalExtendedAttributesEntryNumber)
{
string? tmpDir = Environment.GetEnvironmentVariable("TMPDIR");
if (string.IsNullOrWhiteSpace(tmpDir))
{
tmpDir = "/tmp";
}
else if (Path.EndsInDirectorySeparator(tmpDir))
Debug.Assert(globalExtendedAttributesEntryNumber >= 1);

string tmpDir = Path.GetTempPath();
if (Path.EndsInDirectorySeparator(tmpDir))
{
tmpDir = Path.TrimEndingDirectorySeparator(tmpDir);
}
int processId = Environment.ProcessId;

string result = string.Format(GlobalHeadFormat, tmpDir, processId);
if (result.Length >= FieldLengths.Name)
string result = string.Format(GlobalHeadFormatPrefix, tmpDir, processId);
string suffix = $".{globalExtendedAttributesEntryNumber}"; // GEA sequence number
if (result.Length + suffix.Length >= FieldLengths.Name)
{
result = string.Format(GlobalHeadFormat, "/tmp", processId);
result = string.Format(GlobalHeadFormatPrefix, "/tmp", processId);
}
result += suffix;

return result;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,11 @@ internal partial struct TarHeader
private const string PaxEaDevMajor = "devmajor";
private const string PaxEaDevMinor = "devminor";

// Global Extended Attribute entries have a special format in the Name field:
// "{tmpFolder}/GlobalHead.{processId}.{GEAEntryNumber}"
// Excludes ".{GEAEntryNumber}" because the number gets added on write.
internal const string GlobalHeadFormatPrefix = "{0}/GlobalHead.{1}";

internal Stream? _dataStream;

// Position in the stream where the data ends in this header.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -252,9 +252,10 @@ TarEntryType.HardLink or
TarEntryType.RegularFile or
TarEntryType.SymbolicLink)
{
// GlobalExtendedAttributes is handled via PaxGlobalExtendedAttributesEntry

// Not supported for writing - internally autogenerated:
// - ExtendedAttributes
// - GlobalExtendedAttributes
return;
}
break;
Expand Down
Loading

0 comments on commit d3f7e01

Please sign in to comment.