Commit 557ec033 authored by Ray Koopa's avatar Ray Koopa

Fix UTF32 strings and bad 0-terminator with multi-byte strings.

parent bb9938dc
......@@ -18,6 +18,7 @@ namespace Syroot.BinaryData.Memory.Test
(Encoding.UTF8, "٩(-̮̮̃-̃)۶ ٩(●̮̮̃•̃)۶ ٩(͡๏̯͡๏)۶ ٩(-̮̮̃•̃)."),
(Encoding.UTF8, "Testing «ταБЬℓσ»: 1<2 & 4+1>3, now 20% off!"),
(Encoding.Unicode, "Smiley balls 👨👩👧👧"),
(Encoding.UTF32, "Smiley balls 👨👩👧👧"),
(Encoding.BigEndianUnicode, "Smiley balls 👨👩👧👧"),
(Encodings.ShiftJIS, "Hey you, 日本へようこそ!"),
(Encodings.ShiftJIS, "日本へようこそ!"),
......@@ -134,7 +135,7 @@ namespace Syroot.BinaryData.Memory.Test
[TestMethod]
public void StringFix()
{
int byteCount = _tests.Max(x => (x.encoding ?? Encoding.UTF8).GetByteCount(x.text));
int byteCount = 1024;
Span<byte> buffer = stackalloc byte[byteCount * _tests.Count];
buffer.Fill(0xFF); // Check zeroing out memory.
......
......@@ -341,39 +341,32 @@ namespace Syroot.BinaryData.Memory
[MethodImpl(MethodImplOptions.AggressiveInlining)]
public String ReadString0()
{
// Read byte or word values until a 0 value is found (no encoding's char surrogate should consist of 0).
// Read values until a 0 value is found (no encoding's char surrogate should consist of 0 completely).
// Endianness depends on encoding, not the actual values.
int terminatorSize = _encoding.GetByteCount("A");
int length = 0;
while (true)
int terminatorSize = _encoding.GetByteCount("A");
for (byte lastByte = 1; lastByte != 0; length += terminatorSize)
{
// Check for zero bytes of the terminator.
int i;
for (i = 0; i < terminatorSize; i++)
for (int i = 0; i < terminatorSize; i++)
{
if (Span[Position + length + i] != 0)
{
length++;
lastByte = Span[Position + length + i];
if (lastByte != 0)
break;
}
}
// If a terminator was detected, return the string up to it.
if (i == terminatorSize)
{
}
// Return the string up to the terminator.
#if NETCOREAPP2_1
String value = _encoding.GetString(Span.Slice(Position, length));
String value = _encoding.GetString(Span.Slice(Position, length - terminatorSize));
#else
String value;
unsafe
{
fixed (byte* pSpan = Span.Slice(Position))
value = _encoding.GetString(pSpan, length);
}
#endif
Position += length + terminatorSize;
return value;
}
String value;
unsafe
{
fixed (byte* pSpan = Span.Slice(Position))
value = _encoding.GetString(pSpan, length - terminatorSize);
}
#endif
Position += length;
return value;
}
/// <summary>
......@@ -513,8 +506,7 @@ namespace Syroot.BinaryData.Memory
Decoder decoder = _encoding.GetDecoder();
Span<char> chars = stackalloc char[length];
#if NETCOREAPP2_1
decoder.Convert(Span.Slice(Position), chars, true, out int bytesUsed, out int charsUsed,
out bool completed);
decoder.Convert(Span.Slice(Position), chars, true, out int bytesUsed, out int charsUsed, out bool completed);
#else
int bytesUsed;
unsafe
......
......@@ -18,7 +18,7 @@ namespace Syroot.BinaryData.Memory
private bool _reverseEndian;
private Encoding _encoding;
// ---- CONSTRUCTORS & DESTRUCTOR ------------------------------------------------------------------------------
/// <summary>
......@@ -365,6 +365,12 @@ namespace Syroot.BinaryData.Memory
WriteByte(0);
WriteByte(0);
break;
case sizeof(Int32):
WriteByte(0);
WriteByte(0);
WriteByte(0);
WriteByte(0);
break;
}
}
......@@ -472,6 +478,12 @@ namespace Syroot.BinaryData.Memory
WriteByte(0);
WriteByte(0);
break;
case sizeof(Int32):
WriteByte(0);
WriteByte(0);
WriteByte(0);
WriteByte(0);
break;
}
}
......
......@@ -9,12 +9,12 @@
<PackageIconUrl>https://gitlab.com/Syroot/BinaryData/raw/master/res/icon.png</PackageIconUrl>
<PackageLicenseUrl>https://gitlab.com/Syroot/BinaryData/raw/master/LICENSE</PackageLicenseUrl>
<PackageProjectUrl>https://gitlab.com/Syroot/BinaryData</PackageProjectUrl>
<PackageReleaseNotes>Initial release for span functionality.</PackageReleaseNotes>
<PackageReleaseNotes>Fix parsing 0-terminated multi-byte strings.</PackageReleaseNotes>
<PackageTags>binary data memory span readonlymemory readonlyspan</PackageTags>
<RepositoryType>git</RepositoryType>
<RepositoryUrl>https://gitlab.com/Syroot/BinaryData</RepositoryUrl>
<TargetFrameworks>netcoreapp2.1;netstandard2.0</TargetFrameworks>
<VersionPrefix>5.2.0</VersionPrefix>
<VersionPrefix>5.2.1</VersionPrefix>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="System.Memory" Version="4.5.2" />
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment