Skip to content

Commit d46c313

Browse files
grendellogrendello
committed
More assembly store + assemblies
1 parent d0b8b9b commit d46c313

File tree

6 files changed

+215
-24
lines changed

6 files changed

+215
-24
lines changed

‎tools/apput/src/ApplicationAssembly/ApplicationAssembly.cs‎

Lines changed: 119 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,21 +5,134 @@ namespace ApplicationUtility;
55

66
public class ApplicationAssembly : IAspect
77
{
8+
const string LogTag = "ApplicationAssembly";
9+
const uint COMPRESSED_MAGIC = 0x5A4C4158; // 'XALZ', little-endian
10+
const ushort MSDOS_EXE_MAGIC = 0x5A4D; // 'MZ'
11+
const uint PE_EXE_MAGIC = 0x00004550; // 'PE\0\0'
12+
813
public static string AspectName { get; } = "Application assembly";
914

10-
public bool IsCompressed { get; private set; }
11-
public string Name { get; private set; } = "";
12-
public ulong CompressedSize { get; private set; }
13-
public ulong Size { get; private set; }
14-
public bool IgnoreOnLoad { get; private set; }
15+
public bool IsCompressed { get; }
16+
public string Name { get; }
17+
public ulong CompressedSize { get; }
18+
public ulong Size { get; }
19+
public bool IgnoreOnLoad { get; }
20+
public ulong NameHash { get; internal set; }
21+
22+
readonly Stream? assemblyStream;
23+
24+
ApplicationAssembly (Stream stream, uint uncompressedSize, string? description, bool isCompressed)
25+
{
26+
assemblyStream = stream;
27+
Size = uncompressedSize;
28+
CompressedSize = isCompressed ? (ulong)stream.Length : 0;
29+
IsCompressed = isCompressed;
30+
Name = NameMe (description);
31+
}
32+
33+
ApplicationAssembly (string? description, bool isIgnored)
34+
{
35+
IgnoreOnLoad = isIgnored;
36+
Name = NameMe (description);
37+
}
38+
39+
static string NameMe (string? description) => String.IsNullOrEmpty (description) ? "Unnamed" : description;
40+
41+
// This is a special case, as much as I hate to have one. Ignored assemblies exist only in the assembly store's
42+
// index. They have an associated descriptor, but no data whatsoever. For that reason, we can't go the `ProbeAspect`
43+
// + `LoadAspect` route, so `AssemblyStore` will call this method for them.
44+
public static IAspect CreateIgnoredAssembly (string? description, ulong nameHash)
45+
{
46+
Log.Debug ($"{LogTag}: stream ('{description}') is an ignored assembly.");
47+
return new ApplicationAssembly (description, isIgnored: true) {
48+
NameHash = nameHash,
49+
};
50+
}
1551

1652
public static IAspect LoadAspect (Stream stream, IAspectState state, string? description)
1753
{
18-
throw new NotImplementedException ();
54+
using var reader = Utilities.GetReaderAndRewindStream (stream);
55+
if (ReadCompressedHeader (reader, out uint uncompressedLength)) {
56+
return new ApplicationAssembly (stream, uncompressedLength, description, isCompressed: true);
57+
}
58+
59+
return new ApplicationAssembly (stream, (uint)stream.Length, description, isCompressed: false);
1960
}
2061

2162
public static IAspectState ProbeAspect (Stream stream, string? description)
63+
{
64+
Log.Debug ($"{LogTag}: probing stream ('{description}')");
65+
if (stream.Length == 0) {
66+
// It can happen if the assembly store index or name table are corrupted and we cannot
67+
// determine if an assembly is ignored or not. If it is ignored, it will have no data
68+
// available and so the stream will have length of 0
69+
return new BasicAspectState (false);
70+
}
71+
72+
// If we detect compressed assembly signature, we won't proceed with checking whether
73+
// the rest of data is actually a valid managed assembly. This is to avoid doing a
74+
// costly operation of decompressing when e.g. loading data from an assemblystore, when
75+
// we potentially create a lot of `ApplicationAssembly` instances. Presence of the compression
76+
// header is enough for the probing stage.
77+
78+
using var reader = Utilities.GetReaderAndRewindStream (stream);
79+
if (ReadCompressedHeader (reader, out _)) {
80+
Log.Debug ($"{LogTag}: stream ('{description}') is a compressed assembly.");
81+
return new BasicAspectState (true);
82+
}
83+
84+
// We could use PEReader (https://learn.microsoft.com/en-us/dotnet/api/system.reflection.portableexecutable.pereader)
85+
// but it would be too heavy for our purpose here.
86+
reader.BaseStream.Seek (0, SeekOrigin.Begin);
87+
ushort mzExeMagic = reader.ReadUInt16 ();
88+
if (mzExeMagic != MSDOS_EXE_MAGIC) {
89+
return Utilities.GetFailureAspectState ($"{LogTag}: stream doesn't have MS-DOS executable signature.");
90+
}
91+
92+
const long PE_HEADER_OFFSET = 0x3c;
93+
if (reader.BaseStream.Length <= PE_HEADER_OFFSET) {
94+
return Utilities.GetFailureAspectState ($"{LogTag}: stream contains a corrupted MS-DOS executable image (too short, offset {PE_HEADER_OFFSET} is bigger than stream size).");
95+
}
96+
97+
// Offset at 0x3C is where we can read the 32-bit offset to the PE header
98+
reader.BaseStream.Seek (PE_HEADER_OFFSET, SeekOrigin.Begin);
99+
uint uintVal = reader.ReadUInt32 ();
100+
if (reader.BaseStream.Length <= (long)uintVal) {
101+
return Utilities.GetFailureAspectState ($"{LogTag}: stream contains a corrupted PE executable image (too short, offset {uintVal} is bigger than stream size).");
102+
}
103+
104+
reader.BaseStream.Seek ((long)uintVal, SeekOrigin.Begin);
105+
uintVal = reader.ReadUInt32 ();
106+
if (uintVal != PE_EXE_MAGIC) {
107+
return Utilities.GetFailureAspectState ($"{LogTag}: stream doesn't have PE executable signature.");
108+
}
109+
// This is good enough for us
110+
111+
Log.Debug ($"{LogTag}: stream ('{description}') appears to be a PE image.");
112+
return new BasicAspectState (true);
113+
}
114+
115+
/// <summary>
116+
/// Writes assembly data to the indicated file, uncompressing it if necessary. If the destination
117+
/// file exists, it will be overwritten.
118+
/// </summary>
119+
public void SaveToFile (string filePath)
22120
{
23121
throw new NotImplementedException ();
24122
}
123+
124+
// We don't care about the descriptor index here, it's only needed during the run time
125+
static bool ReadCompressedHeader (BinaryReader reader, out uint uncompressedLength)
126+
{
127+
uncompressedLength = 0;
128+
129+
uint uintVal = reader.ReadUInt32 ();
130+
if (uintVal != COMPRESSED_MAGIC) {
131+
return false;
132+
}
133+
134+
uintVal = reader.ReadUInt32 (); // descriptor index
135+
uncompressedLength = reader.ReadUInt32 ();
136+
return true;
137+
}
25138
}

‎tools/apput/src/AssemblyStore/AssemblyStoreIndexEntry.cs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
namespace ApplicationUtility;
2+
3+
abstract class AssemblyStoreIndexEntry
4+
{}

‎tools/apput/src/AssemblyStore/AssemblyStoreIndexEntryV3.cs‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,15 @@
11
namespace ApplicationUtility;
22

3-
class AssemblyStoreIndexEntryV3
3+
class AssemblyStoreIndexEntryV3 : AssemblyStoreIndexEntry
44
{
5+
public ulong NameHash { get; }
6+
public uint DescriptorIndex { get; }
7+
public bool Ignore { get; }
58

9+
public AssemblyStoreIndexEntryV3 (ulong nameHash, uint descriptorIndex, byte ignore)
10+
{
11+
NameHash = nameHash;
12+
DescriptorIndex = descriptorIndex;
13+
Ignore = ignore != 0;
14+
}
615
}

‎tools/apput/src/AssemblyStore/FormatBase.cs‎

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ protected FormatBase (Stream storeStream, string? description)
3232
public bool Read ()
3333
{
3434
bool success = true;
35-
using var reader = new BinaryReader (StoreStream, Encoding.UTF8, leaveOpen: true);
35+
using var reader = Utilities.GetReaderAndRewindStream (StoreStream);
3636

3737
// They can be `null` if `Validate` wasn't called for some reason.
3838
if (Header == null) {
@@ -67,7 +67,7 @@ public bool Read ()
6767

6868
public IAspectState Validate ()
6969
{
70-
using var reader = new BinaryReader (StoreStream, Encoding.UTF8, leaveOpen: true);
70+
using var reader = Utilities.GetReaderAndRewindStream (StoreStream);
7171

7272
if (ReadHeader (reader, out AssemblyStoreHeader? header)) {
7373
Header = header;
@@ -82,12 +82,6 @@ public IAspectState Validate ()
8282

8383
protected abstract IAspectState ValidateInner ();
8484

85-
protected BasicAspectState ValidationFailed (string message)
86-
{
87-
Log.Debug (message);
88-
return new BasicAspectState (false);
89-
}
90-
9185
protected virtual bool ReadHeader (BinaryReader reader, out AssemblyStoreHeader? header)
9286
{
9387
header = null;

‎tools/apput/src/AssemblyStore/Format_V3.cs‎

Lines changed: 67 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
using System.IO;
55
using System.Text;
66

7+
using Xamarin.Android.Tasks;
8+
79
namespace ApplicationUtility;
810

911
class Format_V3 : FormatBase
@@ -25,12 +27,12 @@ protected bool EnsureValidState (string where, out IAspectState? retval)
2527
{
2628
retval = null;
2729
if (Header == null || Header.EntryCount == null || Header.IndexEntryCount == null || Header.IndexSize == null) {
28-
retval = ValidationFailed ($"{LogTag}: invalid header data in {where}.");
30+
retval = Utilities.GetFailureAspectState ($"{LogTag}: invalid header data in {where}.");
2931
return false;
3032
}
3133

3234
if (Descriptors == null || Descriptors.Count == 0) {
33-
retval = ValidationFailed ($"{LogTag}: no descriptors read in {where}.");
35+
retval = Utilities.GetFailureAspectState ($"{LogTag}: no descriptors read in {where}.");
3436
return false;
3537
}
3638

@@ -47,11 +49,12 @@ protected override IAspectState ValidateInner ()
4749
// Repetitive to `EnsureValidState`, but it's better than using `!` all over the place below...
4850
Debug.Assert (Header != null);
4951
Debug.Assert (Header.EntryCount != null);
52+
Debug.Assert (Header.IndexSize != null);
5053
Debug.Assert (Header.IndexEntryCount != null);
5154
Debug.Assert (Descriptors != null);
5255

5356
ulong indexEntrySize = Header.Version.Is64Bit ? IndexEntrySize64 : IndexEntrySize32;
54-
ulong indexSize = (indexEntrySize * (ulong)Header.IndexEntryCount!);
57+
ulong indexSize = (ulong)Header.IndexSize; // (indexEntrySize * (ulong)Header.IndexEntryCount!);
5558
ulong descriptorsSize = AssemblyDescriptorSize * (ulong)Header.EntryCount!;
5659
ulong requiredStreamSize = HeaderSize + indexSize + descriptorsSize;
5760

@@ -72,11 +75,11 @@ protected override IAspectState ValidateInner ()
7275
Log.Debug ($"{LogTag}: calculated the required stream size to be {requiredStreamSize}");
7376

7477
if (requiredStreamSize > Int64.MaxValue) {
75-
return ValidationFailed ($"{LogTag}: required stream size is too long for the stream API to handle.");
78+
return Utilities.GetFailureAspectState ($"{LogTag}: required stream size is too long for the stream API to handle.");
7679
}
7780

7881
if ((long)requiredStreamSize != StoreStream.Length) {
79-
return ValidationFailed ($"{LogTag}: stream has invalid size, expected {requiredStreamSize} bytes, found {StoreStream.Length} instead.");
82+
return Utilities.GetFailureAspectState ($"{LogTag}: stream has invalid size, expected {requiredStreamSize} bytes, found {StoreStream.Length} instead.");
8083
} else {
8184
Log.Debug ($"{LogTag}: stream size is valid.");
8285
}
@@ -107,6 +110,8 @@ protected override IList<string> ReadAssemblyNames (BinaryReader reader)
107110

108111
protected override bool ReadAssemblies (BinaryReader reader, out IList<ApplicationAssembly>? assemblies)
109112
{
113+
Debug.Assert (Header != null);
114+
Debug.Assert (Header.IndexEntryCount != null);
110115
Debug.Assert (Descriptors != null);
111116

112117
assemblies = null;
@@ -115,27 +120,80 @@ protected override bool ReadAssemblies (BinaryReader reader, out IList<Applicati
115120
}
116121

117122
IList<string> assemblyNames = ReadAssemblyNames (reader);
118-
if (assemblyNames.Count != Descriptors.Count) {
119-
Log.Debug ($"{LogTag}: assembly name count ({assemblyNames.Count}) is different to descriptor count ({Descriptors.Count})");
120-
return false;
123+
bool assemblyNamesUnreliable = assemblyNames.Count != Descriptors.Count;
124+
if (assemblyNamesUnreliable) {
125+
Log.Error ($"{LogTag}: assembly name count ({assemblyNames.Count}) is different to descriptor count ({Descriptors.Count})");
126+
}
127+
128+
bool is64Bit = Header.Version.Is64Bit;
129+
var index = new Dictionary<ulong, AssemblyStoreIndexEntryV3> ();
130+
reader.BaseStream.Seek ((long)HeaderSize, SeekOrigin.Begin);
131+
for (uint i = 0; i < Header.IndexEntryCount; i++) {
132+
ulong hash = is64Bit ? reader.ReadUInt64 () : reader.ReadUInt32 ();
133+
uint descIdx = reader.ReadUInt32 ();
134+
byte ignore = reader.ReadByte ();
135+
136+
if (index.ContainsKey (hash)) {
137+
Log.Error ($"{LogTag}: duplicate assembly name hash (0x{hash:x}) found in the '{Description}' assembly store.");
138+
continue;
139+
}
140+
Log.Debug ($"{LogTag}: index entry {i} hash == 0x{hash:x}");
141+
index.Add (hash, new AssemblyStoreIndexEntryV3 (hash, descIdx, ignore));
121142
}
122143

123144
var ret = new List<ApplicationAssembly> ();
124145
for (int i = 0; i < Descriptors.Count; i++) {
125146
var desc = (AssemblyStoreAssemblyDescriptorV3)Descriptors[i];
126-
string name = assemblyNames[i];
147+
string name = assemblyNamesUnreliable ? "" : assemblyNames[i];
127148
var assemblyStream = new SubStream (reader.BaseStream, (long)desc.DataOffset, (long)desc.DataSize);
149+
150+
ulong hash = NameHash (name);
151+
Log.Debug ($"{LogTag}: hash for assembly '{name}' is 0x{hash:x}");
152+
153+
bool isIgnored = CheckIgnored (hash);
154+
if (isIgnored) {
155+
ret.Add ((ApplicationAssembly)ApplicationAssembly.CreateIgnoredAssembly (name, hash));
156+
continue;
157+
}
158+
128159
IAspectState assemblyState = ApplicationAssembly.ProbeAspect (assemblyStream, name);
129160
if (!assemblyState.Success) {
130161
assemblyStream.Dispose ();
131162
continue;
132163
}
133164

134165
var assembly = (ApplicationAssembly)ApplicationAssembly.LoadAspect (assemblyStream, assemblyState, name);
166+
assembly.NameHash = hash;
135167
ret.Add (assembly);
136168
}
137169

138170
assemblies = ret.AsReadOnly ();
139171
return true;
172+
173+
ulong NameHash (string name)
174+
{
175+
if (String.IsNullOrEmpty (name)) {
176+
return 0;
177+
}
178+
179+
if (name.EndsWith (".dll", StringComparison.OrdinalIgnoreCase)) {
180+
name = Path.GetFileNameWithoutExtension (name);
181+
}
182+
return MonoAndroidHelper.GetXxHash (name, is64Bit);
183+
}
184+
185+
bool CheckIgnored (ulong hash)
186+
{
187+
if (hash == 0) {
188+
return false;
189+
}
190+
191+
if (!index.TryGetValue (hash, out AssemblyStoreIndexEntryV3? entry)) {
192+
Log.Debug ($"{LogTag}: hash 0x{hash:x} not found in the index");
193+
return false;
194+
}
195+
196+
return entry.Ignore;
197+
}
140198
}
141199
}

‎tools/apput/src/Common/Utilities.cs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
using System;
22
using System.IO;
3+
using System.Text;
34

45
namespace ApplicationUtility;
56

@@ -31,4 +32,16 @@ public static void CloseAndDeleteFile (FileStream stream, bool quiet = true)
3132

3233
DeleteFile (path);
3334
}
35+
36+
public static BinaryReader GetReaderAndRewindStream (Stream stream)
37+
{
38+
stream.Seek (0, SeekOrigin.Begin);
39+
return new BinaryReader (stream, Encoding.UTF8, leaveOpen: true);
40+
}
41+
42+
public static BasicAspectState GetFailureAspectState (string message)
43+
{
44+
Log.Debug (message);
45+
return new BasicAspectState (false);
46+
}
3447
}

0 commit comments

Comments
 (0)