Commit 75bb4afc authored by PoroCYon's avatar PoroCYon

Use lazy arrays, add TMLN, EXTN, SHDR(~), GLOB support

parent dbb2d402
Pipeline #35334783 passed with stages
in 1 minute and 54 seconds
......@@ -82,6 +82,7 @@
<Compile Include="Structs.cs" />
<Compile Include="Recomp\TokenTypes.cs" />
<Compile Include="Utils.cs" />
<Compile Include="LazyArray.cs" />
<Compile Include="_LitJson\IJsonWrapper.cs" />
<Compile Include="_LitJson\JsonData.cs" />
<Compile Include="_LitJson\JsonException.cs" />
......@@ -118,4 +119,4 @@
<Target Name="AfterBuild">
</Target>
-->
</Project>
\ No newline at end of file
</Project>
This diff is collapsed.
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Runtime.InteropServices;
using System.Text;
namespace Altar
{
using static SR;
// TODO: LANG DAFL | EMBI
public enum SectionHeaders : uint
{
Form = 0x4D524F46, // FORM
......@@ -18,7 +23,7 @@ namespace Altar
Backgrounds = 0x444E4742, // BGND
Paths = 0x48544150, // PATH
Scripts = 0x54504353, // SCPT
GLOB_Unk = 0x424F4C47, // GLOB
Globals = 0x424F4C47, // GLOB
Shaders = 0x52444853, // SHDR
Fonts = 0x544E4F46, // FONT
Timelines = 0x4E4C4D54, // TMLN
......@@ -32,8 +37,9 @@ namespace Altar
Strings = 0x47525453, // STRG
Textures = 0x52545854, // TXTR
Audio = 0x4F445541, // AUDO
EmbedImage = 0x49424D45, // EMBI
Count = 24
Count = 25
}
public static class SectionHeadersExtensions
{
......@@ -118,31 +124,72 @@ namespace Altar
public SectionHeader * Form ;
public SectionGeneral* General;
public SectionOptions* Options;
public SectionCountOffsets* Sounds ;
public SectionCountOffsets* Sprites ;
public SectionCountOffsets* Backgrounds ;
public SectionCountOffsets* Paths ;
public SectionCountOffsets* Scripts ;
public SectionCountOffsets* Fonts ;
public SectionCountOffsets* Objects ;
public SectionCountOffsets* Rooms ;
public SectionCountOffsets* TexturePages;
public SectionCountOffsets* Code ;
public SectionCountOffsets* Strings ;
public SectionCountOffsets* Textures ;
public SectionCountOffsets* Audio ;
public SectionCountOffsets* AudioGroup ;
public SectionRefDefs* Functions;
public SectionRefDefs* Variables;
public Dictionary<SectionHeaders, IntPtr> UnknownChunks = new Dictionary<SectionHeaders, IntPtr>();
public SectionGeneral* General => (SectionGeneral*)GetChunk(SectionHeaders.General);
public SectionOptions* Options => (SectionOptions*)GetChunk(SectionHeaders.Options);
public SectionGlobals* Globals => (SectionGlobals*)GetChunk(SectionHeaders.Globals);
public SectionCountOffsets* Sounds => (SectionCountOffsets*)GetChunk(SectionHeaders.Sounds );
public SectionCountOffsets* Sprites => (SectionCountOffsets*)GetChunk(SectionHeaders.Sprites );
public SectionCountOffsets* Backgrounds => (SectionCountOffsets*)GetChunk(SectionHeaders.Backgrounds );
public SectionCountOffsets* Paths => (SectionCountOffsets*)GetChunk(SectionHeaders.Paths );
public SectionCountOffsets* Scripts => (SectionCountOffsets*)GetChunk(SectionHeaders.Scripts );
public SectionCountOffsets* Fonts => (SectionCountOffsets*)GetChunk(SectionHeaders.Fonts );
public SectionCountOffsets* Objects => (SectionCountOffsets*)GetChunk(SectionHeaders.Objects );
public SectionCountOffsets* Rooms => (SectionCountOffsets*)GetChunk(SectionHeaders.Rooms );
public SectionCountOffsets* TexturePages => (SectionCountOffsets*)GetChunk(SectionHeaders.TexturePage );
public SectionCountOffsets* Code => (SectionCountOffsets*)GetChunk(SectionHeaders.Code );
public SectionCountOffsets* Strings => (SectionCountOffsets*)GetChunk(SectionHeaders.Strings );
public SectionCountOffsets* Textures => (SectionCountOffsets*)GetChunk(SectionHeaders.Textures );
public SectionCountOffsets* Audio => (SectionCountOffsets*)GetChunk(SectionHeaders.Audio );
public SectionCountOffsets* AudioGroup => (SectionCountOffsets*)GetChunk(SectionHeaders.AudioGroup );
public SectionCountOffsets* Extensions => (SectionCountOffsets*)GetChunk(SectionHeaders.Extensions );
public SectionCountOffsets* Shaders => (SectionCountOffsets*)GetChunk(SectionHeaders.Shaders );
public SectionCountOffsets* Timelines => (SectionCountOffsets*)GetChunk(SectionHeaders.Timelines );
public SectionCountOffsets* EmbedImage => (SectionCountOffsets*)GetChunk(SectionHeaders.EmbedImage );
public SectionRefDefs* Functions => (SectionRefDefs*)GetChunk(SectionHeaders.Functions);
public SectionRefDefs* Variables => (SectionRefDefs*)GetChunk(SectionHeaders.Variables);
public Dictionary<SectionHeaders, IntPtr> Chunks = new Dictionary<SectionHeaders, IntPtr>();
internal long[] HeaderOffsets = new long[(int)SectionHeaders.Count];
public GMFileContent(byte[] data)
{
RawData = new UniquePtr(data);
byte* hdr_b = RawData.BPtr;
var basePtr = (SectionHeader*)hdr_b;
Form = basePtr;
if (Form->Identity != SectionHeaders.Form)
throw new InvalidDataException(ERR_NO_FORM);
SectionHeader*
hdr = basePtr + 1,
hdrEnd = (SectionHeader*)((IntPtr)basePtr + (int)Form->Size);
int headersMet = 0;
for (; hdr < hdrEnd; hdr = unchecked((SectionHeader*)((IntPtr)hdr + (int)hdr->Size) + 1), ++headersMet)
{
Chunks.Add(hdr->Identity, (IntPtr)hdr);
for (int i = 0; i < HeaderOffsets.Length; i++)
if (((SectionHeader*)((byte*)basePtr + HeaderOffsets[i]))->Identity == hdr->Identity)
Console.Error.WriteLine($"WARNING: chunk {hdr->MagicString()} encountered (at least) twice! Only the last occurrence will be exported! (If you see this message, consider reversing manually.)");
if (HeaderOffsets.Length >= headersMet)
{
var ho = HeaderOffsets;
Array.Resize(ref ho, (headersMet == HeaderOffsets.Length) ? 1 : (headersMet + 2));
HeaderOffsets = ho;
}
HeaderOffsets[headersMet] = (byte*)hdr - (byte*)basePtr;
}
}
public void DumpChunkOffs()
{
for (int i = 0; i < HeaderOffsets.Length; ++i)
......@@ -156,56 +203,10 @@ namespace Altar
public SectionHeader* GetChunk(SectionHeaders ident)
{
switch (ident)
{
case SectionHeaders.Form:
return (SectionHeader*)Form;
case SectionHeaders.General:
return (SectionHeader*)General;
case SectionHeaders.Options:
return (SectionHeader*)Options;
case SectionHeaders.Sounds:
return (SectionHeader*)Sounds;
case SectionHeaders.Sprites:
return (SectionHeader*)Sprites;
case SectionHeaders.Backgrounds:
return (SectionHeader*)Backgrounds;
case SectionHeaders.Paths:
return (SectionHeader*)Paths;
case SectionHeaders.Scripts:
return (SectionHeader*)Scripts;
case SectionHeaders.Fonts:
return (SectionHeader*)Fonts;
case SectionHeaders.Objects:
return (SectionHeader*)Objects;
case SectionHeaders.Rooms:
return (SectionHeader*)Rooms;
case SectionHeaders.TexturePage:
return (SectionHeader*)TexturePages;
case SectionHeaders.Code:
return (SectionHeader*)Code;
case SectionHeaders.Strings:
return (SectionHeader*)Strings;
case SectionHeaders.Textures:
return (SectionHeader*)Textures;
case SectionHeaders.Audio:
return (SectionHeader*)Audio;
case SectionHeaders.AudioGroup:
return (SectionHeader*)AudioGroup;
case SectionHeaders.Functions:
return (SectionHeader*)Functions;
case SectionHeaders.Variables:
return (SectionHeader*)Variables;
default:
if (UnknownChunks.ContainsKey(ident))
return (SectionHeader*)UnknownChunks[ident];
return null;
}
if (Chunks.ContainsKey(ident))
return (SectionHeader*)Chunks[ident];
return null;
}
void Disposing()
......@@ -228,34 +229,5 @@ namespace Altar
Disposing();
}
}
public unsafe class AGRPFileContent : IDisposable
{
public UniquePtr RawData;
public SectionHeader* Form;
public SectionCountOffsets* Audo; // not the same as a data.win AGRP!
void Disposing()
{
if (RawData != null)
{
RawData.Dispose();
RawData = null;
}
}
public void Dispose()
{
Disposing();
GC.SuppressFinalize(this);
}
~AGRPFileContent()
{
Disposing();
}
}
}
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
namespace Altar
{
public class LazyArray<T> : IList<T>
{
public struct Etor : IEnumerator<T>
{
uint curind;
LazyArray<T> arr;
internal Etor(LazyArray<T> arr)
{
curind = 0;
this.arr = arr;
}
public T Current => arr[curind];
object IEnumerator.Current => arr[curind];
public bool MoveNext()
{
++curind;
return curind < arr.Length;
}
public void Reset()
{
curind = 0;
}
public void Dispose() { curind = 0; arr = null; }
}
Dictionary<uint, KeyValuePair<bool, T>> cache =
new Dictionary<uint, KeyValuePair<bool, T>>();
Func<uint, KeyValuePair<bool, T>> getter;
uint max;
public LazyArray(Func<uint, KeyValuePair<bool, T>> get, uint max)
{
if (get == null && max != 0)
throw new ArgumentNullException(nameof(get));
getter = get;
this.max = max;
}
public int Count => (int)max;
public int Length => (int)max;
public int InCache => cache.Count;
public bool IsReadOnly => true;
public T this[uint ind]
{
get
{
if (ind >= max) throw new IndexOutOfRangeException();
KeyValuePair<bool, T> v;
if (cache.TryGetValue(ind, out v))
if (v.Key) return v.Value;
else throw new IndexOutOfRangeException();
var res = cache[ind] = getter(ind);
if (res.Key) return res.Value;
throw new IndexOutOfRangeException();
}
}
public T this[int ind]
{
get { return this[(uint)ind]; }
set { throw new NotImplementedException(); }
}
public void Add(T _)
{
throw new NotImplementedException();
}
public void Clear()
{
cache.Clear();
}
public bool Contains(T t)
{
var v = new KeyValuePair<bool, T>(true, t);
return cache.ContainsValue(v);
}
public bool Remove(T v)
{
uint kkk;
foreach (var kvp in cache)
if (kvp.Value.Key && (ReferenceEquals(kvp.Value, v) ||
(!ReferenceEquals(kvp.Value.Value, null)
&& kvp.Value.Value.Equals(v))))
{
kkk = kvp.Key;
goto FoundOne;
}
return false;
FoundOne:
cache.Remove(kkk);
return true;
}
public void CopyTo(T[] dest, int off)
{
for (int i = 0; i < Math.Min(max, dest.Length - off); ++i)
dest[off+i] = this[i];
}
public int IndexOf(T v)
{
foreach (var kvp in cache)
if (kvp.Value.Key && (ReferenceEquals(kvp.Value, v) ||
(!ReferenceEquals(kvp.Value.Value, null)
&& kvp.Value.Value.Equals(v))))
return (int)kvp.Key;
return -1;
}
public void Insert(int ind, T v)
{
throw new NotImplementedException();
}
public void RemoveAt(int ind)
{
KeyValuePair<bool, T> kvp;
if (cache.TryGetValue((uint)ind, out kvp) && kvp.Key)
cache.Remove((uint)ind);
}
public IEnumerator<T> GetEnumerator () => new Etor(this);
IEnumerator IEnumerable.GetEnumerator() => new Etor(this);
public static implicit operator T[](LazyArray<T> la) => la.ToArray();
public static implicit operator LazyArray<T>(T[] ar) =>
new LazyArray<T>(i => new KeyValuePair<bool, T>(true, ar[i]), unchecked((uint)ar.Length));
}
}
......@@ -3,21 +3,6 @@ using CommandLine.Text;
namespace Altar
{
public class ExportAgrpOptions
{
[Option('f', "file", Required = true, DefaultValue = "audiogroup1.dat", HelpText="Input file (default: 'audiogroup1.dat')")]
public string File
{
get;
set;
}
[Option('o', "out", Required = true, DefaultValue = ".", HelpText = "Output directory (default: current)")]
public string OutputDirectory
{
get;
set;
}
}
public class ExportOptions
{
// free short flags: lvxyz
......@@ -243,9 +228,6 @@ namespace Altar
set;
}
[VerbOption("export-agrp", HelpText="Export contents of an audio group file (eg. audiogroup1.dat).")]
public ExportAgrpOptions ExportAgrp{get;set;}
[HelpVerbOption]
public string GetUsage(string v) => HelpText.AutoBuild(this, v);
}
......
......@@ -150,6 +150,14 @@ namespace Altar
public fixed uint _pad1[0xC];
public CountOffsetsPair ConstMap;
}
[StructLayout(LayoutKind.Sequential, Pack = 1)]
public unsafe struct SectionGlobals
{
public SectionHeaders Header;
public uint Count;
public uint CodeIDs;
}
[StructLayout(LayoutKind.Sequential, Pack = 1)]
public unsafe struct SectionUnknown
......@@ -541,4 +549,128 @@ namespace Altar
public uint FunctionName;
public FunctionLocalEntry Locals;
}
public enum ExtensionType : uint
{
Unknown0 = 0,
SharedLib = 1,
GML = 2,
Unknown1 = 3,
DataFile = 4, // or "generic placeholder"
Javascript = 5
}
public enum ExtensionFFIType : uint
{
String = 1,
Double = 2
}
[StructLayout(LayoutKind.Sequential, Pack = 1)]
public unsafe struct ExtensionFunctionEntry
{
public uint GMLName;
public uint ID;
public ExtensionType Type; // is this the correct type?
  • Nope, this is actually the calling convention type. 0x0c for CDECL iirc, there is one more option in the UI but never checked what value it was

Please register or sign in to reply
public ExtensionFFIType ReturnType;
public uint SymbolName;
public uint ArgumentCount;
public ExtensionFFIType ArgumentTypes;
}
[StructLayout(LayoutKind.Sequential, Pack = 1)]
public unsafe struct ExtensionFileEntry
{
public uint Filename;
public uint KillSymbol;
public uint InitSymbol;
public ExtensionType Type;
public CountOffsetsPair Functions;
}
[StructLayout(LayoutKind.Sequential, Pack = 1)]
public unsafe struct ExtensionEntry
{
public uint EmptyString;
public uint Name;
public uint ClassName;
public CountOffsetsPair Includes;
}
[StructLayout(LayoutKind.Sequential, Pack = 1)]
public unsafe struct TimelineEventEntry
{
public uint EventID; // CODE ?
}
[StructLayout(LayoutKind.Sequential, Pack = 1)]
public unsafe struct TimelineKeyframeEntry
{
public uint Time;
public uint OffsetToEvents; // CountOffsetsPair<TimelineEventEntry>*
}
[StructLayout(LayoutKind.Sequential, Pack = 1)]
public unsafe struct TimelineEntry
{
public uint Name;
public uint KeyframeCount;
public TimelineKeyframeEntry Keyframes;
}
public enum ShaderType : uint
{
GLSL_ES = 1,
GLSL = 2,
Unknown3 = 3,
HLSL = 4,
Unknown5 = 5,
Unknown6 = 6,
Unknown7 = 7
}
public enum ShaderTypeEnc : uint
{
WeirdBit = (uint)1<<31,
GLSL_ES = (uint)ShaderType.GLSL_ES | WeirdBit,
GLSL = (uint)ShaderType.GLSL | WeirdBit,
Unknown3 = (uint)ShaderType.Unknown3 | WeirdBit,
HLSL = (uint)ShaderType.HLSL | WeirdBit,
Unknown5 = (uint)ShaderType.Unknown5 | WeirdBit,
Unknown6 = (uint)ShaderType.Unknown6 | WeirdBit,
Unknown7 = (uint)ShaderType.Unknown7 | WeirdBit
}
public static class ShaderTypeExt
{
public static ShaderTypeEnc Encode(this ShaderType st) =>
(ShaderTypeEnc)st | ShaderTypeEnc.WeirdBit;
}
public static class ShaderTypeEncExt
{
public static ShaderType Decode(this ShaderTypeEnc ste) =>
(ShaderType)(ste & ~ShaderTypeEnc.WeirdBit);
}
[StructLayout(LayoutKind.Sequential, Pack = 1)]
public unsafe struct ShaderVxFxStrings
{
public uint VertexSource, FragmentSource; // strg offsets
}
[StructLayout(LayoutKind.Sequential, Pack = 1)]
public unsafe struct ShaderVxFxBlobs
{
public uint VertexData, VertexLength, // *Data: offset to an array of length *Length
FragmentData, FragmentLength;
}
[StructLayout(LayoutKind.Sequential, Pack = 1)]
public unsafe struct ShaderEntry
{
public uint Name;
public ShaderTypeEnc Type;
public ShaderVxFxStrings GLSL_ES, GLSL, HLSL9;
public ShaderVxFxBlobs HLSL11;
public uint AttributeCount;
public uint Attributes; // inline array, all strg offsets
}
[StructLayout(LayoutKind.Sequential, Pack = 1)]
public unsafe struct ShaderEntry2
{
public uint Unknown; // always 2
public ShaderVxFxBlobs PSSL, Cg, Cg_PS3;
// actual shader blobs are somewhere down here
}
}
......@@ -208,14 +208,14 @@ namespace Altar
if (!Directory.Exists(odgrp))
Directory.CreateDirectory(odgrp);
using (var af = AGRPFile.GetFile(agrpfn))
using (var af = GMFile.GetFile(agrpfn))
{
for (int j = 0; j < af.Waves.Length; ++j)
for (int j = 0; j < af.Audio.Length; ++j)
{
SetCAndWr(cl, ct, O_PAREN + (j + 1) + SLASH +
(af.Waves.Length - 1) + C_PAREN);
(af.Audio.Length - 1) + C_PAREN);
File.WriteAllBytes(odgrp + Path.DirectorySeparatorChar
+ infoTable[j].Name + SR.EXT_WAV, af.Waves[j]);
+ infoTable[j].Name + SR.EXT_WAV, af.Audio[j].Wave);
}
}
Console.WriteLine();
......@@ -506,29 +506,9 @@ namespace Altar
var c = f.Content;
if (eo.DumpAllChunks)
{
chunks.Add((IntPtr)c.General );
chunks.Add((IntPtr)c.Options );
chunks.Add((IntPtr)c.Sounds );
chunks.Add((IntPtr)c.AudioGroup );
chunks.Add((IntPtr)c.Sprites );
chunks.Add((IntPtr)c.Backgrounds );
chunks.Add((IntPtr)c.Paths );
chunks.Add((IntPtr)c.Scripts );
chunks.Add((IntPtr)c.Fonts );
chunks.Add((IntPtr)c.Objects );
chunks.Add((IntPtr)c.Rooms );
chunks.Add((IntPtr)c.TexturePages);
chunks.Add((IntPtr)c.Code );
chunks.Add((IntPtr)c.Variables );
chunks.Add((IntPtr)c.Functions );
chunks.Add((IntPtr)c.Strings );
chunks.Add((IntPtr)c.Textures );
chunks.Add((IntPtr)c.Audio );
}
chunks.AddRange(c.Chunks.Values);
chunks.AddRange(c.UnknownChunks.Values);
// TODO: how to filter out unknowns?
for (int i = 0; i < chunks.Count; i++)
DumpUnk(chunks[i]);
......@@ -799,31 +779,6 @@ namespace Altar
return writer.Buffer.ReadBytes(writer.Buffer.Size);
}
static void ExportAgrp(ExportAgrpOptions opt)
{
var file = Path.GetFullPath(opt.File);
var outd = Path.GetFullPath(opt.OutputDirectory);
if (!Directory.Exists(outd))
Directory.CreateDirectory(outd);
if (!File.Exists(file))
throw new ParserException("File \"" + file + "\" doesn't exist.");
nopp = quiet = false; // TODO: add these flags
// or better yet, make agrps work with `export`
using (var f = AGRPFile.GetFile(file))
{
int cl = 0, ct = 0;
WrAndGetC("Exporting AGRP audio... ", out cl, out ct);
for (int i = 0; i < f.Waves.Length; ++i)
{
SetCAndWr(cl, ct, O_PAREN + (i + 1) + SLASH + f.Waves.Length + C_PAREN);
File.WriteAllBytes("audo" + i.ToString("D3") + ".wav", f.Waves[i]);
}
}
Console.WriteLine();
}
[STAThread]
static void Main(string[] args)
{
......@@ -846,9 +801,6 @@ namespace Altar
case "import":
Import((ImportOptions)vo);
break;
case "export-agrp":
ExportAgrp((ExportAgrpOptions)vo);
break;
}
}
catch (Exception)
......
This diff is collapsed.
......@@ -64,6 +64,11 @@ namespace Altar
public uint[] _pad0;
public uint[] _pad1;
}
[StructLayout(LayoutKind.Sequential)]
public struct GlobalInfo
{
public uint[] GlobalCodeIDs;
}
[StructLayout(LayoutKind.Sequential)]
public struct SoundInfo
......@@ -202,7 +207,7 @@ namespace Altar
public int Size;
internal int ArgumentCount;
internal AnyInstruction[] InstructionsCopy; // I ain't dealin' with no pointers
// (TODO: don't use pointers)
// (TODO: don't use pointers) // nah it's fine :D
internal IList<Tuple<ReferenceSignature, uint>> functionReferences;
internal IList<Tuple<ReferenceSignature, uint>> variableReferences;
}
......@@ -295,4 +300,71 @@ namespace Altar
public string FunctionName;
public string[] LocalNames;
}
[StructLayout(LayoutKind.Sequential)]
public struct ExtensionFunctionInfo
{
public string GMLName;
public uint ID;
public ExtensionType Type;
public ExtensionFFIType ReturnType;
public string SymbolName;
public ExtensionFFIType[] Arguments;
}
[StructLayout(LayoutKind.Sequential)]
public struct ExtensionFileInfo
{
public string Filename;
public string KillSymbol;
public string InitSymbol;
public ExtensionType Type;
public ExtensionFunctionInfo[] Functions;
}
[StructLayout(LayoutKind.Sequential)]
public struct ExtensionInfo
{
public string Name;
public string ClassName;
public ExtensionFileInfo[] Includes;
}