Skip to content

Commit

Permalink
Split TokenizeOutput.cs into partials
Browse files Browse the repository at this point in the history
  • Loading branch information
budgetdevv committed Oct 6, 2024
1 parent 15da6e2 commit 09c91ea
Showing 1 changed file with 18 additions and 4 deletions.
22 changes: 18 additions & 4 deletions Tokenizers.NET/TokenizeOutput.cs
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
using System;
using System.Diagnostics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using Tokenizers.NET.Collections;
#if DEBUG
using System.Diagnostics;
#endif

namespace Tokenizers.NET
{
Expand Down Expand Up @@ -37,8 +39,8 @@ public interface ITokenizeOutput
NativeBuffer<uint> ITokenizeOutput.TokenTypeIDs => TokenTypeIDs;
}

[StructLayout(LayoutKind.Sequential)]
public readonly unsafe struct TokenizeOutput: ITokenizeOutput, IDisposable
[StructLayout(LayoutKind.Sequential)] // Data structures
public readonly unsafe partial struct TokenizeOutput: ITokenizeOutput, IDisposable
{
public readonly NativeBuffer<uint> IDs;

Expand Down Expand Up @@ -107,7 +109,11 @@ public static NativeBuffer<uint> AccessField<T>(T item)
return item.TokenTypeIDs;
}
}

}

// Gather APIs
public readonly unsafe partial struct TokenizeOutput
{
public void GatherIDsInclusiveOfOverflowing(
NativeBuffer<uint> idsBuffer,
out nuint totalLength)
Expand Down Expand Up @@ -250,7 +256,11 @@ private void GatherIDsInclusiveOfOverflowingCore<FieldAccessorT>(
return;
}
}
}

// Gather and Widen APIs
public readonly unsafe partial struct TokenizeOutput
{
public void GatherAndWidenIDsInclusiveOfOverflowing(
NativeBuffer<ulong> idsBuffer,
out nuint totalLength)
Expand Down Expand Up @@ -393,7 +403,11 @@ private void GatherAndWidenIDsInclusiveOfOverflowingCore<FieldAccessorT>(
return;
}
}
}

// Dispose
public readonly unsafe partial struct TokenizeOutput
{
private const int NUM_HANDLES = 2;

[InlineArray(NUM_HANDLES)]
Expand Down

0 comments on commit 09c91ea

Please sign in to comment.