Skip to content

Commit

Permalink
libnixt: using C++ streams to handle buffers
Browse files Browse the repository at this point in the history
  • Loading branch information
@inclyc
inclyc committed Dec 23, 2023
1 parent 1839e9e commit a3be3d8
Show file tree
Hide file tree
Showing 4 changed files with 87 additions and 129 deletions.
38 changes: 16 additions & 22 deletions libnixt/include/nixt/Serialize.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,14 @@
#include <cstddef>
#include <cstdint>
#include <cstring>
#include <sstream>

/// \file
/// This file contains serialization method for nix::Expr

namespace nixt::serialize {

using StringIdx = StringTable::Size;
using StringIdx = std::size_t;
using PosIdx = std::size_t;
using ExprIdx = std::size_t;

Expand Down Expand Up @@ -115,33 +116,26 @@ struct ExprAttrSet {
std::vector<DynamicAttrDef> DynamicAttrs;
};

std::size_t encode(char *Dst, const ExprAttrSet &E);

std::size_t getSize(const ExprAttrSet &E);
inline std::ostream &write(std::ostream &OS, const ExprAttrSet &E) {
return OS;
}

struct ASTHeader {
unsigned char NixAST[8] = "\x7fNixAST";
char Magic[8];
uint32_t Version;
std::size_t ExprTableOffset;
std::size_t PosTableOffset;
std::size_t StringTableOffset;
};

class ASTWriter {
public:
ASTWriter(const nix::SymbolTable &STable, const nix::PosTable &PTable)
: STable(STable), PTable(PTable) {}

std::size_t count(const nix::Expr *E);

void write(char *Dst);

private:
const nix::SymbolTable &STable;
const nix::PosTable &PTable;
IndexVector Exprs;
IndexVector Pos;
struct ASTData {
std::ostringstream Exprs;
std::ostringstream Pos;
StringTable Strings;
};

/// Traverse the AST, flatten all nodes into "ASTData".
ASTData collect(const nix::SymbolTable &STable, const nix::PosTable &PTable,
const nix::Expr *E);

/// Emit ASTData into ostream.
std::ostream &write(std::ostream &OS, const ASTData &Data);

} // namespace nixt::serialize
76 changes: 28 additions & 48 deletions libnixt/include/nixt/SerializeSupport.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

#include <cstring>
#include <map>
#include <sstream>
#include <string>
#include <type_traits>
#include <vector>
Expand All @@ -12,70 +13,49 @@ namespace nixt::serialize {
template <class T>
concept pod = std::is_standard_layout_v<T> && std::is_trivial_v<T>;

/// Basic primitives. Trivial data types are just written to a stream.
template <class T>
requires pod<T>
std::size_t encode(char *Dst, const T &Data) {
std::memcpy(Dst, &Data, sizeof(T));
return sizeof(T);
std::ostream &write(std::ostream &OS, const T &Data) {
OS.write(reinterpret_cast<const char *>(&Data), sizeof(Data));
return OS;
}

/// Vector. Write each element.
template <class T>
requires pod<T>
std::size_t getSize(const T &) {
return sizeof(T);
std::ostream &write(std::ostream &OS, const std::vector<T> &Data) {
for (const auto &E : Data)
write(OS, E);
return OS;
}

template <class T> std::size_t encode(char *Dst, const std::vector<T> &Data) {
std::size_t Ret = encode(Dst, Data.size());
for (const auto &E : Data) {
Ret += encode(Dst + Ret, E);
}
return Ret;
/// Place an element into Indexed Vector
/// \returns the position for further element references
template <class T> std::size_t place(std::ostringstream &Self, T Elt) {
std::size_t Loc = Self.tellp();
write(Self, Elt);
return Loc;
}

template <class T> std::size_t getSize(const std::vector<T> &Data) {
std::size_t Ret = sizeof(Data.size());
for (const auto &E : Data) {
Ret += getSize(E);
}
return Ret;
}

/// Wrapper around std::vector that we can easily get indexes of added element.
struct IndexVector {
template <class T> std::size_t add(T Elt) {
std::size_t EltSize = getSize(Elt);
Data.reserve(Data.size() + EltSize);
Data.resize(Data.size() + EltSize);
return encode(Data.data() + Data.size() - EltSize, Elt);
}
IndexVector() = default;
std::vector<char> Data;
};

inline std::size_t encode(char *Dst, const IndexVector &Data) {
return encode(Dst, Data.Data);
}
/// String. Write the string and a null terminator.
std::ostream &write(std::ostream &OS, const std::string &Data);

inline std::size_t getSize(const IndexVector &Data) {
return getSize(Data.Data);
}
std::ostream &write(std::ostream &OS, const std::ostringstream &Data);

/// The string table. This is actually a vector of strings, but de-duplicate
/// while construction.
class StringTable {
public:
using Size = std::size_t;
Size add(std::string Str);

friend std::size_t encode(char *Dst, const StringTable &Table);
friend std::size_t place(StringTable &Self, std::string Str);

friend std::size_t getSize(const StringTable &Table);

void write(char *Dst) const;

StringTable() = default;
friend std::ostream &write(std::ostream &OS, const StringTable &Data) {
for (const auto &E : Data.Strings)
write(OS, E);
return OS;
}

private:
std::map<std::string, Size> Map;
std::map<std::string, std::size_t> Map;
std::vector<std::string> Strings;
uint32_t Length = 0;
};
Expand Down
72 changes: 28 additions & 44 deletions libnixt/lib/Serialize.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,19 +10,18 @@

namespace nixt::serialize {

std::size_t ASTWriter::count(const nix::Expr *E) {
struct ASTWriteVisitor : public RecursiveASTVisitor<ASTWriteVisitor> {
IndexVector &Exprs;
IndexVector &Pos;
StringTable &Strings;
ASTData collect(const nix::SymbolTable &STable, const nix::PosTable &PTable,
const nix::Expr *E) {
ASTData Data;
struct Visitor : public RecursiveASTVisitor<Visitor> {
ASTData &Data;
const nix::SymbolTable &STable;
const nix::PosTable &PTable;
std::map<const nix::Expr *, std::size_t> ExprMap;

ASTWriteVisitor(IndexVector &Exprs, IndexVector &Pos, StringTable &Strings,
const nix::SymbolTable &STable, const nix::PosTable &PTable)
: Exprs(Exprs), Pos(Pos), Strings(Strings), STable(STable),
PTable(PTable) {}
Visitor(ASTData &Data, const nix::SymbolTable &STable,
const nix::PosTable &PTable)
: Data(Data), STable(STable), PTable(PTable) {}

Position getPos(const nix::PosIdx P) {
if (P == nix::noPos) {
Expand All @@ -37,62 +36,47 @@ std::size_t ASTWriter::count(const nix::Expr *E) {
Position::OK_SourcePath};
Ret.Kind = KindTable[Pos.origin.index()];
if (Ret.Kind == Position::OK_SourcePath)
Ret.Source =
Strings.add(std::get<nix::SourcePath>(Pos.origin).to_string());
Ret.Source = place(Data.Strings,
std::get<nix::SourcePath>(Pos.origin).to_string());
return Ret;
}
bool visitExprAttrs(const nix::ExprAttrs *E) {
ExprAttrSet Expr;
Expr.Pos = Pos.add(getPos(E->pos));
Expr.Pos = place(Data.Pos, getPos(E->pos));
for (const auto &[K, V] : E->attrs) {
Expr.Attrs.emplace_back(AttrDefEntry{
Strings.add(STable[K]),
{V.inherited, ExprMap[V.e], Pos.add(getPos(V.pos)), V.displ}});
Expr.Attrs.emplace_back(
AttrDefEntry{place(Data.Strings, (STable[K])),
{V.inherited, ExprMap[V.e],
place(Data.Pos, (getPos(V.pos))), V.displ}});
}
ExprMap[E] = Exprs.add(Expr);
ExprMap[E] = place(Data.Exprs, Expr);
return true;
}
bool visitExprVar(const nix::ExprVar *E) {
SimpleExpr Expr;
Expr.Kind = ExprKind::EK_Var;
Expr.Body.Var.Displ = E->displ;
Expr.Body.Var.FromWidth = E->fromWith;
Expr.Body.Var.Name = Strings.add(STable[E->name]);
Expr.Body.Var.Pos = Pos.add(getPos(E->pos));
ExprMap[E] = Exprs.add(Expr);
Expr.Body.Var.Name = place(Data.Strings, STable[E->name]);
Expr.Body.Var.Pos = place(Data.Pos, getPos(E->pos));
ExprMap[E] = place(Data.Exprs, Expr);
return true;
}

} Writer(Exprs, Pos, Strings, STable, PTable);
} Writer(Data, STable, PTable);
Writer.traverseExpr(E);

return sizeof(ASTHeader) + getSize(Exprs) + getSize(Pos) + getSize(Strings);
return Data;
}

void ASTWriter::write(char *Dst) {
std::ostream &write(std::ostream &OS, const ASTData &Data) {
ASTHeader Header;
strcpy(Header.Magic, "\x7fNixAST");
Header.Version = 1;
Header.ExprTableOffset = sizeof(ASTHeader);
Header.PosTableOffset = Header.ExprTableOffset + getSize(Exprs);
Header.StringTableOffset = Header.PosTableOffset + getSize(Pos);
memcpy(Dst, &Header, sizeof(ASTHeader));
Dst += sizeof(ASTHeader);
Dst += encode(Dst, Exprs);
Dst += encode(Dst, Pos);
Dst += encode(Dst, Strings);
}

std::size_t encode(char *Dst, const ExprAttrSet &E) {
std::size_t Ret = encode(Dst, E.Kind);
Ret += encode(Dst + Ret, E.Pos);
Ret += encode(Dst + Ret, E.Attrs);
Ret += encode(Dst + Ret, E.DynamicAttrs);
return Ret;
}

std::size_t getSize(const ExprAttrSet &E) {
return sizeof(E.Kind) + getSize(E.Pos) + getSize(E.Attrs) +
getSize(E.DynamicAttrs);
write(OS, Header);
write(OS, Data.Exprs);
write(OS, Data.Pos);
write(OS, Data.Strings);
return OS;
}

} // namespace nixt::serialize
30 changes: 15 additions & 15 deletions libnixt/lib/SerializeSupport.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,24 +2,24 @@

namespace nixt::serialize {

StringTable::Size StringTable::add(std::string Str) {
if (Map.contains(Str)) {
return Map[std::move(Str)];
}
Length += Str.size() + 1; // size of string + null terminator
Size Ret = Map[Str] = Length;
Strings.push_back(std::move(Str));
return Ret;
std::ostream &write(std::ostream &OS, const std::string &Data) {
OS.write(Data.c_str(), Data.size() + 1);
return OS;
}

std::size_t encode(char *Dst, const StringTable &Table) {
for (const auto &S : Table.Strings) {
memcpy(Dst, S.c_str(), S.size() + 1);
Dst = static_cast<char *>(Dst) + S.size() + 1;
}
return Table.Length;
std::ostream &write(std::ostream &OS, const std::ostringstream &Data) {
write(OS, Data.str());
return OS;
}

std::size_t getSize(const StringTable &Table) { return Table.Length; }
std::size_t place(StringTable &Self, std::string Str) {
if (Self.Map.contains(Str)) {
return Self.Map[std::move(Str)];
}
Self.Length += Str.size() + 1; // size of string + null terminator
std::size_t Ret = Self.Map[Str] = Self.Length;
Self.Strings.push_back(std::move(Str));
return Ret;
}

} // namespace nixt::serialize

0 comments on commit a3be3d8

Please sign in to comment.