From a3be3d852a922badd0615c7810b0ca2e8780b5e0 Mon Sep 17 00:00:00 2001 From: Yingchi Long Date: Sat, 23 Dec 2023 18:35:23 +0800 Subject: [PATCH] libnixt: using C++ streams to handle buffers --- libnixt/include/nixt/Serialize.h | 38 ++++++------- libnixt/include/nixt/SerializeSupport.h | 76 +++++++++---------------- libnixt/lib/Serialize.cpp | 72 +++++++++-------------- libnixt/lib/SerializeSupport.cpp | 30 +++++----- 4 files changed, 87 insertions(+), 129 deletions(-) diff --git a/libnixt/include/nixt/Serialize.h b/libnixt/include/nixt/Serialize.h index 5f05628ba..ceb0d785b 100644 --- a/libnixt/include/nixt/Serialize.h +++ b/libnixt/include/nixt/Serialize.h @@ -9,13 +9,14 @@ #include #include #include +#include /// \file /// This file contains serialization method for nix::Expr namespace nixt::serialize { -using StringIdx = StringTable::Size; +using StringIdx = std::size_t; using PosIdx = std::size_t; using ExprIdx = std::size_t; @@ -115,33 +116,26 @@ struct ExprAttrSet { std::vector DynamicAttrs; }; -std::size_t encode(char *Dst, const ExprAttrSet &E); - -std::size_t getSize(const ExprAttrSet &E); +inline std::ostream &write(std::ostream &OS, const ExprAttrSet &E) { + return OS; +} struct ASTHeader { - unsigned char NixAST[8] = "\x7fNixAST"; + char Magic[8]; uint32_t Version; - std::size_t ExprTableOffset; - std::size_t PosTableOffset; - std::size_t StringTableOffset; }; -class ASTWriter { -public: - ASTWriter(const nix::SymbolTable &STable, const nix::PosTable &PTable) - : STable(STable), PTable(PTable) {} - - std::size_t count(const nix::Expr *E); - - void write(char *Dst); - -private: - const nix::SymbolTable &STable; - const nix::PosTable &PTable; - IndexVector Exprs; - IndexVector Pos; +struct ASTData { + std::ostringstream Exprs; + std::ostringstream Pos; StringTable Strings; }; +/// Traverse the AST, flatten all nodes into "ASTData". +ASTData collect(const nix::SymbolTable &STable, const nix::PosTable &PTable, + const nix::Expr *E); + +/// Emit ASTData into ostream. +std::ostream &write(std::ostream &OS, const ASTData &Data); + } // namespace nixt::serialize diff --git a/libnixt/include/nixt/SerializeSupport.h b/libnixt/include/nixt/SerializeSupport.h index 57b117b29..bff1e8863 100644 --- a/libnixt/include/nixt/SerializeSupport.h +++ b/libnixt/include/nixt/SerializeSupport.h @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -12,70 +13,49 @@ namespace nixt::serialize { template concept pod = std::is_standard_layout_v && std::is_trivial_v; +/// Basic primitives. Trivial data types are just written to a stream. template requires pod -std::size_t encode(char *Dst, const T &Data) { - std::memcpy(Dst, &Data, sizeof(T)); - return sizeof(T); +std::ostream &write(std::ostream &OS, const T &Data) { + OS.write(reinterpret_cast(&Data), sizeof(Data)); + return OS; } +/// Vector. Write each element. template - requires pod -std::size_t getSize(const T &) { - return sizeof(T); +std::ostream &write(std::ostream &OS, const std::vector &Data) { + for (const auto &E : Data) + write(OS, E); + return OS; } -template std::size_t encode(char *Dst, const std::vector &Data) { - std::size_t Ret = encode(Dst, Data.size()); - for (const auto &E : Data) { - Ret += encode(Dst + Ret, E); - } - return Ret; +/// Place an element into Indexed Vector +/// \returns the position for further element references +template std::size_t place(std::ostringstream &Self, T Elt) { + std::size_t Loc = Self.tellp(); + write(Self, Elt); + return Loc; } -template std::size_t getSize(const std::vector &Data) { - std::size_t Ret = sizeof(Data.size()); - for (const auto &E : Data) { - Ret += getSize(E); - } - return Ret; -} - -/// Wrapper around std::vector that we can easily get indexes of added element. -struct IndexVector { - template std::size_t add(T Elt) { - std::size_t EltSize = getSize(Elt); - Data.reserve(Data.size() + EltSize); - Data.resize(Data.size() + EltSize); - return encode(Data.data() + Data.size() - EltSize, Elt); - } - IndexVector() = default; - std::vector Data; -}; - -inline std::size_t encode(char *Dst, const IndexVector &Data) { - return encode(Dst, Data.Data); -} +/// String. Write the string and a null terminator. +std::ostream &write(std::ostream &OS, const std::string &Data); -inline std::size_t getSize(const IndexVector &Data) { - return getSize(Data.Data); -} +std::ostream &write(std::ostream &OS, const std::ostringstream &Data); +/// The string table. This is actually a vector of strings, but de-duplicate +/// while construction. class StringTable { public: - using Size = std::size_t; - Size add(std::string Str); - - friend std::size_t encode(char *Dst, const StringTable &Table); + friend std::size_t place(StringTable &Self, std::string Str); - friend std::size_t getSize(const StringTable &Table); - - void write(char *Dst) const; - - StringTable() = default; + friend std::ostream &write(std::ostream &OS, const StringTable &Data) { + for (const auto &E : Data.Strings) + write(OS, E); + return OS; + } private: - std::map Map; + std::map Map; std::vector Strings; uint32_t Length = 0; }; diff --git a/libnixt/lib/Serialize.cpp b/libnixt/lib/Serialize.cpp index 6baba35ea..f5bb1ab4f 100644 --- a/libnixt/lib/Serialize.cpp +++ b/libnixt/lib/Serialize.cpp @@ -10,19 +10,18 @@ namespace nixt::serialize { -std::size_t ASTWriter::count(const nix::Expr *E) { - struct ASTWriteVisitor : public RecursiveASTVisitor { - IndexVector &Exprs; - IndexVector &Pos; - StringTable &Strings; +ASTData collect(const nix::SymbolTable &STable, const nix::PosTable &PTable, + const nix::Expr *E) { + ASTData Data; + struct Visitor : public RecursiveASTVisitor { + ASTData &Data; const nix::SymbolTable &STable; const nix::PosTable &PTable; std::map ExprMap; - ASTWriteVisitor(IndexVector &Exprs, IndexVector &Pos, StringTable &Strings, - const nix::SymbolTable &STable, const nix::PosTable &PTable) - : Exprs(Exprs), Pos(Pos), Strings(Strings), STable(STable), - PTable(PTable) {} + Visitor(ASTData &Data, const nix::SymbolTable &STable, + const nix::PosTable &PTable) + : Data(Data), STable(STable), PTable(PTable) {} Position getPos(const nix::PosIdx P) { if (P == nix::noPos) { @@ -37,19 +36,20 @@ std::size_t ASTWriter::count(const nix::Expr *E) { Position::OK_SourcePath}; Ret.Kind = KindTable[Pos.origin.index()]; if (Ret.Kind == Position::OK_SourcePath) - Ret.Source = - Strings.add(std::get(Pos.origin).to_string()); + Ret.Source = place(Data.Strings, + std::get(Pos.origin).to_string()); return Ret; } bool visitExprAttrs(const nix::ExprAttrs *E) { ExprAttrSet Expr; - Expr.Pos = Pos.add(getPos(E->pos)); + Expr.Pos = place(Data.Pos, getPos(E->pos)); for (const auto &[K, V] : E->attrs) { - Expr.Attrs.emplace_back(AttrDefEntry{ - Strings.add(STable[K]), - {V.inherited, ExprMap[V.e], Pos.add(getPos(V.pos)), V.displ}}); + Expr.Attrs.emplace_back( + AttrDefEntry{place(Data.Strings, (STable[K])), + {V.inherited, ExprMap[V.e], + place(Data.Pos, (getPos(V.pos))), V.displ}}); } - ExprMap[E] = Exprs.add(Expr); + ExprMap[E] = place(Data.Exprs, Expr); return true; } bool visitExprVar(const nix::ExprVar *E) { @@ -57,42 +57,26 @@ std::size_t ASTWriter::count(const nix::Expr *E) { Expr.Kind = ExprKind::EK_Var; Expr.Body.Var.Displ = E->displ; Expr.Body.Var.FromWidth = E->fromWith; - Expr.Body.Var.Name = Strings.add(STable[E->name]); - Expr.Body.Var.Pos = Pos.add(getPos(E->pos)); - ExprMap[E] = Exprs.add(Expr); + Expr.Body.Var.Name = place(Data.Strings, STable[E->name]); + Expr.Body.Var.Pos = place(Data.Pos, getPos(E->pos)); + ExprMap[E] = place(Data.Exprs, Expr); return true; } - } Writer(Exprs, Pos, Strings, STable, PTable); + } Writer(Data, STable, PTable); Writer.traverseExpr(E); - - return sizeof(ASTHeader) + getSize(Exprs) + getSize(Pos) + getSize(Strings); + return Data; } -void ASTWriter::write(char *Dst) { +std::ostream &write(std::ostream &OS, const ASTData &Data) { ASTHeader Header; + strcpy(Header.Magic, "\x7fNixAST"); Header.Version = 1; - Header.ExprTableOffset = sizeof(ASTHeader); - Header.PosTableOffset = Header.ExprTableOffset + getSize(Exprs); - Header.StringTableOffset = Header.PosTableOffset + getSize(Pos); - memcpy(Dst, &Header, sizeof(ASTHeader)); - Dst += sizeof(ASTHeader); - Dst += encode(Dst, Exprs); - Dst += encode(Dst, Pos); - Dst += encode(Dst, Strings); -} - -std::size_t encode(char *Dst, const ExprAttrSet &E) { - std::size_t Ret = encode(Dst, E.Kind); - Ret += encode(Dst + Ret, E.Pos); - Ret += encode(Dst + Ret, E.Attrs); - Ret += encode(Dst + Ret, E.DynamicAttrs); - return Ret; -} - -std::size_t getSize(const ExprAttrSet &E) { - return sizeof(E.Kind) + getSize(E.Pos) + getSize(E.Attrs) + - getSize(E.DynamicAttrs); + write(OS, Header); + write(OS, Data.Exprs); + write(OS, Data.Pos); + write(OS, Data.Strings); + return OS; } } // namespace nixt::serialize diff --git a/libnixt/lib/SerializeSupport.cpp b/libnixt/lib/SerializeSupport.cpp index c4cf6c3eb..889b81974 100644 --- a/libnixt/lib/SerializeSupport.cpp +++ b/libnixt/lib/SerializeSupport.cpp @@ -2,24 +2,24 @@ namespace nixt::serialize { -StringTable::Size StringTable::add(std::string Str) { - if (Map.contains(Str)) { - return Map[std::move(Str)]; - } - Length += Str.size() + 1; // size of string + null terminator - Size Ret = Map[Str] = Length; - Strings.push_back(std::move(Str)); - return Ret; +std::ostream &write(std::ostream &OS, const std::string &Data) { + OS.write(Data.c_str(), Data.size() + 1); + return OS; } -std::size_t encode(char *Dst, const StringTable &Table) { - for (const auto &S : Table.Strings) { - memcpy(Dst, S.c_str(), S.size() + 1); - Dst = static_cast(Dst) + S.size() + 1; - } - return Table.Length; +std::ostream &write(std::ostream &OS, const std::ostringstream &Data) { + write(OS, Data.str()); + return OS; } -std::size_t getSize(const StringTable &Table) { return Table.Length; } +std::size_t place(StringTable &Self, std::string Str) { + if (Self.Map.contains(Str)) { + return Self.Map[std::move(Str)]; + } + Self.Length += Str.size() + 1; // size of string + null terminator + std::size_t Ret = Self.Map[Str] = Self.Length; + Self.Strings.push_back(std::move(Str)); + return Ret; +} } // namespace nixt::serialize