From 7c6d2b44a55ad077fad6b1d7d3a199bd691fadaa Mon Sep 17 00:00:00 2001 From: "Emil J. Tywoniak" Date: Wed, 11 Dec 2024 15:35:43 +0100 Subject: [PATCH 01/14] io: refactor string and file work into new unit --- Makefile | 5 +- backends/btor/btor.cc | 1 + backends/smt2/smt2.cc | 1 + frontends/ast/simplify.cc | 14 +- frontends/verific/verific.cc | 1 + kernel/io.cc | 457 +++++++++++++++++++++++++++++++++++ kernel/io.h | 100 ++++++++ kernel/register.cc | 91 +------ kernel/utils.h | 14 ++ kernel/yosys.cc | 364 ---------------------------- kernel/yosys_common.h | 58 +---- passes/fsm/fsm_recode.cc | 1 + passes/opt/pmux2shiftx.cc | 1 + passes/opt/wreduce.cc | 1 + passes/pmgen/peepopt.cc | 1 + passes/pmgen/pmgen.py | 1 + passes/sat/assertpmux.cc | 1 + 17 files changed, 592 insertions(+), 520 deletions(-) create mode 100644 kernel/io.cc create mode 100644 kernel/io.h diff --git a/Makefile b/Makefile index 0614235ca0b..1506d08b95a 100644 --- a/Makefile +++ b/Makefile @@ -97,7 +97,7 @@ YOSYS_SRC := $(dir $(firstword $(MAKEFILE_LIST))) VPATH := $(YOSYS_SRC) CXXSTD ?= c++17 -CXXFLAGS := $(CXXFLAGS) -Wall -Wextra -ggdb -I. -I"$(YOSYS_SRC)" -MD -MP -D_YOSYS_ -fPIC -I$(PREFIX)/include +CXXFLAGS := $(CXXFLAGS) -Wall -Wextra -Werror=implicit-function-declaration -ggdb -I. -I"$(YOSYS_SRC)" -MD -MP -D_YOSYS_ -fPIC -I$(PREFIX)/include LIBS := $(LIBS) -lstdc++ -lm PLUGIN_LINKFLAGS := PLUGIN_LIBS := @@ -579,6 +579,7 @@ ifeq ($(ENABLE_ZLIB),1) $(eval $(call add_include_file,kernel/fstdata.h)) endif $(eval $(call add_include_file,kernel/hashlib.h)) +$(eval $(call add_include_file,kernel/io.h)) $(eval $(call add_include_file,kernel/json.h)) $(eval $(call add_include_file,kernel/log.h)) $(eval $(call add_include_file,kernel/macc.h)) @@ -609,7 +610,7 @@ $(eval $(call add_include_file,frontends/ast/ast_binding.h)) $(eval $(call add_include_file,frontends/blif/blifparse.h)) $(eval $(call add_include_file,backends/rtlil/rtlil_backend.h)) -OBJS += kernel/driver.o kernel/register.o kernel/rtlil.o kernel/log.o kernel/calc.o kernel/yosys.o +OBJS += kernel/driver.o kernel/register.o kernel/rtlil.o kernel/log.o kernel/calc.o kernel/yosys.o kernel/io.o OBJS += kernel/binding.o kernel/tclapi.o OBJS += kernel/cellaigs.o kernel/celledges.o kernel/cost.o kernel/satgen.o kernel/scopeinfo.o kernel/qcsat.o kernel/mem.o kernel/ffmerge.o kernel/ff.o kernel/yw.o kernel/json.o kernel/fmt.o kernel/sexpr.o OBJS += kernel/drivertools.o kernel/functional.o diff --git a/backends/btor/btor.cc b/backends/btor/btor.cc index 43d1896797e..fa3cc728e60 100644 --- a/backends/btor/btor.cc +++ b/backends/btor/btor.cc @@ -30,6 +30,7 @@ #include "kernel/mem.h" #include "kernel/json.h" #include "kernel/yw.h" +#include "kernel/utils.h" #include USING_YOSYS_NAMESPACE diff --git a/backends/smt2/smt2.cc b/backends/smt2/smt2.cc index a6e1965ba5c..02784b97502 100644 --- a/backends/smt2/smt2.cc +++ b/backends/smt2/smt2.cc @@ -24,6 +24,7 @@ #include "kernel/log.h" #include "kernel/mem.h" #include "libs/json11/json11.hpp" +#include "kernel/utils.h" #include USING_YOSYS_NAMESPACE diff --git a/frontends/ast/simplify.cc b/frontends/ast/simplify.cc index b00cde28e2c..475f27cdfbf 100644 --- a/frontends/ast/simplify.cc +++ b/frontends/ast/simplify.cc @@ -2777,13 +2777,13 @@ bool AstNode::simplify(bool const_fold, int stage, int width_hint, bool sign_hin mux_input = new AstNode(AST_BIT_NOT, mux_input); } AstNode *node = new AstNode(AST_TERNARY, children_list.at(2)); - if (str == "bufif0") { - node->children.push_back(AstNode::mkconst_bits(z_const, false)); - node->children.push_back(mux_input); - } else { - node->children.push_back(mux_input); - node->children.push_back(AstNode::mkconst_bits(z_const, false)); - } + // if (str == "bufif0") { + // node->children.push_back(AstNode::mkconst_bits(z_const, false)); + // node->children.push_back(mux_input); + // } else { + // node->children.push_back(mux_input); + // node->children.push_back(AstNode::mkconst_bits(z_const, false)); + // } str.clear(); type = AST_ASSIGN; diff --git a/frontends/verific/verific.cc b/frontends/verific/verific.cc index 8f1b07b100e..86e798ad157 100644 --- a/frontends/verific/verific.cc +++ b/frontends/verific/verific.cc @@ -21,6 +21,7 @@ #include "kernel/sigtools.h" #include "kernel/celltypes.h" #include "kernel/log.h" +#include "kernel/utils.h" #include "libs/sha1/sha1.h" #include #include diff --git a/kernel/io.cc b/kernel/io.cc new file mode 100644 index 00000000000..a91827c697b --- /dev/null +++ b/kernel/io.cc @@ -0,0 +1,457 @@ +#include "kernel/yosys_common.h" +#include "kernel/log.h" +#include +#include +#include +#include +#include + +YOSYS_NAMESPACE_BEGIN + +// Set of utilities for handling files + +int readsome(std::istream &f, char *s, int n) +{ + int rc = int(f.readsome(s, n)); + + // f.readsome() sometimes returns 0 on a non-empty stream.. + if (rc == 0) { + int c = f.get(); + if (c != EOF) { + *s = c; + rc = 1; + } + } + + return rc; +} + +std::string next_token(std::string &text, const char *sep, bool long_strings) +{ + size_t pos_begin = text.find_first_not_of(sep); + + if (pos_begin == std::string::npos) + pos_begin = text.size(); + + if (long_strings && pos_begin != text.size() && text[pos_begin] == '"') { + std::string sep_string = sep; + for (size_t i = pos_begin+1; i < text.size(); i++) { + if (text[i] == '"' && (i+1 == text.size() || sep_string.find(text[i+1]) != std::string::npos)) { + std::string token = text.substr(pos_begin, i-pos_begin+1); + text = text.substr(i+1); + return token; + } + if (i+1 < text.size() && text[i] == '"' && text[i+1] == ';' && (i+2 == text.size() || sep_string.find(text[i+2]) != std::string::npos)) { + std::string token = text.substr(pos_begin, i-pos_begin+1); + text = text.substr(i+2); + return token + ";"; + } + } + } + + size_t pos_end = text.find_first_of(sep, pos_begin); + + if (pos_end == std::string::npos) + pos_end = text.size(); + + std::string token = text.substr(pos_begin, pos_end-pos_begin); + text = text.substr(pos_end); + return token; +} + +std::vector split_tokens(const std::string &text, const char *sep) +{ + std::vector tokens; + std::string current_token; + for (char c : text) { + if (strchr(sep, c)) { + if (!current_token.empty()) { + tokens.push_back(current_token); + current_token.clear(); + } + } else + current_token += c; + } + if (!current_token.empty()) { + tokens.push_back(current_token); + current_token.clear(); + } + return tokens; +} + +// this is very similar to fnmatch(). the exact rules used by this +// function are: +// +// ? matches any character except +// * matches any sequence of characters +// [...] matches any of the characters in the list +// [!..] matches any of the characters not in the list +// +// a backslash may be used to escape the next characters in the +// pattern. each special character can also simply match itself. +// +bool patmatch(const char *pattern, const char *string) +{ + if (*pattern == 0) + return *string == 0; + + if (*pattern == '\\') { + if (pattern[1] == string[0] && patmatch(pattern+2, string+1)) + return true; + } + + if (*pattern == '?') { + if (*string == 0) + return false; + return patmatch(pattern+1, string+1); + } + + if (*pattern == '*') { + while (*string) { + if (patmatch(pattern+1, string++)) + return true; + } + return pattern[1] == 0; + } + + if (*pattern == '[') { + bool found_match = false; + bool inverted_list = pattern[1] == '!'; + const char *p = pattern + (inverted_list ? 1 : 0); + + while (*++p) { + if (*p == ']') { + if (found_match != inverted_list && patmatch(p+1, string+1)) + return true; + break; + } + + if (*p == '\\') { + if (*++p == *string) + found_match = true; + } else + if (*p == *string) + found_match = true; + } + } + + if (*pattern == *string) + return patmatch(pattern+1, string+1); + + return false; +} + +std::string get_base_tmpdir() +{ + static std::string tmpdir; + + if (!tmpdir.empty()) { + return tmpdir; + } + +#if defined(_WIN32) +# ifdef __MINGW32__ + char longpath[MAX_PATH + 1]; + char shortpath[MAX_PATH + 1]; +# else + WCHAR longpath[MAX_PATH + 1]; + TCHAR shortpath[MAX_PATH + 1]; +# endif + if (!GetTempPath(MAX_PATH+1, longpath)) + log_error("GetTempPath() failed.\n"); + if (!GetShortPathName(longpath, shortpath, MAX_PATH + 1)) + log_error("GetShortPathName() failed.\n"); + for (int i = 0; shortpath[i]; i++) + tmpdir += char(shortpath[i]); +#else + char * var = std::getenv("TMPDIR"); + if (var && strlen(var)!=0) { + tmpdir.assign(var); + // We return the directory name without the trailing '/' + while (!tmpdir.empty() && (tmpdir.back() == '/')) { + tmpdir.pop_back(); + } + } else { + tmpdir.assign("/tmp"); + } +#endif + return tmpdir; +} + +std::string make_temp_file(std::string template_str) +{ + size_t pos = template_str.rfind("XXXXXX"); + log_assert(pos != std::string::npos); +#if defined(__wasm) + static size_t index = 0; + template_str.replace(pos, 6, stringf("%06zu", index++)); +#elif defined(_WIN32) +#ifndef YOSYS_WIN32_UNIX_DIR + std::replace(template_str.begin(), template_str.end(), '/', '\\'); +#endif + while (1) { + for (int i = 0; i < 6; i++) { + static std::string y = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; + static uint32_t x = 314159265 ^ uint32_t(time(NULL)); + x ^= x << 13, x ^= x >> 17, x ^= x << 5; + template_str[pos+i] = y[x % y.size()]; + } + if (_access(template_str.c_str(), 0) != 0) + break; + } +#else + int suffixlen = template_str.size() - pos - 6; + + char *p = strdup(template_str.c_str()); + close(mkstemps(p, suffixlen)); + template_str = p; + free(p); +#endif + + return template_str; +} + +std::string make_temp_dir(std::string template_str) +{ +#if defined(_WIN32) + template_str = make_temp_file(template_str); + mkdir(template_str.c_str()); + return template_str; +#elif defined(__wasm) + template_str = make_temp_file(template_str); + mkdir(template_str.c_str(), 0777); + return template_str; +#else +# ifndef NDEBUG + size_t pos = template_str.rfind("XXXXXX"); + log_assert(pos != std::string::npos); + + int suffixlen = template_str.size() - pos - 6; + log_assert(suffixlen == 0); +# endif + + char *p = strdup(template_str.c_str()); + char *res = mkdtemp(p); + log_assert(res != NULL); + template_str = p; + free(p); + + return template_str; +#endif +} + +bool check_directory_exists(const std::string& dirname) +{ +#if defined(_WIN32) + struct _stat info; + if (_stat(dirname.c_str(), &info) != 0) + { + return false; + } + return (info.st_mode & _S_IFDIR) != 0; +#else + struct stat info; + if (stat(dirname.c_str(), &info) != 0) + { + return false; + } + return (info.st_mode & S_IFDIR) != 0; +#endif +} + +#ifdef _WIN32 +bool check_file_exists(std::string filename, bool) +{ + return _access(filename.c_str(), 0) == 0; +} +#else +bool check_file_exists(std::string filename, bool is_exec) +{ + return access(filename.c_str(), is_exec ? X_OK : F_OK) == 0; +} +#endif + +bool is_absolute_path(std::string filename) +{ +#ifdef _WIN32 + return filename[0] == '/' || filename[0] == '\\' || (filename[0] != 0 && filename[1] == ':'); +#else + return filename[0] == '/'; +#endif +} + +void remove_directory(std::string dirname) +{ +#ifdef _WIN32 + run_command(stringf("rmdir /s /q \"%s\"", dirname.c_str())); +#else + struct stat stbuf; + struct dirent **namelist; + int n = scandir(dirname.c_str(), &namelist, nullptr, alphasort); + log_assert(n >= 0); + for (int i = 0; i < n; i++) { + if (strcmp(namelist[i]->d_name, ".") && strcmp(namelist[i]->d_name, "..")) { + std::string buffer = stringf("%s/%s", dirname.c_str(), namelist[i]->d_name); + if (!stat(buffer.c_str(), &stbuf) && S_ISREG(stbuf.st_mode)) { + remove(buffer.c_str()); + } else + remove_directory(buffer); + } + free(namelist[i]); + } + free(namelist); + rmdir(dirname.c_str()); +#endif +} + +bool create_directory(const std::string& dirname) +{ +#if defined(_WIN32) + int ret = _mkdir(dirname.c_str()); +#else + mode_t mode = 0755; + int ret = mkdir(dirname.c_str(), mode); +#endif + if (ret == 0) + return true; + + switch (errno) + { + case ENOENT: + // parent didn't exist, try to create it + { + std::string::size_type pos = dirname.find_last_of('/'); + if (pos == std::string::npos) +#if defined(_WIN32) + pos = dirname.find_last_of('\\'); + if (pos == std::string::npos) +#endif + return false; + if (!create_directory( dirname.substr(0, pos) )) + return false; + } + // now, try to create again +#if defined(_WIN32) + return 0 == _mkdir(dirname.c_str()); +#else + return 0 == mkdir(dirname.c_str(), mode); +#endif + + case EEXIST: + // done! + return check_directory_exists(dirname); + + default: + return false; + } +} + +std::string escape_filename_spaces(const std::string& filename) +{ + std::string out; + out.reserve(filename.size()); + for (auto c : filename) + { + if (c == ' ') + out += "\\ "; + else + out.push_back(c); + } + return out; +} + +#ifdef YOSYS_ENABLE_ZLIB + +PRIVATE_NAMESPACE_BEGIN + +static const size_t GZ_BUFFER_SIZE = 8192; +static void decompress_gzip(const std::string &filename, std::stringstream &out) +{ + char buffer[GZ_BUFFER_SIZE]; + int bytes_read; + gzFile gzf = gzopen(filename.c_str(), "rb"); + while(!gzeof(gzf)) { + bytes_read = gzread(gzf, reinterpret_cast(buffer), GZ_BUFFER_SIZE); + out.write(buffer, bytes_read); + } + gzclose(gzf); +} + +PRIVATE_NAMESPACE_END + +gzip_ostream::gzip_ostream() : std::ostream(nullptr) { + rdbuf(&outbuf); +} + +bool gzip_ostream::open(const std::string &filename) { + return outbuf.open(filename); +} + +gzip_ostream::gzip_streambuf::gzip_streambuf() { + setp(buffer, buffer + buffer_size - 1); +} + +bool gzip_ostream::gzip_streambuf::open(const std::string &filename) { + gzf = gzopen(filename.c_str(), "wb"); + return gzf != nullptr; +} + +int gzip_ostream::gzip_streambuf::sync() { + int num = pptr() - pbase(); + if (num > 0) { + if (gzwrite(gzf, reinterpret_cast(pbase()), num) != num) { + return -1; + } + pbump(-num); + } + return 0; +} + +gzip_ostream::gzip_streambuf::~gzip_streambuf() { + if (gzf) { + sync(); + gzclose(gzf); + } +} + +#endif // YOSYS_ENABLE_ZLIB + + +// Takes a successfully opened ifstream. If it's gzipped, returns an istream +// over a buffer of the file fully decompressed in memory. Otherwise, +// returns the original ifstream, rewound to the start. +std::istream* uncompressed(std::ifstream* f, const std::string filename) { + if (!f) + return nullptr; + // Check for gzip magic + unsigned char magic[3]; + int n = 0; + while (n < 3) + { + int c = f->get(); + if (c != EOF) { + magic[n] = (unsigned char) c; + } + n++; + } + if (n == 3 && magic[0] == 0x1f && magic[1] == 0x8b) { +#ifdef YOSYS_ENABLE_ZLIB + log("Found gzip magic in file `%s', decompressing using zlib.\n", filename.c_str()); + if (magic[2] != 8) + log_cmd_error("gzip file `%s' uses unsupported compression type %02x\n", + filename.c_str(), unsigned(magic[2])); + delete f; + std::stringstream *df = new std::stringstream(); + decompress_gzip(filename, *df); + return df; +#else + log_cmd_error("File `%s' is a gzip file, but Yosys is compiled without zlib.\n", filename.c_str()); +#endif // YOSYS_ENABLE_ZLIB + } else { + f->clear(); + f->seekg(0, std::ios::beg); + return f; + } +} + +YOSYS_NAMESPACE_END diff --git a/kernel/io.h b/kernel/io.h new file mode 100644 index 00000000000..99b2dc62ac0 --- /dev/null +++ b/kernel/io.h @@ -0,0 +1,100 @@ +#include "kernel/yosys_common.h" +#include + +#ifndef YOSYS_IO_H +#define YOSYS_IO_H + +#ifdef YOSYS_ENABLE_ZLIB +#include +#endif + +YOSYS_NAMESPACE_BEGIN + +inline std::string vstringf(const char *fmt, va_list ap) +{ + // For the common case of strings shorter than 128, save a heap + // allocation by using a stack allocated buffer. + const int kBufSize = 128; + char buf[kBufSize]; + buf[0] = '\0'; + va_list apc; + va_copy(apc, ap); + int n = vsnprintf(buf, kBufSize, fmt, apc); + va_end(apc); + if (n < kBufSize) + return std::string(buf); + + std::string string; + char *str = NULL; +#if defined(_WIN32) || defined(__CYGWIN__) + int sz = 2 * kBufSize, rc; + while (1) { + va_copy(apc, ap); + str = (char *)realloc(str, sz); + rc = vsnprintf(str, sz, fmt, apc); + va_end(apc); + if (rc >= 0 && rc < sz) + break; + sz *= 2; + } + if (str != NULL) { + string = str; + free(str); + } + return string; +#else + if (vasprintf(&str, fmt, ap) < 0) + str = NULL; + if (str != NULL) { + string = str; + free(str); + } + return string; +#endif +} + +std::string stringf(const char *fmt, ...) YS_ATTRIBUTE(format(printf, 1, 2)); + +inline std::string stringf(const char *fmt, ...) +{ + std::string string; + va_list ap; + + va_start(ap, fmt); + string = vstringf(fmt, ap); + va_end(ap); + + return string; +} + +#ifdef YOSYS_ENABLE_ZLIB +/* +An output stream that uses a stringbuf to buffer data internally, +using zlib to write gzip-compressed data every time the stream is flushed. +*/ +class gzip_ostream : public std::ostream { +public: + gzip_ostream(); + bool open(const std::string &filename); +private: + class gzip_streambuf : public std::stringbuf { + public: + gzip_streambuf(); + bool open(const std::string &filename); + virtual int sync() override; + virtual ~gzip_streambuf(); + private: + static const int buffer_size = 4096; // Size of the internal buffer + char buffer[buffer_size]; // Internal buffer for compressed data + gzFile gzf = nullptr; // Handle to the gzip file + }; + + gzip_streambuf outbuf; // The stream buffer instance +}; +#endif // YOSYS_ENABLE_ZLIB + +std::istream* uncompressed(std::ifstream* f, const std::string filename); + +YOSYS_NAMESPACE_END + +#endif // YOSYS_IO_H diff --git a/kernel/register.cc b/kernel/register.cc index d6e765ce416..00e0c29fda8 100644 --- a/kernel/register.cc +++ b/kernel/register.cc @@ -26,65 +26,6 @@ #include #include -#ifdef YOSYS_ENABLE_ZLIB -#include - -PRIVATE_NAMESPACE_BEGIN -#define GZ_BUFFER_SIZE 8192 -void decompress_gzip(const std::string &filename, std::stringstream &out) -{ - char buffer[GZ_BUFFER_SIZE]; - int bytes_read; - gzFile gzf = gzopen(filename.c_str(), "rb"); - while(!gzeof(gzf)) { - bytes_read = gzread(gzf, reinterpret_cast(buffer), GZ_BUFFER_SIZE); - out.write(buffer, bytes_read); - } - gzclose(gzf); -} - -/* -An output stream that uses a stringbuf to buffer data internally, -using zlib to write gzip-compressed data every time the stream is flushed. -*/ -class gzip_ostream : public std::ostream { -public: - gzip_ostream() : std::ostream(nullptr) - { - rdbuf(&outbuf); - } - bool open(const std::string &filename) - { - return outbuf.open(filename); - } -private: - class gzip_streambuf : public std::stringbuf { - public: - gzip_streambuf() { }; - bool open(const std::string &filename) - { - gzf = gzopen(filename.c_str(), "wb"); - return gzf != nullptr; - } - virtual int sync() override - { - gzwrite(gzf, reinterpret_cast(str().c_str()), unsigned(str().size())); - str(""); - return 0; - } - virtual ~gzip_streambuf() - { - sync(); - gzclose(gzf); - } - private: - gzFile gzf = nullptr; - } outbuf; -}; -PRIVATE_NAMESPACE_END - -#endif - YOSYS_NAMESPACE_BEGIN #define MAX_REG_COUNT 1000 @@ -534,37 +475,7 @@ void Frontend::extra_args(std::istream *&f, std::string &filename, std::vectorget(); - if (c != EOF) { - magic[n] = (unsigned char) c; - } - n++; - } - if (n == 3 && magic[0] == 0x1f && magic[1] == 0x8b) { - #ifdef YOSYS_ENABLE_ZLIB - log("Found gzip magic in file `%s', decompressing using zlib.\n", filename.c_str()); - if (magic[2] != 8) - log_cmd_error("gzip file `%s' uses unsupported compression type %02x\n", - filename.c_str(), unsigned(magic[2])); - delete ff; - std::stringstream *df = new std::stringstream(); - decompress_gzip(filename, *df); - f = df; - #else - log_cmd_error("File `%s' is a gzip file, but Yosys is compiled without zlib.\n", filename.c_str()); - #endif - } else { - ff->clear(); - ff->seekg(0, std::ios::beg); - } - } + f = uncompressed(ff, filename); } if (f == NULL) log_cmd_error("Can't open input file `%s' for reading: %s\n", filename.c_str(), strerror(errno)); diff --git a/kernel/utils.h b/kernel/utils.h index 07edad07479..6c9fe36a5f8 100644 --- a/kernel/utils.h +++ b/kernel/utils.h @@ -262,6 +262,20 @@ struct arrow_proxy { T* operator->() { return &v; } }; +inline int ceil_log2(int x) +{ +#if defined(__GNUC__) + return x > 1 ? (8*sizeof(int)) - __builtin_clz(x-1) : 0; +#else + if (x <= 0) + return 0; + for (int i = 0; i < 32; i++) + if (((x-1) >> i) == 0) + return i; + log_abort(); +#endif +} + YOSYS_NAMESPACE_END #endif diff --git a/kernel/yosys.cc b/kernel/yosys.cc index 7cf60f068f1..40c05075aba 100644 --- a/kernel/yosys.cc +++ b/kernel/yosys.cc @@ -147,151 +147,6 @@ void yosys_banner() log(" %s\n", yosys_version_str); } -int ceil_log2(int x) -{ -#if defined(__GNUC__) - return x > 1 ? (8*sizeof(int)) - __builtin_clz(x-1) : 0; -#else - if (x <= 0) - return 0; - for (int i = 0; i < 32; i++) - if (((x-1) >> i) == 0) - return i; - log_abort(); -#endif -} - -int readsome(std::istream &f, char *s, int n) -{ - int rc = int(f.readsome(s, n)); - - // f.readsome() sometimes returns 0 on a non-empty stream.. - if (rc == 0) { - int c = f.get(); - if (c != EOF) { - *s = c; - rc = 1; - } - } - - return rc; -} - -std::string next_token(std::string &text, const char *sep, bool long_strings) -{ - size_t pos_begin = text.find_first_not_of(sep); - - if (pos_begin == std::string::npos) - pos_begin = text.size(); - - if (long_strings && pos_begin != text.size() && text[pos_begin] == '"') { - string sep_string = sep; - for (size_t i = pos_begin+1; i < text.size(); i++) { - if (text[i] == '"' && (i+1 == text.size() || sep_string.find(text[i+1]) != std::string::npos)) { - std::string token = text.substr(pos_begin, i-pos_begin+1); - text = text.substr(i+1); - return token; - } - if (i+1 < text.size() && text[i] == '"' && text[i+1] == ';' && (i+2 == text.size() || sep_string.find(text[i+2]) != std::string::npos)) { - std::string token = text.substr(pos_begin, i-pos_begin+1); - text = text.substr(i+2); - return token + ";"; - } - } - } - - size_t pos_end = text.find_first_of(sep, pos_begin); - - if (pos_end == std::string::npos) - pos_end = text.size(); - - std::string token = text.substr(pos_begin, pos_end-pos_begin); - text = text.substr(pos_end); - return token; -} - -std::vector split_tokens(const std::string &text, const char *sep) -{ - std::vector tokens; - std::string current_token; - for (char c : text) { - if (strchr(sep, c)) { - if (!current_token.empty()) { - tokens.push_back(current_token); - current_token.clear(); - } - } else - current_token += c; - } - if (!current_token.empty()) { - tokens.push_back(current_token); - current_token.clear(); - } - return tokens; -} - -// this is very similar to fnmatch(). the exact rules used by this -// function are: -// -// ? matches any character except -// * matches any sequence of characters -// [...] matches any of the characters in the list -// [!..] matches any of the characters not in the list -// -// a backslash may be used to escape the next characters in the -// pattern. each special character can also simply match itself. -// -bool patmatch(const char *pattern, const char *string) -{ - if (*pattern == 0) - return *string == 0; - - if (*pattern == '\\') { - if (pattern[1] == string[0] && patmatch(pattern+2, string+1)) - return true; - } - - if (*pattern == '?') { - if (*string == 0) - return false; - return patmatch(pattern+1, string+1); - } - - if (*pattern == '*') { - while (*string) { - if (patmatch(pattern+1, string++)) - return true; - } - return pattern[1] == 0; - } - - if (*pattern == '[') { - bool found_match = false; - bool inverted_list = pattern[1] == '!'; - const char *p = pattern + (inverted_list ? 1 : 0); - - while (*++p) { - if (*p == ']') { - if (found_match != inverted_list && patmatch(p+1, string+1)) - return true; - break; - } - - if (*p == '\\') { - if (*++p == *string) - found_match = true; - } else - if (*p == *string) - found_match = true; - } - } - - if (*pattern == *string) - return patmatch(pattern+1, string+1); - - return false; -} - #if !defined(YOSYS_DISABLE_SPAWN) int run_command(const std::string &command, std::function process_line) { @@ -323,225 +178,6 @@ int run_command(const std::string &command, std::function> 17, x ^= x << 5; - template_str[pos+i] = y[x % y.size()]; - } - if (_access(template_str.c_str(), 0) != 0) - break; - } -#else - int suffixlen = GetSize(template_str) - pos - 6; - - char *p = strdup(template_str.c_str()); - close(mkstemps(p, suffixlen)); - template_str = p; - free(p); -#endif - - return template_str; -} - -std::string make_temp_dir(std::string template_str) -{ -#if defined(_WIN32) - template_str = make_temp_file(template_str); - mkdir(template_str.c_str()); - return template_str; -#elif defined(__wasm) - template_str = make_temp_file(template_str); - mkdir(template_str.c_str(), 0777); - return template_str; -#else -# ifndef NDEBUG - size_t pos = template_str.rfind("XXXXXX"); - log_assert(pos != std::string::npos); - - int suffixlen = GetSize(template_str) - pos - 6; - log_assert(suffixlen == 0); -# endif - - char *p = strdup(template_str.c_str()); - char *res = mkdtemp(p); - log_assert(res != NULL); - template_str = p; - free(p); - - return template_str; -#endif -} - -bool check_directory_exists(const std::string& dirname) -{ -#if defined(_WIN32) - struct _stat info; - if (_stat(dirname.c_str(), &info) != 0) - { - return false; - } - return (info.st_mode & _S_IFDIR) != 0; -#else - struct stat info; - if (stat(dirname.c_str(), &info) != 0) - { - return false; - } - return (info.st_mode & S_IFDIR) != 0; -#endif -} - -#ifdef _WIN32 -bool check_file_exists(std::string filename, bool) -{ - return _access(filename.c_str(), 0) == 0; -} -#else -bool check_file_exists(std::string filename, bool is_exec) -{ - return access(filename.c_str(), is_exec ? X_OK : F_OK) == 0; -} -#endif - -bool is_absolute_path(std::string filename) -{ -#ifdef _WIN32 - return filename[0] == '/' || filename[0] == '\\' || (filename[0] != 0 && filename[1] == ':'); -#else - return filename[0] == '/'; -#endif -} - -void remove_directory(std::string dirname) -{ -#ifdef _WIN32 - run_command(stringf("rmdir /s /q \"%s\"", dirname.c_str())); -#else - struct stat stbuf; - struct dirent **namelist; - int n = scandir(dirname.c_str(), &namelist, nullptr, alphasort); - log_assert(n >= 0); - for (int i = 0; i < n; i++) { - if (strcmp(namelist[i]->d_name, ".") && strcmp(namelist[i]->d_name, "..")) { - std::string buffer = stringf("%s/%s", dirname.c_str(), namelist[i]->d_name); - if (!stat(buffer.c_str(), &stbuf) && S_ISREG(stbuf.st_mode)) { - remove(buffer.c_str()); - } else - remove_directory(buffer); - } - free(namelist[i]); - } - free(namelist); - rmdir(dirname.c_str()); -#endif -} - -bool create_directory(const std::string& dirname) -{ -#if defined(_WIN32) - int ret = _mkdir(dirname.c_str()); -#else - mode_t mode = 0755; - int ret = mkdir(dirname.c_str(), mode); -#endif - if (ret == 0) - return true; - - switch (errno) - { - case ENOENT: - // parent didn't exist, try to create it - { - std::string::size_type pos = dirname.find_last_of('/'); - if (pos == std::string::npos) -#if defined(_WIN32) - pos = dirname.find_last_of('\\'); - if (pos == std::string::npos) -#endif - return false; - if (!create_directory( dirname.substr(0, pos) )) - return false; - } - // now, try to create again -#if defined(_WIN32) - return 0 == _mkdir(dirname.c_str()); -#else - return 0 == mkdir(dirname.c_str(), mode); -#endif - - case EEXIST: - // done! - return check_directory_exists(dirname); - - default: - return false; - } -} - -std::string escape_filename_spaces(const std::string& filename) -{ - std::string out; - out.reserve(filename.size()); - for (auto c : filename) - { - if (c == ' ') - out += "\\ "; - else - out.push_back(c); - } - return out; -} - bool already_setup = false; void yosys_setup() diff --git a/kernel/yosys_common.h b/kernel/yosys_common.h index 25a97a8ddb0..b8c2c8fc90c 100644 --- a/kernel/yosys_common.h +++ b/kernel/yosys_common.h @@ -128,6 +128,7 @@ # error "C++17 or later compatible compiler is required" #endif +#include "kernel/io.h" YOSYS_NAMESPACE_BEGIN @@ -245,63 +246,6 @@ inline void memhasher() { if (memhasher_active) memhasher_do(); } void yosys_banner(); int ceil_log2(int x) YS_ATTRIBUTE(const); -inline std::string vstringf(const char *fmt, va_list ap) -{ - // For the common case of strings shorter than 128, save a heap - // allocation by using a stack allocated buffer. - const int kBufSize = 128; - char buf[kBufSize]; - buf[0] = '\0'; - va_list apc; - va_copy(apc, ap); - int n = vsnprintf(buf, kBufSize, fmt, apc); - va_end(apc); - if (n < kBufSize) - return std::string(buf); - - std::string string; - char *str = NULL; -#if defined(_WIN32 )|| defined(__CYGWIN__) - int sz = 2 * kBufSize, rc; - while (1) { - va_copy(apc, ap); - str = (char*)realloc(str, sz); - rc = vsnprintf(str, sz, fmt, apc); - va_end(apc); - if (rc >= 0 && rc < sz) - break; - sz *= 2; - } - if (str != NULL) { - string = str; - free(str); - } - return string; -#else - if (vasprintf(&str, fmt, ap) < 0) - str = NULL; - if (str != NULL) { - string = str; - free(str); - } - return string; -#endif -} - -std::string stringf(const char *fmt, ...) YS_ATTRIBUTE(format(printf, 1, 2)); - -inline std::string stringf(const char *fmt, ...) -{ - std::string string; - va_list ap; - - va_start(ap, fmt); - string = vstringf(fmt, ap); - va_end(ap); - - return string; -} - int readsome(std::istream &f, char *s, int n); std::string next_token(std::string &text, const char *sep = " \t\r\n", bool long_strings = false); std::vector split_tokens(const std::string &text, const char *sep = " \t\r\n"); diff --git a/passes/fsm/fsm_recode.cc b/passes/fsm/fsm_recode.cc index cf6622d550f..bcdea9d873b 100644 --- a/passes/fsm/fsm_recode.cc +++ b/passes/fsm/fsm_recode.cc @@ -22,6 +22,7 @@ #include "kernel/sigtools.h" #include "kernel/consteval.h" #include "kernel/celltypes.h" +#include "kernel/utils.h" #include "fsmdata.h" #include #include diff --git a/passes/opt/pmux2shiftx.cc b/passes/opt/pmux2shiftx.cc index 9873f12a5e4..e512a5be179 100644 --- a/passes/opt/pmux2shiftx.cc +++ b/passes/opt/pmux2shiftx.cc @@ -20,6 +20,7 @@ #include "kernel/yosys.h" #include "kernel/sigtools.h" #include "kernel/ffinit.h" +#include "kernel/utils.h" USING_YOSYS_NAMESPACE PRIVATE_NAMESPACE_BEGIN diff --git a/passes/opt/wreduce.cc b/passes/opt/wreduce.cc index 02678d67680..a13d039a42c 100644 --- a/passes/opt/wreduce.cc +++ b/passes/opt/wreduce.cc @@ -21,6 +21,7 @@ #include "kernel/sigtools.h" #include "kernel/modtools.h" #include "kernel/ffinit.h" +#include "kernel/utils.h" USING_YOSYS_NAMESPACE diff --git a/passes/pmgen/peepopt.cc b/passes/pmgen/peepopt.cc index 5b678ee556d..664563e24d7 100644 --- a/passes/pmgen/peepopt.cc +++ b/passes/pmgen/peepopt.cc @@ -19,6 +19,7 @@ #include "kernel/yosys.h" #include "kernel/sigtools.h" +#include "kernel/utils.h" USING_YOSYS_NAMESPACE PRIVATE_NAMESPACE_BEGIN diff --git a/passes/pmgen/pmgen.py b/passes/pmgen/pmgen.py index 6e2fabeebde..97c6f560022 100644 --- a/passes/pmgen/pmgen.py +++ b/passes/pmgen/pmgen.py @@ -354,6 +354,7 @@ def process_pmgfile(f, filename): if genhdr: print("#include \"kernel/yosys.h\"", file=f) print("#include \"kernel/sigtools.h\"", file=f) + print("#include \"kernel/utils.h\"", file=f) print("", file=f) print("YOSYS_NAMESPACE_BEGIN", file=f) print("", file=f) diff --git a/passes/sat/assertpmux.cc b/passes/sat/assertpmux.cc index abdcb224096..991977ab921 100644 --- a/passes/sat/assertpmux.cc +++ b/passes/sat/assertpmux.cc @@ -19,6 +19,7 @@ #include "kernel/yosys.h" #include "kernel/sigtools.h" +#include "kernel/utils.h" USING_YOSYS_NAMESPACE PRIVATE_NAMESPACE_BEGIN From 6c55df34678b888b40c6b488803897c44eaef905 Mon Sep 17 00:00:00 2001 From: "Emil J. Tywoniak" Date: Fri, 20 Dec 2024 12:35:57 +0100 Subject: [PATCH 02/14] gzip: istream --- Makefile | 5 +- frontends/ast/simplify.cc | 14 ++-- kernel/gzip.cc | 132 ++++++++++++++++++++++++++++++++++++++ kernel/gzip.h | 79 +++++++++++++++++++++++ kernel/io.cc | 96 --------------------------- kernel/io.h | 36 +---------- kernel/register.cc | 1 + 7 files changed, 225 insertions(+), 138 deletions(-) create mode 100644 kernel/gzip.cc create mode 100644 kernel/gzip.h diff --git a/Makefile b/Makefile index 1506d08b95a..cf7465f90ea 100644 --- a/Makefile +++ b/Makefile @@ -97,7 +97,7 @@ YOSYS_SRC := $(dir $(firstword $(MAKEFILE_LIST))) VPATH := $(YOSYS_SRC) CXXSTD ?= c++17 -CXXFLAGS := $(CXXFLAGS) -Wall -Wextra -Werror=implicit-function-declaration -ggdb -I. -I"$(YOSYS_SRC)" -MD -MP -D_YOSYS_ -fPIC -I$(PREFIX)/include +CXXFLAGS := $(CXXFLAGS) -Wall -Wextra -ggdb -I. -I"$(YOSYS_SRC)" -MD -MP -D_YOSYS_ -fPIC -I$(PREFIX)/include LIBS := $(LIBS) -lstdc++ -lm PLUGIN_LINKFLAGS := PLUGIN_LIBS := @@ -578,6 +578,7 @@ $(eval $(call add_include_file,kernel/fmt.h)) ifeq ($(ENABLE_ZLIB),1) $(eval $(call add_include_file,kernel/fstdata.h)) endif +$(eval $(call add_include_file,kernel/gzip.h)) $(eval $(call add_include_file,kernel/hashlib.h)) $(eval $(call add_include_file,kernel/io.h)) $(eval $(call add_include_file,kernel/json.h)) @@ -610,7 +611,7 @@ $(eval $(call add_include_file,frontends/ast/ast_binding.h)) $(eval $(call add_include_file,frontends/blif/blifparse.h)) $(eval $(call add_include_file,backends/rtlil/rtlil_backend.h)) -OBJS += kernel/driver.o kernel/register.o kernel/rtlil.o kernel/log.o kernel/calc.o kernel/yosys.o kernel/io.o +OBJS += kernel/driver.o kernel/register.o kernel/rtlil.o kernel/log.o kernel/calc.o kernel/yosys.o kernel/io.o kernel/gzip.o OBJS += kernel/binding.o kernel/tclapi.o OBJS += kernel/cellaigs.o kernel/celledges.o kernel/cost.o kernel/satgen.o kernel/scopeinfo.o kernel/qcsat.o kernel/mem.o kernel/ffmerge.o kernel/ff.o kernel/yw.o kernel/json.o kernel/fmt.o kernel/sexpr.o OBJS += kernel/drivertools.o kernel/functional.o diff --git a/frontends/ast/simplify.cc b/frontends/ast/simplify.cc index 475f27cdfbf..b00cde28e2c 100644 --- a/frontends/ast/simplify.cc +++ b/frontends/ast/simplify.cc @@ -2777,13 +2777,13 @@ bool AstNode::simplify(bool const_fold, int stage, int width_hint, bool sign_hin mux_input = new AstNode(AST_BIT_NOT, mux_input); } AstNode *node = new AstNode(AST_TERNARY, children_list.at(2)); - // if (str == "bufif0") { - // node->children.push_back(AstNode::mkconst_bits(z_const, false)); - // node->children.push_back(mux_input); - // } else { - // node->children.push_back(mux_input); - // node->children.push_back(AstNode::mkconst_bits(z_const, false)); - // } + if (str == "bufif0") { + node->children.push_back(AstNode::mkconst_bits(z_const, false)); + node->children.push_back(mux_input); + } else { + node->children.push_back(mux_input); + node->children.push_back(AstNode::mkconst_bits(z_const, false)); + } str.clear(); type = AST_ASSIGN; diff --git a/kernel/gzip.cc b/kernel/gzip.cc new file mode 100644 index 00000000000..900e33bf49d --- /dev/null +++ b/kernel/gzip.cc @@ -0,0 +1,132 @@ +#include "kernel/yosys_common.h" +#include "kernel/log.h" +#include "kernel/gzip.h" +#include +#include +#include +#include +#include + +YOSYS_NAMESPACE_BEGIN + + +#ifdef YOSYS_ENABLE_ZLIB + +PRIVATE_NAMESPACE_BEGIN + +using namespace Zlib; + +PRIVATE_NAMESPACE_END + +gzip_ostream::obuf::obuf() { + setp(buffer, buffer + buffer_size - 1); +} + +bool gzip_ostream::obuf::open(const std::string &filename) { + gzf = gzopen(filename.c_str(), "wb"); + return gzf != nullptr; +} + +int gzip_ostream::obuf::sync() { + int num = pptr() - pbase(); + if (num > 0) { + if (gzwrite(gzf, reinterpret_cast(pbase()), num) != num) { + return -1; + } + pbump(-num); + } + return 0; +} + +gzip_ostream::obuf::~obuf() { + if (gzf) { + sync(); + gzclose(gzf); + } +} + +bool gzip_istream::ibuf::open(const std::string& filename) { + if (gzf) { + Zlib::gzclose(gzf); + } + gzf = Zlib::gzopen(filename.c_str(), "rb"); + if (!gzf) { + return false; + } + // Empty and point to start + setg(buffer, buffer, buffer); + return true; +} + +// Called when the buffer is empty and more input is needed +std::istream::int_type gzip_istream::ibuf::underflow() { + log_assert(gzf && "No gzfile opened\n"); + int bytes_read = Zlib::gzread(gzf, buffer, buffer_size); + if (bytes_read <= 0) { + if (Zlib::gzeof(gzf)) + return traits_type::eof(); + + int err; + const char* error_msg = Zlib::gzerror(gzf, &err); + if (err != Z_OK) + log_error("%s", error_msg); + else + log_error("Decompression logic failure: "\ + "read <=0 bytes but neither EOF nor error\n"); + } + + // Keep size and point to start + setg(buffer, buffer, buffer + bytes_read); + return traits_type::to_int_type(buffer[0]); +} + +gzip_istream::ibuf::~ibuf() { + if (gzf) { + int err = Zlib::gzclose(gzf); + if (err != Z_OK) { + // OK to overwrite rr it, it doesn't change + const char* error_msg = Zlib::gzerror(gzf, &err); + log_error("%s", error_msg); + } + } +} + +#endif // YOSYS_ENABLE_ZLIB + + +// Takes a successfully opened ifstream. If it's gzipped, returns an istream. Otherwise, +// returns the original ifstream, rewound to the start. +std::istream* uncompressed(std::ifstream* f, const std::string filename) { + if (!f) + return nullptr; + // Check for gzip magic + unsigned char magic[3]; + int n = 0; + while (n < 3) + { + int c = f->get(); + if (c != EOF) { + magic[n] = (unsigned char) c; + } + n++; + } + if (n == 3 && magic[0] == 0x1f && magic[1] == 0x8b) { +#ifdef YOSYS_ENABLE_ZLIB + log("Found gzip magic in file `%s', decompressing using zlib.\n", filename.c_str()); + if (magic[2] != 8) + log_cmd_error("gzip file `%s' uses unsupported compression type %02x\n", + filename.c_str(), unsigned(magic[2])); + delete f; + gzip_istream* s = new gzip_istream(); + return s->open(filename.c_str()) ? s : nullptr; +#else + log_cmd_error("File `%s' is a gzip file, but Yosys is compiled without zlib.\n", filename.c_str()); +#endif // YOSYS_ENABLE_ZLIB + } else { + f->clear(); + f->seekg(0, std::ios::beg); + return f; + } +} + +YOSYS_NAMESPACE_END diff --git a/kernel/gzip.h b/kernel/gzip.h new file mode 100644 index 00000000000..22f5093a0bf --- /dev/null +++ b/kernel/gzip.h @@ -0,0 +1,79 @@ +#include +#include +#include "kernel/yosys_common.h" + +#ifndef YOSYS_GZIP_H +#define YOSYS_GZIP_H + +YOSYS_NAMESPACE_BEGIN + +#ifdef YOSYS_ENABLE_ZLIB + +namespace Zlib { +#include +} + +/* +An output stream that uses a stringbuf to buffer data internally, +using zlib to write gzip-compressed data every time the stream is flushed. +*/ +class gzip_ostream : public std::ostream { +public: + gzip_ostream(): std::ostream(nullptr) { + rdbuf(&outbuf); + } + bool open(const std::string &filename) { + return outbuf.open(filename); + } +private: + class obuf : public std::stringbuf { + public: + obuf(); + bool open(const std::string &filename); + virtual int sync() override; + virtual ~obuf(); + private: + static const int buffer_size = 4096; + char buffer[buffer_size]; // Internal buffer for compressed data + Zlib::gzFile gzf = nullptr; // Handle to the gzip file + }; + + obuf outbuf; // The stream buffer instance +}; + +/* +An input stream that uses zlib to read gzip-compressed data from a file, +buffering the decompressed data internally using its own buffer. +*/ +class gzip_istream final : public std::istream { +public: + gzip_istream() : std::istream(&inbuf) {} + bool open(const std::string& filename) { + return inbuf.open(filename); + } +private: + class ibuf final : public std::streambuf { + public: + ibuf() : gzf(nullptr) {} + bool open(const std::string& filename); + virtual ~ibuf(); + + protected: + // Called when the buffer is empty and more input is needed + virtual int_type underflow() override; + private: + static const int buffer_size = 8192; + char buffer[buffer_size]; + Zlib::gzFile gzf; + }; + + ibuf inbuf; // The stream buffer instance +}; + +#endif // YOSYS_ENABLE_ZLIB + +std::istream* uncompressed(std::ifstream* f, const std::string filename); + +YOSYS_NAMESPACE_END + +#endif // YOSYS_GZIP_H diff --git a/kernel/io.cc b/kernel/io.cc index a91827c697b..0fdbd3c3791 100644 --- a/kernel/io.cc +++ b/kernel/io.cc @@ -3,8 +3,6 @@ #include #include #include -#include -#include YOSYS_NAMESPACE_BEGIN @@ -360,98 +358,4 @@ std::string escape_filename_spaces(const std::string& filename) return out; } -#ifdef YOSYS_ENABLE_ZLIB - -PRIVATE_NAMESPACE_BEGIN - -static const size_t GZ_BUFFER_SIZE = 8192; -static void decompress_gzip(const std::string &filename, std::stringstream &out) -{ - char buffer[GZ_BUFFER_SIZE]; - int bytes_read; - gzFile gzf = gzopen(filename.c_str(), "rb"); - while(!gzeof(gzf)) { - bytes_read = gzread(gzf, reinterpret_cast(buffer), GZ_BUFFER_SIZE); - out.write(buffer, bytes_read); - } - gzclose(gzf); -} - -PRIVATE_NAMESPACE_END - -gzip_ostream::gzip_ostream() : std::ostream(nullptr) { - rdbuf(&outbuf); -} - -bool gzip_ostream::open(const std::string &filename) { - return outbuf.open(filename); -} - -gzip_ostream::gzip_streambuf::gzip_streambuf() { - setp(buffer, buffer + buffer_size - 1); -} - -bool gzip_ostream::gzip_streambuf::open(const std::string &filename) { - gzf = gzopen(filename.c_str(), "wb"); - return gzf != nullptr; -} - -int gzip_ostream::gzip_streambuf::sync() { - int num = pptr() - pbase(); - if (num > 0) { - if (gzwrite(gzf, reinterpret_cast(pbase()), num) != num) { - return -1; - } - pbump(-num); - } - return 0; -} - -gzip_ostream::gzip_streambuf::~gzip_streambuf() { - if (gzf) { - sync(); - gzclose(gzf); - } -} - -#endif // YOSYS_ENABLE_ZLIB - - -// Takes a successfully opened ifstream. If it's gzipped, returns an istream -// over a buffer of the file fully decompressed in memory. Otherwise, -// returns the original ifstream, rewound to the start. -std::istream* uncompressed(std::ifstream* f, const std::string filename) { - if (!f) - return nullptr; - // Check for gzip magic - unsigned char magic[3]; - int n = 0; - while (n < 3) - { - int c = f->get(); - if (c != EOF) { - magic[n] = (unsigned char) c; - } - n++; - } - if (n == 3 && magic[0] == 0x1f && magic[1] == 0x8b) { -#ifdef YOSYS_ENABLE_ZLIB - log("Found gzip magic in file `%s', decompressing using zlib.\n", filename.c_str()); - if (magic[2] != 8) - log_cmd_error("gzip file `%s' uses unsupported compression type %02x\n", - filename.c_str(), unsigned(magic[2])); - delete f; - std::stringstream *df = new std::stringstream(); - decompress_gzip(filename, *df); - return df; -#else - log_cmd_error("File `%s' is a gzip file, but Yosys is compiled without zlib.\n", filename.c_str()); -#endif // YOSYS_ENABLE_ZLIB - } else { - f->clear(); - f->seekg(0, std::ios::beg); - return f; - } -} - YOSYS_NAMESPACE_END diff --git a/kernel/io.h b/kernel/io.h index 99b2dc62ac0..1eb3a0fc100 100644 --- a/kernel/io.h +++ b/kernel/io.h @@ -1,13 +1,11 @@ -#include "kernel/yosys_common.h" #include +#include +#include +#include "kernel/yosys_common.h" #ifndef YOSYS_IO_H #define YOSYS_IO_H -#ifdef YOSYS_ENABLE_ZLIB -#include -#endif - YOSYS_NAMESPACE_BEGIN inline std::string vstringf(const char *fmt, va_list ap) @@ -67,34 +65,6 @@ inline std::string stringf(const char *fmt, ...) return string; } -#ifdef YOSYS_ENABLE_ZLIB -/* -An output stream that uses a stringbuf to buffer data internally, -using zlib to write gzip-compressed data every time the stream is flushed. -*/ -class gzip_ostream : public std::ostream { -public: - gzip_ostream(); - bool open(const std::string &filename); -private: - class gzip_streambuf : public std::stringbuf { - public: - gzip_streambuf(); - bool open(const std::string &filename); - virtual int sync() override; - virtual ~gzip_streambuf(); - private: - static const int buffer_size = 4096; // Size of the internal buffer - char buffer[buffer_size]; // Internal buffer for compressed data - gzFile gzf = nullptr; // Handle to the gzip file - }; - - gzip_streambuf outbuf; // The stream buffer instance -}; -#endif // YOSYS_ENABLE_ZLIB - -std::istream* uncompressed(std::ifstream* f, const std::string filename); - YOSYS_NAMESPACE_END #endif // YOSYS_IO_H diff --git a/kernel/register.cc b/kernel/register.cc index 00e0c29fda8..a2e7cb5772a 100644 --- a/kernel/register.cc +++ b/kernel/register.cc @@ -20,6 +20,7 @@ #include "kernel/yosys.h" #include "kernel/satgen.h" #include "kernel/json.h" +#include "kernel/gzip.h" #include #include From 094ab8c017894c82550eb7ff3dd7fc365faaff7a Mon Sep 17 00:00:00 2001 From: "Emil J. Tywoniak" Date: Fri, 3 Jan 2025 17:51:38 +0100 Subject: [PATCH 03/14] dfflibmap: allow gzipped liberty files --- passes/techmap/dfflibmap.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/passes/techmap/dfflibmap.cc b/passes/techmap/dfflibmap.cc index b475cdd2d1a..fedcba1e1ce 100644 --- a/passes/techmap/dfflibmap.cc +++ b/passes/techmap/dfflibmap.cc @@ -19,6 +19,7 @@ #include "kernel/yosys.h" #include "kernel/sigtools.h" +#include "kernel/gzip.h" #include "libparse.h" #include #include @@ -632,11 +633,12 @@ struct DfflibmapPass : public Pass { for (auto path : liberty_files) { std::ifstream f; f.open(path.c_str()); - if (f.fail()) + std::istream* ff = uncompressed(&f, path); + if (ff->fail()) log_cmd_error("Can't open liberty file `%s': %s\n", path.c_str(), strerror(errno)); - LibertyParser p(f); + LibertyParser p(*ff); merged.merge(p); - f.close(); + delete ff; } find_cell(merged.cells, ID($_DFF_N_), false, false, false, false, false, false, dont_use_cells); From 0b96aa192317fd9d6a2d0090c1579525f3859d57 Mon Sep 17 00:00:00 2001 From: "Emil J. Tywoniak" Date: Fri, 3 Jan 2025 18:40:40 +0100 Subject: [PATCH 04/14] gzip: simplify uncompressed interface --- kernel/gzip.cc | 21 ++++++++++++--------- kernel/gzip.h | 2 +- kernel/register.cc | 8 +------- passes/techmap/dfflibmap.cc | 10 ++++------ 4 files changed, 18 insertions(+), 23 deletions(-) diff --git a/kernel/gzip.cc b/kernel/gzip.cc index 900e33bf49d..0da53ff68c6 100644 --- a/kernel/gzip.cc +++ b/kernel/gzip.cc @@ -96,15 +96,17 @@ gzip_istream::ibuf::~ibuf() { // Takes a successfully opened ifstream. If it's gzipped, returns an istream. Otherwise, // returns the original ifstream, rewound to the start. -std::istream* uncompressed(std::ifstream* f, const std::string filename) { - if (!f) - return nullptr; +std::istream& uncompressed(const std::string filename, std::ios_base::openmode mode) { + std::ifstream& f = *new std::ifstream(); + f.open(filename, mode); + if (f.fail()) + return f; // Check for gzip magic unsigned char magic[3]; int n = 0; while (n < 3) { - int c = f->get(); + int c = f.get(); if (c != EOF) { magic[n] = (unsigned char) c; } @@ -116,15 +118,16 @@ std::istream* uncompressed(std::ifstream* f, const std::string filename) { if (magic[2] != 8) log_cmd_error("gzip file `%s' uses unsupported compression type %02x\n", filename.c_str(), unsigned(magic[2])); - delete f; - gzip_istream* s = new gzip_istream(); - return s->open(filename.c_str()) ? s : nullptr; + gzip_istream& s = *new gzip_istream(); + delete &f; + s.open(filename.c_str()); + return s; #else log_cmd_error("File `%s' is a gzip file, but Yosys is compiled without zlib.\n", filename.c_str()); #endif // YOSYS_ENABLE_ZLIB } else { - f->clear(); - f->seekg(0, std::ios::beg); + f.clear(); + f.seekg(0, std::ios::beg); return f; } } diff --git a/kernel/gzip.h b/kernel/gzip.h index 22f5093a0bf..0201d2340bf 100644 --- a/kernel/gzip.h +++ b/kernel/gzip.h @@ -72,7 +72,7 @@ class gzip_istream final : public std::istream { #endif // YOSYS_ENABLE_ZLIB -std::istream* uncompressed(std::ifstream* f, const std::string filename); +std::istream& uncompressed(const std::string filename, std::ios_base::openmode mode = std::ios_base::in); YOSYS_NAMESPACE_END diff --git a/kernel/register.cc b/kernel/register.cc index a2e7cb5772a..040cc62076d 100644 --- a/kernel/register.cc +++ b/kernel/register.cc @@ -469,14 +469,8 @@ void Frontend::extra_args(std::istream *&f, std::string &filename, std::vectoropen(filename.c_str(), bin_input ? std::ifstream::binary : std::ifstream::in); yosys_input_files.insert(filename); - if (ff->fail()) { - delete ff; - ff = nullptr; - } - f = uncompressed(ff, filename); + f = &uncompressed(filename, bin_input ? std::ifstream::binary : std::ifstream::in); } if (f == NULL) log_cmd_error("Can't open input file `%s' for reading: %s\n", filename.c_str(), strerror(errno)); diff --git a/passes/techmap/dfflibmap.cc b/passes/techmap/dfflibmap.cc index fedcba1e1ce..df2bfa88f74 100644 --- a/passes/techmap/dfflibmap.cc +++ b/passes/techmap/dfflibmap.cc @@ -631,14 +631,12 @@ struct DfflibmapPass : public Pass { LibertyMergedCells merged; for (auto path : liberty_files) { - std::ifstream f; - f.open(path.c_str()); - std::istream* ff = uncompressed(&f, path); - if (ff->fail()) + std::istream& f = uncompressed(path); + if (f.fail()) log_cmd_error("Can't open liberty file `%s': %s\n", path.c_str(), strerror(errno)); - LibertyParser p(*ff); + LibertyParser p(f); merged.merge(p); - delete ff; + delete &f; } find_cell(merged.cells, ID($_DFF_N_), false, false, false, false, false, false, dont_use_cells); From 63104548908235867e371312c719af26e98df790 Mon Sep 17 00:00:00 2001 From: "Emil J. Tywoniak" Date: Sat, 4 Jan 2025 12:18:02 +0100 Subject: [PATCH 05/14] io: remove unused unistd.h to fix windows build --- kernel/io.h | 1 - 1 file changed, 1 deletion(-) diff --git a/kernel/io.h b/kernel/io.h index 1eb3a0fc100..b21d4d9c809 100644 --- a/kernel/io.h +++ b/kernel/io.h @@ -1,5 +1,4 @@ #include -#include #include #include "kernel/yosys_common.h" From d75b6bb3d71afb04071c8d79c38b6e5d168a1dab Mon Sep 17 00:00:00 2001 From: "Emil J. Tywoniak" Date: Sat, 4 Jan 2025 12:18:19 +0100 Subject: [PATCH 06/14] dfflibmap: allow gzipped liberty files --- passes/techmap/clockgate.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/passes/techmap/clockgate.cc b/passes/techmap/clockgate.cc index bdfb94170d8..03c43699647 100644 --- a/passes/techmap/clockgate.cc +++ b/passes/techmap/clockgate.cc @@ -1,5 +1,6 @@ #include "kernel/yosys.h" #include "kernel/ff.h" +#include "kernel/gzip.h" #include "libparse.h" #include @@ -308,13 +309,12 @@ struct ClockgatePass : public Pass { if (!liberty_files.empty()) { LibertyMergedCells merged; for (auto path : liberty_files) { - std::ifstream f; - f.open(path.c_str()); + std::istream& f = uncompressed(path); if (f.fail()) log_cmd_error("Can't open liberty file `%s': %s\n", path.c_str(), strerror(errno)); LibertyParser p(f); merged.merge(p); - f.close(); + delete &f; } std::tie(pos_icg_desc, neg_icg_desc) = find_icgs(merged.cells, dont_use_cells); From b3610c4ab3e5b7ff42f330c1a9b1f359b3876030 Mon Sep 17 00:00:00 2001 From: "Emil J. Tywoniak" Date: Sun, 5 Jan 2025 23:55:38 +0100 Subject: [PATCH 07/14] io: smooth out non-POSIX function usage across platforms --- kernel/gzip.cc | 8 +++++++- kernel/gzip.h | 1 - kernel/io.cc | 8 +++++++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/kernel/gzip.cc b/kernel/gzip.cc index 0da53ff68c6..71a498bb002 100644 --- a/kernel/gzip.cc +++ b/kernel/gzip.cc @@ -2,11 +2,17 @@ #include "kernel/log.h" #include "kernel/gzip.h" #include -#include #include #include #include +#if !defined(WIN32) +#include +#include +#else +#include +#endif + YOSYS_NAMESPACE_BEGIN diff --git a/kernel/gzip.h b/kernel/gzip.h index 0201d2340bf..57eac73c80a 100644 --- a/kernel/gzip.h +++ b/kernel/gzip.h @@ -1,5 +1,4 @@ #include -#include #include "kernel/yosys_common.h" #ifndef YOSYS_GZIP_H diff --git a/kernel/io.cc b/kernel/io.cc index 0fdbd3c3791..7bb29499d67 100644 --- a/kernel/io.cc +++ b/kernel/io.cc @@ -1,9 +1,15 @@ #include "kernel/yosys_common.h" #include "kernel/log.h" #include -#include #include +#if !defined(WIN32) +#include +#include +#else +#include +#endif + YOSYS_NAMESPACE_BEGIN // Set of utilities for handling files From 00071c1cd124147acd2d3fb4a8f3a3b3b424afa5 Mon Sep 17 00:00:00 2001 From: "Emil J. Tywoniak" Date: Tue, 7 Jan 2025 11:19:30 +0100 Subject: [PATCH 08/14] gzip: minor refactor --- kernel/gzip.cc | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/kernel/gzip.cc b/kernel/gzip.cc index 71a498bb002..b1f61ea2671 100644 --- a/kernel/gzip.cc +++ b/kernel/gzip.cc @@ -15,28 +15,21 @@ YOSYS_NAMESPACE_BEGIN - #ifdef YOSYS_ENABLE_ZLIB -PRIVATE_NAMESPACE_BEGIN - -using namespace Zlib; - -PRIVATE_NAMESPACE_END - gzip_ostream::obuf::obuf() { setp(buffer, buffer + buffer_size - 1); } bool gzip_ostream::obuf::open(const std::string &filename) { - gzf = gzopen(filename.c_str(), "wb"); + gzf = Zlib::gzopen(filename.c_str(), "wb"); return gzf != nullptr; } int gzip_ostream::obuf::sync() { int num = pptr() - pbase(); if (num > 0) { - if (gzwrite(gzf, reinterpret_cast(pbase()), num) != num) { + if (Zlib::gzwrite(gzf, reinterpret_cast(pbase()), num) != num) { return -1; } pbump(-num); @@ -47,7 +40,7 @@ int gzip_ostream::obuf::sync() { gzip_ostream::obuf::~obuf() { if (gzf) { sync(); - gzclose(gzf); + Zlib::gzclose(gzf); } } From fa8642be020faab5313eed55d8822da9011a483b Mon Sep 17 00:00:00 2001 From: "Emil J. Tywoniak" Date: Thu, 9 Jan 2025 13:38:51 +0100 Subject: [PATCH 09/14] gzip: uphold rules for basic_streambuf::underflow overrides --- kernel/gzip.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/kernel/gzip.cc b/kernel/gzip.cc index b1f61ea2671..f31d535dfd0 100644 --- a/kernel/gzip.cc +++ b/kernel/gzip.cc @@ -62,8 +62,13 @@ std::istream::int_type gzip_istream::ibuf::underflow() { log_assert(gzf && "No gzfile opened\n"); int bytes_read = Zlib::gzread(gzf, buffer, buffer_size); if (bytes_read <= 0) { - if (Zlib::gzeof(gzf)) + if (Zlib::gzeof(gzf)) { + // "On failure, the function ensures that either + // gptr() == nullptr or gptr() == egptr." + // Let's set gptr to egptr + setg(eback(), egptr(), egptr()); return traits_type::eof(); + } int err; const char* error_msg = Zlib::gzerror(gzf, &err); From b85dda9cb9f9f125a9a8668a2edbbdfdfdda3359 Mon Sep 17 00:00:00 2001 From: "Emil J. Tywoniak" Date: Thu, 9 Jan 2025 16:26:03 +0100 Subject: [PATCH 10/14] gzip: back to pointers --- kernel/gzip.cc | 20 ++++++++++---------- kernel/gzip.h | 2 +- kernel/register.cc | 2 +- passes/techmap/clockgate.cc | 8 ++++---- passes/techmap/dfflibmap.cc | 8 ++++---- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/kernel/gzip.cc b/kernel/gzip.cc index f31d535dfd0..d44b0351759 100644 --- a/kernel/gzip.cc +++ b/kernel/gzip.cc @@ -100,17 +100,17 @@ gzip_istream::ibuf::~ibuf() { // Takes a successfully opened ifstream. If it's gzipped, returns an istream. Otherwise, // returns the original ifstream, rewound to the start. -std::istream& uncompressed(const std::string filename, std::ios_base::openmode mode) { - std::ifstream& f = *new std::ifstream(); - f.open(filename, mode); - if (f.fail()) +std::istream* uncompressed(const std::string filename, std::ios_base::openmode mode) { + std::ifstream* f = new std::ifstream(); + f->open(filename, mode); + if (f->fail()) return f; // Check for gzip magic unsigned char magic[3]; int n = 0; while (n < 3) { - int c = f.get(); + int c = f->get(); if (c != EOF) { magic[n] = (unsigned char) c; } @@ -122,16 +122,16 @@ std::istream& uncompressed(const std::string filename, std::ios_base::openmode m if (magic[2] != 8) log_cmd_error("gzip file `%s' uses unsupported compression type %02x\n", filename.c_str(), unsigned(magic[2])); - gzip_istream& s = *new gzip_istream(); - delete &f; - s.open(filename.c_str()); + gzip_istream* s = new gzip_istream(); + delete f; + s->open(filename.c_str()); return s; #else log_cmd_error("File `%s' is a gzip file, but Yosys is compiled without zlib.\n", filename.c_str()); #endif // YOSYS_ENABLE_ZLIB } else { - f.clear(); - f.seekg(0, std::ios::beg); + f->clear(); + f->seekg(0, std::ios::beg); return f; } } diff --git a/kernel/gzip.h b/kernel/gzip.h index 57eac73c80a..aac5340de8e 100644 --- a/kernel/gzip.h +++ b/kernel/gzip.h @@ -71,7 +71,7 @@ class gzip_istream final : public std::istream { #endif // YOSYS_ENABLE_ZLIB -std::istream& uncompressed(const std::string filename, std::ios_base::openmode mode = std::ios_base::in); +std::istream* uncompressed(const std::string filename, std::ios_base::openmode mode = std::ios_base::in); YOSYS_NAMESPACE_END diff --git a/kernel/register.cc b/kernel/register.cc index 040cc62076d..7add24601df 100644 --- a/kernel/register.cc +++ b/kernel/register.cc @@ -470,7 +470,7 @@ void Frontend::extra_args(std::istream *&f, std::string &filename, std::vectorfail()) log_cmd_error("Can't open liberty file `%s': %s\n", path.c_str(), strerror(errno)); - LibertyParser p(f); + LibertyParser p(*f); merged.merge(p); - delete &f; + delete f; } std::tie(pos_icg_desc, neg_icg_desc) = find_icgs(merged.cells, dont_use_cells); diff --git a/passes/techmap/dfflibmap.cc b/passes/techmap/dfflibmap.cc index df2bfa88f74..fe4f8aadaf6 100644 --- a/passes/techmap/dfflibmap.cc +++ b/passes/techmap/dfflibmap.cc @@ -631,12 +631,12 @@ struct DfflibmapPass : public Pass { LibertyMergedCells merged; for (auto path : liberty_files) { - std::istream& f = uncompressed(path); - if (f.fail()) + std::istream* f = uncompressed(path); + if (f->fail()) log_cmd_error("Can't open liberty file `%s': %s\n", path.c_str(), strerror(errno)); - LibertyParser p(f); + LibertyParser p(*f); merged.merge(p); - delete &f; + delete f; } find_cell(merged.cells, ID($_DFF_N_), false, false, false, false, false, false, dont_use_cells); From 4d6f16567c8cedd235d385f0d4c2830739a42bc9 Mon Sep 17 00:00:00 2001 From: "Emil J. Tywoniak" Date: Fri, 10 Jan 2025 23:31:52 +0100 Subject: [PATCH 11/14] abc_new: extract .lib.gz to temporary directory --- passes/techmap/abc_new.cc | 59 +++++++++++++++++++++++++++++++++++---- 1 file changed, 54 insertions(+), 5 deletions(-) diff --git a/passes/techmap/abc_new.cc b/passes/techmap/abc_new.cc index 5be8239163e..0e533dd9b6c 100644 --- a/passes/techmap/abc_new.cc +++ b/passes/techmap/abc_new.cc @@ -20,6 +20,8 @@ #include "kernel/register.h" #include "kernel/rtlil.h" #include "kernel/utils.h" +#include "kernel/io.h" +#include "kernel/gzip.h" USING_YOSYS_NAMESPACE PRIVATE_NAMESPACE_BEGIN @@ -80,6 +82,8 @@ struct AbcNewPass : public ScriptPass { bool cleanup; std::string abc_exe_options; + std::string top_tmpdir; + std::vector liberty_files; void execute(std::vector args, RTLIL::Design *d) override { @@ -89,8 +93,7 @@ struct AbcNewPass : public ScriptPass { for (argidx = 1; argidx < args.size(); argidx++) { if (args[argidx] == "-exe" || args[argidx] == "-script" || args[argidx] == "-D" || - args[argidx] == "-constr" || args[argidx] == "-dont_use" || - args[argidx] == "-liberty") { + args[argidx] == "-constr" || args[argidx] == "-dont_use") { abc_exe_options += " " + args[argidx] + " " + args[argidx + 1]; argidx++; } else if (args[argidx] == "-run" && argidx + 1 < args.size()) { @@ -101,6 +104,9 @@ struct AbcNewPass : public ScriptPass { run_to = args[argidx].substr(pos + 1); } else if (args[argidx] == "-nocleanup") { cleanup = false; + } else if (args[argidx] == "-liberty") { + liberty_files.push_back(args[argidx + 1]); + argidx++; } else { break; } @@ -109,7 +115,13 @@ struct AbcNewPass : public ScriptPass { log_header(d, "Executing ABC_NEW pass.\n"); log_push(); + top_tmpdir = cleanup ? (get_base_tmpdir() + "/") : "_tmp_"; + top_tmpdir += proc_program_prefix() + "yosys-abc-XXXXXX"; + top_tmpdir = make_temp_dir(top_tmpdir); + lib_to_tmp(); run_script(d, run_from, run_to); + if (cleanup) + remove_directory(top_tmpdir); log_pop(); } @@ -146,12 +158,11 @@ struct AbcNewPass : public ScriptPass { } for (auto mod : selected_modules) { - std::string tmpdir = ""; + std::string tmpdir = "/"; std::string modname = ""; std::string exe_options = "[options]"; if (!help_mode) { - tmpdir = cleanup ? (get_base_tmpdir() + "/") : "_tmp_"; - tmpdir += proc_program_prefix() + "yosys-abc-XXXXXX"; + tmpdir = top_tmpdir + "/XXXXXX"; tmpdir = make_temp_dir(tmpdir); modname = mod->name.str(); exe_options = abc_exe_options; @@ -197,6 +208,44 @@ struct AbcNewPass : public ScriptPass { active_design->selection_stack.pop_back(); } } + + if (cleanup) + { + log("Removing temp directory.\n"); + remove_directory(top_tmpdir); + } + } + + void lib_to_tmp() + { + for (auto f : liberty_files) { + bool ends_gz = false; + auto dot_pos = f.find_last_of("."); + if(dot_pos != std::string::npos) + ends_gz = f.substr(dot_pos+1) == "gz"; + log_debug("Does %s end with .gz? %d\n", f.c_str(), ends_gz); + if (ends_gz) { + auto filename_pos = f.find_last_of("/"); + if(filename_pos == std::string::npos) + filename_pos = f.find_last_of("\\"); + if(filename_pos == std::string::npos) { + filename_pos = 0; + } else { + filename_pos++; + } + std::istream* s = uncompressed(f); + std::string base = f.substr(filename_pos, dot_pos - filename_pos); + log_debug("base %s\n", base.c_str()); + std::string tmp_f = top_tmpdir + "/" + base + "-XXXXXX"; + tmp_f = make_temp_file(tmp_f); + log_debug("tmp_f %s\n", tmp_f.c_str()); + std::ofstream out(tmp_f); + out << s->rdbuf(); + abc_exe_options += stringf(" -liberty %s", tmp_f.c_str()); + } else { + abc_exe_options += stringf(" -liberty %s", f.c_str()); + } + } } } AbcNewPass; From ee0b91417cb375f8db66090ffec0bfca02fe99c6 Mon Sep 17 00:00:00 2001 From: "Emil J. Tywoniak" Date: Mon, 13 Jan 2025 12:34:01 +0100 Subject: [PATCH 12/14] abc: centralize gzipped Liberty extraction, use a separate directory, still allow per-module tmp dirs --- passes/techmap/abc.cc | 59 ++++++++++++++++++++++++++++++++++---- passes/techmap/abc_new.cc | 60 +++++++++++---------------------------- 2 files changed, 70 insertions(+), 49 deletions(-) diff --git a/passes/techmap/abc.cc b/passes/techmap/abc.cc index cc37677ce21..e44b33fc391 100644 --- a/passes/techmap/abc.cc +++ b/passes/techmap/abc.cc @@ -48,6 +48,7 @@ #include "kernel/ff.h" #include "kernel/cost.h" #include "kernel/log.h" +#include "kernel/gzip.h" #include #include #include @@ -71,6 +72,51 @@ namespace abc { #endif USING_YOSYS_NAMESPACE + +std::string tmp_base(bool cleanup) +{ + std::string base; + if (cleanup) + base = get_base_tmpdir() + "/"; + else + base = "_tmp_"; + return base + proc_program_prefix(); +} + +/** + * Extracts gzipped liberty_files and rewrites their paths + * to the new temporary file paths + */ +void lib_to_tmp(std::string top_tmpdir, std::vector& liberty_files) +{ + for (std::string& f : liberty_files) { + bool ends_gz = false; + auto dot_pos = f.find_last_of("."); + if(dot_pos != std::string::npos) + ends_gz = f.substr(dot_pos+1) == "gz"; + log_debug("Does %s end with .gz? %d\n", f.c_str(), ends_gz); + if (ends_gz) { + auto filename_pos = f.find_last_of("/"); + if(filename_pos == std::string::npos) + filename_pos = f.find_last_of("\\"); + if(filename_pos == std::string::npos) { + filename_pos = 0; + } else { + filename_pos++; + } + std::istream* s = uncompressed(f); + std::string base = f.substr(filename_pos, dot_pos - filename_pos); + log_debug("base %s\n", base.c_str()); + std::string tmp_f = top_tmpdir + "/" + base + "-XXXXXX"; + tmp_f = make_temp_file(tmp_f); + log_debug("tmp_f %s\n", tmp_f.c_str()); + std::ofstream out(tmp_f); + out << s->rdbuf(); + f = tmp_f; + } + } +} + PRIVATE_NAMESPACE_BEGIN enum class gate_type_t { @@ -787,12 +833,7 @@ void abc_module(RTLIL::Design *design, RTLIL::Module *current_module, std::strin if (dff_mode && clk_sig.empty()) log_cmd_error("Clock domain %s not found.\n", clk_str.c_str()); - std::string tempdir_name; - if (cleanup) - tempdir_name = get_base_tmpdir() + "/"; - else - tempdir_name = "_tmp_"; - tempdir_name += proc_program_prefix() + "yosys-abc-XXXXXX"; + std::string tempdir_name = tmp_base(cleanup) + "yosys-abc-XXXXXX"; tempdir_name = make_temp_dir(tempdir_name); log_header(design, "Extracting gate netlist of module `%s' to `%s/input.blif'..\n", module->name.c_str(), replace_tempdir(tempdir_name, tempdir_name, show_tempdir).c_str()); @@ -2041,6 +2082,9 @@ struct AbcPass : public Pass { enabled_gates.insert("MUX"); // enabled_gates.insert("NMUX"); } + std::string lib_tempdir_name = tmp_base(cleanup) + "yosys-abc-lib-XXXXXX"; + lib_tempdir_name = make_temp_dir(lib_tempdir_name); + lib_to_tmp(lib_tempdir_name, liberty_files); for (auto mod : design->selected_modules()) { @@ -2221,6 +2265,9 @@ struct AbcPass : public Pass { } } + if (cleanup) + remove_directory(lib_tempdir_name); + assign_map.clear(); signal_list.clear(); signal_map.clear(); diff --git a/passes/techmap/abc_new.cc b/passes/techmap/abc_new.cc index 0e533dd9b6c..dbf3982c9bc 100644 --- a/passes/techmap/abc_new.cc +++ b/passes/techmap/abc_new.cc @@ -24,6 +24,10 @@ #include "kernel/gzip.h" USING_YOSYS_NAMESPACE + +void lib_to_tmp(std::string top_tmpdir, std::vector& liberty_files); +std::string tmp_base(bool cleanup); + PRIVATE_NAMESPACE_BEGIN std::vector order_modules(Design *design, std::vector modules) @@ -82,7 +86,6 @@ struct AbcNewPass : public ScriptPass { bool cleanup; std::string abc_exe_options; - std::string top_tmpdir; std::vector liberty_files; void execute(std::vector args, RTLIL::Design *d) override @@ -115,13 +118,12 @@ struct AbcNewPass : public ScriptPass { log_header(d, "Executing ABC_NEW pass.\n"); log_push(); - top_tmpdir = cleanup ? (get_base_tmpdir() + "/") : "_tmp_"; - top_tmpdir += proc_program_prefix() + "yosys-abc-XXXXXX"; - top_tmpdir = make_temp_dir(top_tmpdir); - lib_to_tmp(); + std::string lib_tmpdir = tmp_base(cleanup) + "yosys-abc-lib-XXXXXX"; + lib_tmpdir = make_temp_dir(lib_tmpdir); + lib_to_tmp(lib_tmpdir, liberty_files); run_script(d, run_from, run_to); if (cleanup) - remove_directory(top_tmpdir); + remove_directory(lib_tmpdir); log_pop(); } @@ -157,12 +159,15 @@ struct AbcNewPass : public ScriptPass { run("foreach module in selection"); } + for (auto f : liberty_files) + abc_exe_options += stringf(" -liberty %s", f.c_str()); + for (auto mod : selected_modules) { std::string tmpdir = "/"; std::string modname = ""; std::string exe_options = "[options]"; if (!help_mode) { - tmpdir = top_tmpdir + "/XXXXXX"; + tmpdir = tmp_base(cleanup) + "yosys-abc-XXXXXX"; tmpdir = make_temp_dir(tmpdir); modname = mod->name.str(); exe_options = abc_exe_options; @@ -202,6 +207,11 @@ struct AbcNewPass : public ScriptPass { run("abc9_ops -prep_box"); } } + if (cleanup) + { + log("Removing temp directory.\n"); + remove_directory(tmpdir); + } } if (!help_mode) { @@ -209,44 +219,8 @@ struct AbcNewPass : public ScriptPass { } } - if (cleanup) - { - log("Removing temp directory.\n"); - remove_directory(top_tmpdir); - } } - void lib_to_tmp() - { - for (auto f : liberty_files) { - bool ends_gz = false; - auto dot_pos = f.find_last_of("."); - if(dot_pos != std::string::npos) - ends_gz = f.substr(dot_pos+1) == "gz"; - log_debug("Does %s end with .gz? %d\n", f.c_str(), ends_gz); - if (ends_gz) { - auto filename_pos = f.find_last_of("/"); - if(filename_pos == std::string::npos) - filename_pos = f.find_last_of("\\"); - if(filename_pos == std::string::npos) { - filename_pos = 0; - } else { - filename_pos++; - } - std::istream* s = uncompressed(f); - std::string base = f.substr(filename_pos, dot_pos - filename_pos); - log_debug("base %s\n", base.c_str()); - std::string tmp_f = top_tmpdir + "/" + base + "-XXXXXX"; - tmp_f = make_temp_file(tmp_f); - log_debug("tmp_f %s\n", tmp_f.c_str()); - std::ofstream out(tmp_f); - out << s->rdbuf(); - abc_exe_options += stringf(" -liberty %s", tmp_f.c_str()); - } else { - abc_exe_options += stringf(" -liberty %s", f.c_str()); - } - } - } } AbcNewPass; PRIVATE_NAMESPACE_END From 08857804601fb9739b3d7adecb27334d5eeca376 Mon Sep 17 00:00:00 2001 From: "Emil J. Tywoniak" Date: Mon, 13 Jan 2025 12:40:32 +0100 Subject: [PATCH 13/14] abc_prep: refactor utilities into new namespace --- passes/techmap/abc.cc | 53 +++----------------------------------- passes/techmap/abc9.cc | 10 +++----- passes/techmap/abc_new.cc | 13 ++++------ passes/techmap/abc_prep.h | 54 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 63 deletions(-) create mode 100644 passes/techmap/abc_prep.h diff --git a/passes/techmap/abc.cc b/passes/techmap/abc.cc index e44b33fc391..a6f346ec512 100644 --- a/passes/techmap/abc.cc +++ b/passes/techmap/abc.cc @@ -48,7 +48,6 @@ #include "kernel/ff.h" #include "kernel/cost.h" #include "kernel/log.h" -#include "kernel/gzip.h" #include #include #include @@ -64,6 +63,7 @@ #endif #include "frontends/blif/blifparse.h" +#include "passes/techmap/abc_prep.h" #ifdef YOSYS_LINK_ABC namespace abc { @@ -72,51 +72,6 @@ namespace abc { #endif USING_YOSYS_NAMESPACE - -std::string tmp_base(bool cleanup) -{ - std::string base; - if (cleanup) - base = get_base_tmpdir() + "/"; - else - base = "_tmp_"; - return base + proc_program_prefix(); -} - -/** - * Extracts gzipped liberty_files and rewrites their paths - * to the new temporary file paths - */ -void lib_to_tmp(std::string top_tmpdir, std::vector& liberty_files) -{ - for (std::string& f : liberty_files) { - bool ends_gz = false; - auto dot_pos = f.find_last_of("."); - if(dot_pos != std::string::npos) - ends_gz = f.substr(dot_pos+1) == "gz"; - log_debug("Does %s end with .gz? %d\n", f.c_str(), ends_gz); - if (ends_gz) { - auto filename_pos = f.find_last_of("/"); - if(filename_pos == std::string::npos) - filename_pos = f.find_last_of("\\"); - if(filename_pos == std::string::npos) { - filename_pos = 0; - } else { - filename_pos++; - } - std::istream* s = uncompressed(f); - std::string base = f.substr(filename_pos, dot_pos - filename_pos); - log_debug("base %s\n", base.c_str()); - std::string tmp_f = top_tmpdir + "/" + base + "-XXXXXX"; - tmp_f = make_temp_file(tmp_f); - log_debug("tmp_f %s\n", tmp_f.c_str()); - std::ofstream out(tmp_f); - out << s->rdbuf(); - f = tmp_f; - } - } -} - PRIVATE_NAMESPACE_BEGIN enum class gate_type_t { @@ -833,7 +788,7 @@ void abc_module(RTLIL::Design *design, RTLIL::Module *current_module, std::strin if (dff_mode && clk_sig.empty()) log_cmd_error("Clock domain %s not found.\n", clk_str.c_str()); - std::string tempdir_name = tmp_base(cleanup) + "yosys-abc-XXXXXX"; + std::string tempdir_name = AbcPrep::tmp_base(cleanup) + "yosys-abc-XXXXXX"; tempdir_name = make_temp_dir(tempdir_name); log_header(design, "Extracting gate netlist of module `%s' to `%s/input.blif'..\n", module->name.c_str(), replace_tempdir(tempdir_name, tempdir_name, show_tempdir).c_str()); @@ -2082,9 +2037,9 @@ struct AbcPass : public Pass { enabled_gates.insert("MUX"); // enabled_gates.insert("NMUX"); } - std::string lib_tempdir_name = tmp_base(cleanup) + "yosys-abc-lib-XXXXXX"; + std::string lib_tempdir_name = AbcPrep::tmp_base(cleanup) + "yosys-abc-lib-XXXXXX"; lib_tempdir_name = make_temp_dir(lib_tempdir_name); - lib_to_tmp(lib_tempdir_name, liberty_files); + AbcPrep::lib_to_tmp(lib_tempdir_name, liberty_files); for (auto mod : design->selected_modules()) { diff --git a/passes/techmap/abc9.cc b/passes/techmap/abc9.cc index a96a82659ac..44a033611ec 100644 --- a/passes/techmap/abc9.cc +++ b/passes/techmap/abc9.cc @@ -27,6 +27,8 @@ #include "kernel/rtlil.h" #include "kernel/log.h" +#include "passes/techmap/abc_prep.h" + // abc9_exe.cc std::string fold_abc9_cmd(std::string str); @@ -405,12 +407,8 @@ struct Abc9Pass : public ScriptPass if (!active_design->selected_whole_module(mod)) log_error("Can't handle partially selected module %s!\n", log_id(mod)); - std::string tempdir_name; - if (cleanup) - tempdir_name = get_base_tmpdir() + "/"; - else - tempdir_name = "_tmp_"; - tempdir_name += proc_program_prefix() + "yosys-abc-XXXXXX"; + std::string tempdir_name = AbcPrep::tmp_base(cleanup); + tempdir_name += "yosys-abc-XXXXXX"; tempdir_name = make_temp_dir(tempdir_name); if (!lut_mode) diff --git a/passes/techmap/abc_new.cc b/passes/techmap/abc_new.cc index dbf3982c9bc..15890acf30e 100644 --- a/passes/techmap/abc_new.cc +++ b/passes/techmap/abc_new.cc @@ -21,13 +21,10 @@ #include "kernel/rtlil.h" #include "kernel/utils.h" #include "kernel/io.h" -#include "kernel/gzip.h" -USING_YOSYS_NAMESPACE - -void lib_to_tmp(std::string top_tmpdir, std::vector& liberty_files); -std::string tmp_base(bool cleanup); +#include "passes/techmap/abc_prep.h" +USING_YOSYS_NAMESPACE PRIVATE_NAMESPACE_BEGIN std::vector order_modules(Design *design, std::vector modules) @@ -118,9 +115,9 @@ struct AbcNewPass : public ScriptPass { log_header(d, "Executing ABC_NEW pass.\n"); log_push(); - std::string lib_tmpdir = tmp_base(cleanup) + "yosys-abc-lib-XXXXXX"; + std::string lib_tmpdir = AbcPrep::tmp_base(cleanup) + "yosys-abc-lib-XXXXXX"; lib_tmpdir = make_temp_dir(lib_tmpdir); - lib_to_tmp(lib_tmpdir, liberty_files); + AbcPrep::lib_to_tmp(lib_tmpdir, liberty_files); run_script(d, run_from, run_to); if (cleanup) remove_directory(lib_tmpdir); @@ -167,7 +164,7 @@ struct AbcNewPass : public ScriptPass { std::string modname = ""; std::string exe_options = "[options]"; if (!help_mode) { - tmpdir = tmp_base(cleanup) + "yosys-abc-XXXXXX"; + tmpdir = AbcPrep::tmp_base(cleanup) + "yosys-abc-XXXXXX"; tmpdir = make_temp_dir(tmpdir); modname = mod->name.str(); exe_options = abc_exe_options; diff --git a/passes/techmap/abc_prep.h b/passes/techmap/abc_prep.h new file mode 100644 index 00000000000..70be20c9ff5 --- /dev/null +++ b/passes/techmap/abc_prep.h @@ -0,0 +1,54 @@ +#ifndef ABC_PREP_H +#define ABC_PREP_H + +#include "kernel/gzip.h" + +USING_YOSYS_NAMESPACE + +namespace AbcPrep { + inline std::string tmp_base(bool cleanup) + { + std::string base; + if (cleanup) + base = get_base_tmpdir() + "/"; + else + base = "_tmp_"; + return base + proc_program_prefix(); + } + + /** + * Extracts gzipped liberty_files and rewrites their paths + * to the new temporary file paths + */ + inline void lib_to_tmp(std::string top_tmpdir, std::vector& liberty_files) + { + for (std::string& f : liberty_files) { + bool ends_gz = false; + auto dot_pos = f.find_last_of("."); + if(dot_pos != std::string::npos) + ends_gz = f.substr(dot_pos+1) == "gz"; + log_debug("Does %s end with .gz? %d\n", f.c_str(), ends_gz); + if (ends_gz) { + auto filename_pos = f.find_last_of("/"); + if(filename_pos == std::string::npos) + filename_pos = f.find_last_of("\\"); + if(filename_pos == std::string::npos) { + filename_pos = 0; + } else { + filename_pos++; + } + std::istream* s = uncompressed(f); + std::string base = f.substr(filename_pos, dot_pos - filename_pos); + log_debug("base %s\n", base.c_str()); + std::string tmp_f = top_tmpdir + "/" + base + "-XXXXXX"; + tmp_f = make_temp_file(tmp_f); + log_debug("tmp_f %s\n", tmp_f.c_str()); + std::ofstream out(tmp_f); + out << s->rdbuf(); + f = tmp_f; + } + } + } +}; + +#endif /* ABC_PREP_H */ From b7fe85df4dbb1af0d4960c245cb6ae95a0101b9d Mon Sep 17 00:00:00 2001 From: "Emil J. Tywoniak" Date: Mon, 13 Jan 2025 12:53:47 +0100 Subject: [PATCH 14/14] abc9: enable gzipped Liberty files, extend abc_prep --- passes/techmap/abc.cc | 4 +--- passes/techmap/abc9.cc | 10 +++++++++- passes/techmap/abc_new.cc | 4 +--- passes/techmap/abc_prep.h | 12 ++++++++++++ 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/passes/techmap/abc.cc b/passes/techmap/abc.cc index a6f346ec512..184ce51d073 100644 --- a/passes/techmap/abc.cc +++ b/passes/techmap/abc.cc @@ -2037,9 +2037,7 @@ struct AbcPass : public Pass { enabled_gates.insert("MUX"); // enabled_gates.insert("NMUX"); } - std::string lib_tempdir_name = AbcPrep::tmp_base(cleanup) + "yosys-abc-lib-XXXXXX"; - lib_tempdir_name = make_temp_dir(lib_tempdir_name); - AbcPrep::lib_to_tmp(lib_tempdir_name, liberty_files); + auto lib_tempdir_name = AbcPrep::make_tmp_extract_lib(liberty_files, cleanup); for (auto mod : design->selected_modules()) { diff --git a/passes/techmap/abc9.cc b/passes/techmap/abc9.cc index 44a033611ec..7879bba9fca 100644 --- a/passes/techmap/abc9.cc +++ b/passes/techmap/abc9.cc @@ -191,6 +191,7 @@ struct Abc9Pass : public ScriptPass bool lut_mode; int maxlut; std::string box_file; + std::vector liberty_files; void clear_flags() override { @@ -223,13 +224,17 @@ struct Abc9Pass : public ScriptPass if ((arg == "-exe" || arg == "-script" || arg == "-D" || /*arg == "-S" ||*/ arg == "-lut" || arg == "-luts" || /*arg == "-box" ||*/ arg == "-W" || arg == "-genlib" || - arg == "-constr" || arg == "-dont_use" || arg == "-liberty") && + arg == "-constr" || arg == "-dont_use") && argidx+1 < args.size()) { if (arg == "-lut" || arg == "-luts") lut_mode = true; exe_cmd << " " << arg << " " << args[++argidx]; continue; } + if (arg == "-liberty") { + liberty_files.push_back(args[++argidx]); + continue; + } if (arg == "-fast" || /* arg == "-dff" || */ /* arg == "-nocleanup" || */ arg == "-showtmp") { exe_cmd << " " << arg; @@ -276,7 +281,10 @@ struct Abc9Pass : public ScriptPass log_header(design, "Executing ABC9 pass.\n"); log_push(); + auto lib_tmpdir = AbcPrep::make_tmp_extract_lib(liberty_files, cleanup); run_script(design, run_from, run_to); + if (cleanup) + remove_directory(lib_tmpdir); log_pop(); } diff --git a/passes/techmap/abc_new.cc b/passes/techmap/abc_new.cc index 15890acf30e..7d7ba9a18f9 100644 --- a/passes/techmap/abc_new.cc +++ b/passes/techmap/abc_new.cc @@ -115,9 +115,7 @@ struct AbcNewPass : public ScriptPass { log_header(d, "Executing ABC_NEW pass.\n"); log_push(); - std::string lib_tmpdir = AbcPrep::tmp_base(cleanup) + "yosys-abc-lib-XXXXXX"; - lib_tmpdir = make_temp_dir(lib_tmpdir); - AbcPrep::lib_to_tmp(lib_tmpdir, liberty_files); + auto lib_tmpdir = AbcPrep::make_tmp_extract_lib(liberty_files, cleanup); run_script(d, run_from, run_to); if (cleanup) remove_directory(lib_tmpdir); diff --git a/passes/techmap/abc_prep.h b/passes/techmap/abc_prep.h index 70be20c9ff5..af88371e9b0 100644 --- a/passes/techmap/abc_prep.h +++ b/passes/techmap/abc_prep.h @@ -49,6 +49,18 @@ namespace AbcPrep { } } } + + inline std::string make_tmp_extract_lib(std::vector& liberty_files, bool cleanup) + { + // Compose the path + std::string lib_tmpdir = AbcPrep::tmp_base(cleanup) + "yosys-abc-lib-XXXXXX"; + // Create the directory + lib_tmpdir = make_temp_dir(lib_tmpdir); + // Extract compressed liberty files to directory, rewrite liberty_files + AbcPrep::lib_to_tmp(lib_tmpdir, liberty_files); + // Caller responsible for cleanup + return lib_tmpdir; + } }; #endif /* ABC_PREP_H */