diff --git a/.gitignore b/.gitignore index 8442e0d..268bd3e 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ Release/* .cproject .project .settings/* +*.orig diff --git a/AUTHORS.md b/AUTHORS.md new file mode 100644 index 0000000..5a35afb --- /dev/null +++ b/AUTHORS.md @@ -0,0 +1,7 @@ +Authors +======= + +The authors and developers of MiniDNN are: + - [Yixuan Qiu](https://statr.me/about/) () + - [Giovanni Stabile](https://www.giovannistabile.com/) () + diff --git a/include/Activation/Identity.h b/include/Activation/Identity.h index dd94f60..2e8b79e 100644 --- a/include/Activation/Identity.h +++ b/include/Activation/Identity.h @@ -40,6 +40,11 @@ class Identity { G.noalias() = F; } + + static std::string return_type() + { + return "Identity"; + } }; diff --git a/include/Activation/Mish.h b/include/Activation/Mish.h new file mode 100644 index 0000000..a9dbaa9 --- /dev/null +++ b/include/Activation/Mish.h @@ -0,0 +1,59 @@ +#ifndef ACTIVATION_MISH_H_ +#define ACTIVATION_MISH_H_ + +#include +#include "../Config.h" + +namespace MiniDNN +{ + + +/// +/// \ingroup Activations +/// +/// The Mish activation function +/// +/// from : https://arxiv.org/abs/1908.08681 +/// +class Mish +{ + private: + typedef Eigen::Matrix Matrix; + + public: + // a = activation(z) = max(z, 0) + // Z = [z1, ..., zn], A = [a1, ..., an], n observations + static inline void activate(const Matrix& Z, Matrix& A) + { + A.array() = Z.array() * ((((Z.array()).exp()).log1p()).tanh()); + } + + // Apply the Jacobian matrix J to a vector f + // J = d_a / d_z = diag(sign(a)) = diag(a > 0) + // g = J * f = (a > 0) .* f + // Z = [z1, ..., zn], G = [g1, ..., gn], F = [f1, ..., fn] + // Note: When entering this function, Z and G may point to the same matrix + static inline void apply_jacobian(const Matrix& Z, const Matrix& A, + const Matrix& F, Matrix& G) + { + Matrix tempSoftplus; + Matrix tempSech; + Matrix ex; + ex.array() = Z.array().exp(); + tempSoftplus.array() = ex.array().log1p(); + tempSech.array() = Scalar(1) / (tempSoftplus.array().cosh()); + G.array() = tempSoftplus.array().tanh() + Z.array() * ex.array() * + tempSech.array() * (tempSech.array() / (Scalar(1) + ex.array())) * F.array(); + } + + static std::string return_type() + { + return "Mish"; + } +}; + + +} // namespace MiniDNN + + +#endif /* ACTIVATION_MISH_H_ */ diff --git a/include/Activation/ReLU.h b/include/Activation/ReLU.h index 686d157..572bdd8 100644 --- a/include/Activation/ReLU.h +++ b/include/Activation/ReLU.h @@ -36,6 +36,11 @@ class ReLU { G.array() = (A.array() > Scalar(0)).select(F, Scalar(0)); } + + static std::string return_type() + { + return "ReLU"; + } }; diff --git a/include/Activation/Sigmoid.h b/include/Activation/Sigmoid.h index b9f2db5..8b3f432 100644 --- a/include/Activation/Sigmoid.h +++ b/include/Activation/Sigmoid.h @@ -36,6 +36,11 @@ class Sigmoid { G.array() = A.array() * (Scalar(1) - A.array()) * F.array(); } + + static std::string return_type() + { + return "Sigmoid"; + } }; diff --git a/include/Activation/Softmax.h b/include/Activation/Softmax.h index 6d39913..60f3d8e 100644 --- a/include/Activation/Softmax.h +++ b/include/Activation/Softmax.h @@ -37,6 +37,11 @@ class Softmax RowArray a_dot_f = A.cwiseProduct(F).colwise().sum(); G.array() = A.array() * (F.array().rowwise() - a_dot_f); } + + static std::string return_type() + { + return "Softmax"; + } }; diff --git a/include/Layer.h b/include/Layer.h index 95f9b25..e0e99d9 100644 --- a/include/Layer.h +++ b/include/Layer.h @@ -72,6 +72,13 @@ class Layer /// \param rng The random number generator of type RNG. virtual void init(const Scalar& mu, const Scalar& sigma, RNG& rng) = 0; + /// + /// Initialize layer parameters using without distribution, used just when the layer is read from file + /// + virtual void init() = 0; + + + /// /// Compute the output of this layer /// @@ -152,6 +159,22 @@ class Layer /// Get serialized values of the gradient of parameters /// virtual std::vector get_derivatives() const = 0; + + /// + /// @brief Return the layer type, useful to export the NN model + /// + /// @return Type of the layer + /// + virtual std::string layer_type() const = 0; + + /// + /// @brief Return the activation type, useful to export the NN model + /// + /// @return Type of the activation type + /// + virtual std::string activation_type() const = 0; + + }; diff --git a/include/Layer/Convolutional.h b/include/Layer/Convolutional.h index eb317c8..9c96104 100644 --- a/include/Layer/Convolutional.h +++ b/include/Layer/Convolutional.h @@ -82,6 +82,12 @@ class Convolutional: public Layer internal::set_normal_random(m_bias.data(), m_dim.out_channels, rng, mu, sigma); } + void init() + { + M_assert(1 = 2, + "At the moment the readNet method is implemented only for fully connected layers!!"); + } + // http://cs231n.github.io/convolutional-networks/ void forward(const Matrix& prev_layer_data) { @@ -211,6 +217,17 @@ class Convolutional: public Layer res.begin() + m_df_data.size()); return res; } + + std::string layer_type() const + { + return "Convolutional"; + } + + std::string activation_type() const + { + return Activation::return_type(); + } + }; diff --git a/include/Layer/FullyConnected.h b/include/Layer/FullyConnected.h index 9a549a9..284e87b 100644 --- a/include/Layer/FullyConnected.h +++ b/include/Layer/FullyConnected.h @@ -3,6 +3,7 @@ #include #include +#include #include #include "../Config.h" #include "../Layer.h" @@ -57,6 +58,14 @@ class FullyConnected: public Layer internal::set_normal_random(m_bias.data(), m_bias.size(), rng, mu, sigma); } + void init() + { + m_weight.resize(this->m_in_size, this->m_out_size); + m_bias.resize(this->m_out_size); + m_dw.resize(this->m_in_size, this->m_out_size); + m_db.resize(this->m_out_size); + } + // prev_layer_data: in_size x nobs void forward(const Matrix& prev_layer_data) { @@ -140,6 +149,16 @@ class FullyConnected: public Layer std::copy(m_db.data(), m_db.data() + m_db.size(), res.begin() + m_dw.size()); return res; } + + std::string layer_type() const + { + return "FullyConnected"; + } + + std::string activation_type() const + { + return Activation::return_type(); + } }; diff --git a/include/Layer/MaxPooling.h b/include/Layer/MaxPooling.h index 402658f..7aaeb2f 100644 --- a/include/Layer/MaxPooling.h +++ b/include/Layer/MaxPooling.h @@ -65,6 +65,10 @@ class MaxPooling: public Layer void init(const Scalar& mu, const Scalar& sigma, RNG& rng) {} + void init() {} + + + void forward(const Matrix& prev_layer_data) { // Each column is an observation @@ -166,6 +170,16 @@ class MaxPooling: public Layer { return std::vector(); } + + std::string layer_type() const + { + return "MaxPooling"; + } + + std::string activation_type() const + { + return Activation::return_type(); + } }; diff --git a/include/MiniDNN.h b/include/MiniDNN.h index d4d6002..1d95719 100644 --- a/include/MiniDNN.h +++ b/include/MiniDNN.h @@ -13,6 +13,7 @@ #include "Layer/MaxPooling.h" #include "Activation/ReLU.h" +#include "Activation/Mish.h" #include "Activation/Identity.h" #include "Activation/Sigmoid.h" #include "Activation/Softmax.h" @@ -31,6 +32,8 @@ #include "Callback.h" #include "Callback/VerboseCallback.h" +#include "Utils/MiniDNNStream.h" + #include "Network.h" diff --git a/include/Network.h b/include/Network.h index b43417f..03fa70d 100644 --- a/include/Network.h +++ b/include/Network.h @@ -10,6 +10,7 @@ #include "Output.h" #include "Callback.h" #include "Utils/Random.h" +#include "Utils/MiniDNNStream.h" namespace MiniDNN { @@ -42,6 +43,8 @@ class Network Callback* m_callback; // Points to user-provided callback function, // otherwise points to m_default_callback + std::map netMap; + std::vector< std::vector > params; // Check dimensions of layers void check_unit_sizes() const @@ -484,6 +487,148 @@ class Network this->forward(x); return m_layers[nlayer - 1]->output(); } + + /// + /// @brief Export a net to file inside a certain folder with a certain file name + /// + /// @param[in] folder The folder where you want to save the net + /// @param[in] fileName The filename you want to use for the net + /// + void export_net(std::string folder, std::string fileName) + { + system(("mkdir " + folder).c_str()); + fill_map(); + write_map(folder + "/" + fileName, netMap); + write_parameters(folder, fileName, params); + } + + + /// + /// @brief Reads a net from a specific folder with a specific name + /// + /// @param[in] folder The folder where the net is located + /// @param[in] fileName The file name of the net + /// + void read_net(std::string folder, std::string fileName) + { + netMap.clear(); + read_map(folder + "/" + fileName, netMap); + int Nlayers = netMap.find("Nlayers")->second; + params = read_parameters(folder, fileName, Nlayers); + m_layers.clear(); + + for (int i = 0; i < Nlayers; i++) + { + add_layer(create_layer(i)); + } + + this->set_parameters(params); + set_output(create_output()); + } + + /// + /// @brief Creates a layer from the netMap given the index of the layer + /// + /// @param[in] index The index of the layer + /// + /// @return a pointer to the layer object + /// + Layer* create_layer(int index) + { + int layer_type = netMap.find("Layer" + to_string(index))->second; + int activation_type = netMap.find("Activation" + to_string(index))->second; + Layer* layer; + + if (layer_type == 2) + { + int m_in_size = netMap.find("m_in_size" + to_string(index))->second; + int m_out_size = netMap.find("m_out_size" + to_string(index))->second; + + if (activation_type == 0) + { + layer = new FullyConnected(m_in_size, m_out_size); + } + + if (activation_type == 1) + { + layer = new FullyConnected(m_in_size, m_out_size); + } + + if (activation_type == 2) + { + layer = new FullyConnected(m_in_size, m_out_size); + } + + if (activation_type == 3) + { + layer = new FullyConnected(m_in_size, m_out_size); + } + + if (activation_type == 4) + { + layer = new FullyConnected(m_in_size, m_out_size); + } + } + + layer->init(); + return layer; + } + + /// + /// @brief Creates an output layer from the netMap + /// + /// @return a pointer to the output layer + /// + Output* create_output() + { + Output* output; + int output_type = netMap.find("OutputLayer")->second; + + if (output_type == 0) + { + output = new RegressionMSE(); + } + else + { + std::cout << "This function is implemented only for RegressionMSE output layers" << std::endl; + exit(0); + } + + return output; + } + + /// + /// @brief Fill the netMap object, useful to export the net + /// + /// @return the netMap + /// + std::map fill_map() + { + netMap.clear(); + M_Assert(num_layers() > 0, "The net has zero layers"); + netMap.insert(std::pair("Nlayers", num_layers())); + params = get_parameters(); + + for (int i = 0; i < num_layers(); i++) + { + netMap.insert(std::pair("Layer" + to_string(i), + layer_type(m_layers[i]->layer_type()))); + netMap.insert(std::pair("Activation" + to_string( + i), activation_type(m_layers[i]->activation_type()))); + netMap.insert(std::pair("m_in_size" + to_string( + i), m_layers[i]->in_size())); + netMap.insert(std::pair("m_out_size" + to_string( + i), m_layers[i]->out_size())); + } + + netMap.insert(std::pair("OutputLayer", + output_type(m_output->output_type()))); + return netMap; + } + std::map getNetMap() + { + return netMap; + } }; diff --git a/include/Output.h b/include/Output.h index 0b04bd4..854538f 100644 --- a/include/Output.h +++ b/include/Output.h @@ -64,6 +64,13 @@ class Output // This function can be assumed to be called after evaluate(), so that it can make use of the // intermediate result to save some computation virtual Scalar loss() const = 0; + + /// + /// @brief Return the layer type, useful to export the NN model + /// + /// @return Type of the layer + /// + virtual std::string output_type() const = 0; }; diff --git a/include/Output/BinaryClassEntropy.h b/include/Output/BinaryClassEntropy.h index f3ea0b3..e1a734a 100644 --- a/include/Output/BinaryClassEntropy.h +++ b/include/Output/BinaryClassEntropy.h @@ -110,6 +110,12 @@ class BinaryClassEntropy: public Output // L = log(abs(m_din)).sum() return m_din.array().abs().log().sum() / m_din.cols(); } + + + std::string output_type() const + { + return "BinaryClassEntropy"; + } }; diff --git a/include/Output/MultiClassEntropy.h b/include/Output/MultiClassEntropy.h index b7d7ee4..902e8cb 100644 --- a/include/Output/MultiClassEntropy.h +++ b/include/Output/MultiClassEntropy.h @@ -139,6 +139,11 @@ class MultiClassEntropy: public Output return res / m_din.cols(); } + + std::string output_type() const + { + return "MultiClassEntropy"; + } }; diff --git a/include/Output/RegressionMSE.h b/include/Output/RegressionMSE.h index 15dc5f4..08b1470 100644 --- a/include/Output/RegressionMSE.h +++ b/include/Output/RegressionMSE.h @@ -53,6 +53,11 @@ class RegressionMSE: public Output // L = 0.5 * ||yhat - y||^2 return m_din.squaredNorm() / m_din.cols() * Scalar(0.5); } + + std::string output_type() const + { + return "RegressionMSE"; + } }; diff --git a/include/Utils/EigenStream.h b/include/Utils/EigenStream.h deleted file mode 100644 index 966d53a..0000000 --- a/include/Utils/EigenStream.h +++ /dev/null @@ -1,239 +0,0 @@ -/// \file -/// Source code file of the EigenStream class, it contains the implementation of -/// several methods for input output operations. - -#ifndef EigenStream_H -#define EigenStream_H - -#include -#include -#include -#include -#include -#include "M_Assert.h" -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wold-style-cast" -#include -#include -#pragma GCC diagnostic pop -#define MAXBUFSIZE (static_cast (1e6)) - - -// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // - -/// Class for input-output manipulation -class EigenStream -{ - private: - - public: - - //-------------------------------------------------------------------------- - /// Export the matrices in numpy (type=python), matlab (type=matlab) format and txt (type=eigen) format - /* In this case the function is implemented for a second order matrix */ - /// - /// @param[in] matrice Eigen::MatrixXd that you want to export. - /// @param[in] Name string to identify the name you want to use to save the file. - /// @param[in] type string to identify format to export the matrix if numpy (type="python"), if matlab (type="matlab") if txt (type="eigen"). - /// @param[in] folder string to identify the folder where you want to save the file. - /// - static void exportMatrix(Eigen::MatrixXd& matrix, std::string Name, - std::string type = "python", std::string folder = "./Model") - { - mkdir(folder.c_str(), ACCESSPERMS); - std::string est; - - if (type == "python") - { - est = ".py"; - std::string filename(folder + "/" + Name + "_mat" + est); - std::ofstream str(filename.c_str()); - str << Name << "=np.array(["; - - for (int i = 0; i < matrix.rows(); i++) - { - for (int j = 0; j < matrix.cols(); j++) - { - if (j == 0) - { - str << "[" << matrix(i, j); - } - else - { - str << "," << matrix(i, j); - } - } - - if (i != (matrix.rows() - 1)) - { - str << "]," << std::endl; - } - } - - str << "]])" << std::endl; - } - - if (type == "matlab") - { - est = ".m"; - std::string filename(folder + "/" + Name + "_mat" + est); - std::ofstream str(filename.c_str()); - str << Name << "=["; - - for (int i = 0; i < matrix.rows(); i++) - { - for (int j = 0; j < matrix.cols(); j++) - { - str << " " << matrix(i, j); - } - - if (i != (matrix.rows() - 1)) - { - str << ";" << std::endl; - } - } - - str << "];" << std::endl; - } - - if (type == "eigen") - { - std::ofstream ofs; - std::string filename(folder + "/" + Name + "_mat.txt"); - ofs.open(filename.c_str()); - ofs << matrix << std::endl; - ofs.close(); - } - } - - //-------------------------------------------------------------------------- - /// @brief Saves a dense matrix to a binary format file - /// - /// @param[in] Matrix The Eigen dense matrix - /// @param[in] folder the folder where you want to save the matrix - /// @param[in] MatrixName The matrix name for the output file - /// - /// @tparam MatrixType type of the matrix, i.e. double, float, ... - /// - template - static void SaveDenseMatrix(MatrixType& Matrix, std::string folder, - std::string MatrixName) - { - mkdir(folder.c_str(), ACCESSPERMS); - std::ofstream out(folder + MatrixName, - std::ios::out | std::ios::binary | std::ios::trunc); - typename MatrixType::Index rows = Matrix.rows(), cols = Matrix.cols(); - out.write(reinterpret_cast (&rows), sizeof(typename MatrixType::Index)); - out.write(reinterpret_cast (&cols), sizeof(typename MatrixType::Index)); - out.write(reinterpret_cast (Matrix.data()), - rows * cols * sizeof(typename MatrixType::Scalar) ); - out.close(); - } - - //-------------------------------------------------------------------------- - /// @brief Reads a dense matrix from a binary format file - /// - /// @param[in,out] Matrix The Eigen dense matrix - /// @param[in] folder The folder from where you want to read the matrix - /// @param[in] MatrixName The matrix name of the input file - /// - /// @tparam MatrixType type of the matrix, i.e. double, float, ... - /// - template - void ReadDenseMatrix(MatrixType& Matrix, std::string folder, - std::string MatrixName) - { - std::ifstream in; - in.open((folder + MatrixName).c_str(), std::ios::in | std::ios::binary); - std::string message(folder + MatrixName + - " file does not exist, Check if the file is existing"); - M_Assert(in.good(), message.c_str()); - - if (in.is_open()) - { - typename MatrixType::Index rows = 0, cols = 0; - in.read(reinterpret_cast (&rows), sizeof(typename MatrixType::Index)); - in.read(reinterpret_cast (&cols), sizeof(typename MatrixType::Index)); - Matrix.resize(rows, cols); - in.read( reinterpret_cast(Matrix.data()), - rows * cols * sizeof(typename MatrixType::Scalar) ); - in.close(); - } - } - - //---------------------------------------------------------------------- - /// @brief Saves a dense tensor. - /// - /// @param Tensor The tensor - /// @param[in] folder The folder - /// @param[in] MatrixName The matrix name - /// - /// @tparam TensorType type of the tensor, i.e. double, float, ... - /// - template - static void SaveDenseTensor(TensorType& Tensor, std::string folder, - std::string MatrixName) - { - std::ofstream out(folder + MatrixName, - std::ios::out | std::ios::binary | std::ios::trunc); - typename TensorType::Dimensions dim = Tensor.dimensions(); - int tot = 1; - - for (unsigned int k = 0; k < dim.size(); k++) - { - tot *= dim[k]; - } - - out.write(reinterpret_cast (&dim), - sizeof(typename TensorType::Dimensions)); - out.write(reinterpret_cast (Tensor.data()), - tot * sizeof(typename TensorType::Scalar) ); - out.close(); - } - - //---------------------------------------------------------------------- - /// @brief Reads a dense tensor. - /// - /// @param Tensor The tensor - /// @param[in] folder The folder - /// @param[in] MatrixName The matrix name - /// - /// @tparam TensorType type of the tensor, i.e. double, float, ... - /// - template - static void ReadDenseTensor(TensorType& Tensor, std::string folder, - std::string MatrixName) - { - std::ifstream in; - in.open((folder + MatrixName).c_str(), std::ios::in | std::ios::binary); - typename TensorType::Dimensions dim; - in.read(reinterpret_cast (&dim), - sizeof(typename TensorType::Dimensions)); - auto dims = Tensor.dimensions(); - M_Assert(dims.size() == dim.size(), - "The rank of the tensor you want to fill does not coincide with the rank of the tensor you are reading"); - int tot = 1; - - for (unsigned int k = 0; k < dim.size(); k++) - { - tot *= dim[k]; - } - - Tensor.resize(dim); - in.read( reinterpret_cast(Tensor.data()), - tot * sizeof(typename TensorType::Scalar) ); - in.close(); - } - - - - -}; - -#endif - - - - - - diff --git a/include/Utils/MiniDNNStream.h b/include/Utils/MiniDNNStream.h new file mode 100644 index 0000000..0ce7a20 --- /dev/null +++ b/include/Utils/MiniDNNStream.h @@ -0,0 +1,546 @@ +/// \file +/// Source code file of the EigenStream class, it contains the implementation of +/// several methods for input output operations. + +#ifndef MiniDNNStream_H +#define MiniDNNStream_H + +#include +#include +#include +#include +#include +#include "Assert.h" +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wold-style-cast" +#include +#include +#pragma GCC diagnostic pop +#define MAXBUFSIZE (static_cast (1e6)) + + +// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * // + +/// Class for input-output manipulation +namespace MiniDNN +{ + +//-------------------------------------------------------------------------- +/// @brief function to convert a number to a string in plan c++98 +/// +/// @param[in] num The number +/// +/// @tparam numberType type of number +/// +/// @return a std::string containing the number +/// +template +std::string to_string(numberType num) +{ + std::ostringstream convert; + convert << num; + return convert.str(); +} + + +//-------------------------------------------------------------------------- +/// Export the matrices in numpy (type=python), matlab (type=matlab) format and txt (type=eigen) format +/* In this case the function is implemented for a second order matrix */ +/// +/// @param[in] matrice Eigen::MatrixXd that you want to export. +/// @param[in] Name string to identify the name you want to use to save the file. +/// @param[in] type string to identify format to export the matrix if numpy (type="python"), if matlab (type="matlab") if txt (type="eigen"). +/// @param[in] folder string to identify the folder where you want to save the file. +/// +void export_matrix(Eigen::MatrixXd& matrix, std::string Name, + std::string type = "python", std::string folder = "./Model") +{ + mkdir(folder.c_str(), ACCESSPERMS); + std::string est; + + if (type == "python") + { + est = ".py"; + std::string filename(folder + "/" + Name + "_mat" + est); + std::ofstream str(filename.c_str()); + str << Name << "=np.array(["; + + for (int i = 0; i < matrix.rows(); i++) + { + for (int j = 0; j < matrix.cols(); j++) + { + if (j == 0) + { + str << "[" << matrix(i, j); + } + + else + { + str << "," << matrix(i, j); + } + } + + if (i != (matrix.rows() - 1)) + { + str << "]," << std::endl; + } + } + + str << "]])" << std::endl; + } + + if (type == "matlab") + { + est = ".m"; + std::string filename(folder + "/" + Name + "_mat" + est); + std::ofstream str(filename.c_str()); + str << Name << "=["; + + for (int i = 0; i < matrix.rows(); i++) + { + for (int j = 0; j < matrix.cols(); j++) + { + str << " " << matrix(i, j); + } + + if (i != (matrix.rows() - 1)) + { + str << ";" << std::endl; + } + } + + str << "];" << std::endl; + } + + if (type == "eigen") + { + std::ofstream ofs; + std::string filename(folder + "/" + Name + "_mat.txt"); + ofs.open(filename.c_str()); + ofs << matrix << std::endl; + ofs.close(); + } +} + +//-------------------------------------------------------------------------- +/// @brief Saves a dense matrix to a binary format file +/// +/// @param[in] Matrix The Eigen dense matrix +/// @param[in] folder the folder where you want to save the matrix +/// @param[in] MatrixName The matrix name for the output file +/// +/// @tparam MatrixType type of the matrix, i.e. double, float, ... +/// +template +void save_dense_matrix(MatrixType& Matrix, std::string folder, + std::string MatrixName) +{ + mkdir(folder.c_str(), ACCESSPERMS); + std::ofstream out(folder + MatrixName, + std::ios::out | std::ios::binary | std::ios::trunc); + typename MatrixType::Index rows = Matrix.rows(), cols = Matrix.cols(); + out.write(reinterpret_cast (&rows), sizeof(typename MatrixType::Index)); + out.write(reinterpret_cast (&cols), sizeof(typename MatrixType::Index)); + out.write(reinterpret_cast (Matrix.data()), + rows * cols * sizeof(typename MatrixType::Scalar) ); + out.close(); +} + +//-------------------------------------------------------------------------- +/// @brief Reads a dense matrix from a binary format file +/// +/// @param[in,out] Matrix The Eigen dense matrix +/// @param[in] folder The folder from where you want to read the matrix +/// @param[in] MatrixName The matrix name of the input file +/// +/// @tparam MatrixType type of the matrix, i.e. double, float, ... +/// +template +void read_dense_matrix(MatrixType& Matrix, std::string folder, + std::string MatrixName) +{ + std::ifstream in; + in.open((folder + MatrixName).c_str(), std::ios::in | std::ios::binary); + std::string message(folder + MatrixName + + " file does not exist, Check if the file is existing"); + M_Assert(in.good(), message.c_str()); + + if (in.is_open()) + { + typename MatrixType::Index rows = 0, cols = 0; + in.read(reinterpret_cast (&rows), sizeof(typename MatrixType::Index)); + in.read(reinterpret_cast (&cols), sizeof(typename MatrixType::Index)); + Matrix.resize(rows, cols); + in.read( reinterpret_cast(Matrix.data()), + rows * cols * sizeof(typename MatrixType::Scalar) ); + in.close(); + } +} + +//---------------------------------------------------------------------- +/// @brief Saves a dense tensor. +/// +/// @param Tensor The tensor +/// @param[in] folder The folder +/// @param[in] MatrixName The matrix name +/// +/// @tparam TensorType type of the tensor, i.e. double, float, ... +/// +template +void save_dense_tensor(TensorType& Tensor, std::string folder, + std::string MatrixName) +{ + std::ofstream out(folder + MatrixName, + std::ios::out | std::ios::binary | std::ios::trunc); + typename TensorType::Dimensions dim = Tensor.dimensions(); + int tot = 1; + + for (unsigned int k = 0; k < dim.size(); k++) + { + tot *= dim[k]; + } + + out.write(reinterpret_cast (&dim), + sizeof(typename TensorType::Dimensions)); + out.write(reinterpret_cast (Tensor.data()), + tot * sizeof(typename TensorType::Scalar) ); + out.close(); +} + +//---------------------------------------------------------------------- +/// @brief Reads a dense tensor. +/// +/// @param Tensor The tensor +/// @param[in] folder The folder +/// @param[in] MatrixName The matrix name +/// +/// @tparam TensorType type of the tensor, i.e. double, float, ... +/// +template +void read_dense_tensor(TensorType& Tensor, std::string folder, + std::string MatrixName) +{ + std::ifstream in; + in.open((folder + MatrixName).c_str(), std::ios::in | std::ios::binary); + typename TensorType::Dimensions dim; + in.read(reinterpret_cast (&dim), + sizeof(typename TensorType::Dimensions)); + const typename TensorType::Dimensions& dims = Tensor.dimensions(); + M_Assert(dims.size() == dim.size(), + "The rank of the tensor you want to fill does not coincide with the rank of the tensor you are reading"); + int tot = 1; + + for (unsigned int k = 0; k < dim.size(); k++) + { + tot *= dim[k]; + } + + Tensor.resize(dim); + in.read( reinterpret_cast(Tensor.data()), + tot * sizeof(typename TensorType::Scalar) ); + in.close(); +} + +/// +/// @brief Write a std::vector to file. +/// +/// @param[in] myVector The vector you want to write +/// @param[in] filename The filename of the std::vector +/// +void write_vector_to_file(const std::vector& myVector, + std::string filename) +{ + std::ofstream ofs(filename.c_str(), std::ios::out | std::ios::binary); + std::ostream_iterator osi(ofs); + const char* beginByte = (char*)&myVector[0]; + const char* endByte = (char*)&myVector.back() + sizeof(double); + std::copy(beginByte, endByte, osi); +} +/// +/// @brief Reads a std::vector from file. +/// +/// @param[in] filename The filename of the vector +/// +/// @return The vector +/// +std::vector read_vector_from_file(std::string filename) +{ + std::vector buffer; + std::ifstream ifs(filename.c_str(), std::ios::in | std::ifstream::binary); + std::istreambuf_iterator iter(ifs); + std::istreambuf_iterator end; + std::copy(iter, end, std::back_inserter(buffer)); + std::vector newVector(buffer.size() / sizeof(double)); + memcpy(&newVector[0], &buffer[0], buffer.size()); + return newVector; +} + +/// +/// @brief Writes parameters of the net from file. +/// +/// @param[in] folder The folder where the parameter files are stored +/// @param[in] fileName The file name prefix of the parameter files +/// @param params The parameters of the net +/// +void write_parameters(std::string folder, std::string fileName, + std::vector >& params) +{ + for (int i = 0; i < params.size(); i++) + { + write_vector_to_file(params[i], folder + "/" + fileName + to_string(i)); + } +} + +/// +/// @brief Reads parameters of the net from file +/// +/// @param[in] folder The folder where the parameter files are stored +/// @param[in] fileName The file name prefix of the parameter files +/// @param[in] Nlayers The number of layers +/// +/// @return a vector which contains the parameters +/// +std::vector > read_parameters(std::string folder, + std::string fileName, int Nlayers) +{ + std::vector > params; + + for (int i = 0; i < Nlayers; i++) + { + params.push_back(read_vector_from_file(folder + "/" + fileName + to_string(i))); + } + + return params; +} + + + +/// +/// @brief Convert a layer type string to an integer +/// +/// @param[in] type The type of layer (string) +/// +/// @return 0 for Convolutional, 1 for MaxPooling, 2 for FullyConnected +/// +int layer_type(std::string type) +{ + M_Assert(type == "Convolutional" || type == "MaxPooling" || + type == "FullyConnected", "Layer is not of a known type"); + int out; + + if (type == "Convolutional") + { + out = 0; + } + + if (type == "MaxPooling") + { + out = 1; + } + + if (type == "FullyConnected") + { + out = 2; + } + + return out; +} +/// +/// @brief Convert an activation type string to an integer +/// +/// @param[in] type The type of the activation (string) +/// +/// @return 0 for Identity, 1 for ReLU, 2 for Sigmoid. 3 for Softmax, 4 for Mish +/// +int activation_type(std::string type) +{ + M_Assert(type == "Identity" || type == "ReLU" || + type == "Sigmoid" || type == "Softmax" || + type == "Mish", "Activation is not of a known type"); + int out; + + if (type == "Identity") + { + out = 0; + } + + if (type == "ReLU") + { + out = 1; + } + + if (type == "Sigmoid") + { + out = 2; + } + + if (type == "Softmax") + { + out = 3; + } + + if (type == "Mish") + { + out = 4; + } + + return out; +} +/// +/// @brief Convert an output layer type string to an integer +/// +/// @param[in] type The type of the output layer in string +/// +/// @return 0 for RegressionMSE, 1 for MultiClassEntropy, 2 for BinaryClassEntropy. +/// +int output_type(std::string type) +{ + M_Assert(type == "RegressionMSE" || type == "MultiClassEntropy" || + type == "BinaryClassEntropy" , "output is not of a known type"); + int out; + + if (type == "RegressionMSE") + { + out = 0; + } + + if (type == "MultiClassEntropy") + { + out = 1; + } + + if (type == "BinaryClassEntropy") + { + out = 2; + } + + return out; +} + + + +//-------------------------------------------------------------------------- +/// @brief Writes a map model. +/// +/// @param[in] fileName The file name of the model you want to write +/// @param[in] map The model you want to write +/// +/// @return integer for success +/// +int write_map (std::string fileName, std::map map) +{ + int count = 0; + + if (map.empty()) + { + return 0; + } + + FILE* fp = fopen(fileName.c_str(), "w"); + + if (! fp) + { + return - errno; + } + + for (std::map::iterator it = map.begin(); + it != map.end(); it++) + { + fprintf(fp, "%s=%d\n", it->first.c_str(), it->second); + count++; + } + + fclose(fp); + return count; +} + +//-------------------------------------------------------------------------- +/// @brief Read a map Model. +/// +/// @param[in] fileName The file name of the model you want to read. +/// @param[in] map Map to store the model. +/// +/// @return integer for success +/// +int read_map (std::string fname, std::map& map) +{ + int count = 0; + FILE* fp = fopen(fname.c_str(), "r"); + + if (!fp) + { + return -errno; + } + + map.clear(); + char* buf = 0; + size_t buflen = 0; + + while (getline(&buf, &buflen, fp) > 0) + { + char* nl = strchr(buf, '\n'); + + if (nl == NULL) + { + continue; + } + + *nl = 0; + char* sep = strchr(buf, '='); + + if (sep == NULL) + { + continue; + } + + *sep = 0; + sep++; + std::string s1 = buf; + int s2 = atoi(sep); + (map)[s1] = s2; + count++; + } + + if (buf) + { + free(buf); + } + + fclose(fp); + return count; +} +}; + + +std::ostream& operator<<(std::ostream& os, + std::map const& myMap) +{ + for (std::map::const_iterator it = myMap.begin(); + it != myMap.end(); + ++it) + { + os << it->first << " " << it->second << "\n"; + } + + return os; +} + +std::ostream& operator<<(std::ostream& os, std::vector& myVector) +{ + for (int i = 0; i < myVector.size(); i++) + { + os << myVector[i] << "\n"; + } + + return os; +} + + + +#endif + + + + + + diff --git a/tutorials/test1/.gitignore b/tutorials/test1/.gitignore new file mode 100644 index 0000000..929ed64 --- /dev/null +++ b/tutorials/test1/.gitignore @@ -0,0 +1,2 @@ +*.o +NetFolder \ No newline at end of file diff --git a/tutorials/test1/Makefile b/tutorials/test1/Makefile new file mode 100644 index 0000000..36e8f2b --- /dev/null +++ b/tutorials/test1/Makefile @@ -0,0 +1,12 @@ +.PHONY: all +all: test +# This rule tells make how to build hello from test1.cpp +test: test1.cpp + g++ -O2 -I../../include test1.cpp -o test1.o + +# This rule tells make to delete the program +.PHONY: clean +clean: + rm -f test1.o + + diff --git a/tutorials/test1/test1.cpp b/tutorials/test1/test1.cpp new file mode 100644 index 0000000..08f270b --- /dev/null +++ b/tutorials/test1/test1.cpp @@ -0,0 +1,74 @@ +#include "MiniDNN.h" +#include "Utils/MiniDNNStream.h" +using namespace MiniDNN; + +typedef Eigen::MatrixXd Matrix; +typedef Eigen::VectorXd Vector; + + +int main() +{ + // Create two dimensional input data + Vector x1 = Vector::LinSpaced(1000, 0.0, 3.15); + Vector x2 = Vector::LinSpaced(1000, 0.0, 3.15); + // Predictors -- each column is an observation + Matrix x = Matrix::Random(2, 1000); + x.row(0) = x1; + x.row(1) = x2; + // Response variables -- each column is an observation + Matrix y = Matrix::Random(1, 1000); + + // Fill the output for the training + for (int i = 0; i < y.cols(); i++) + { + y(0, i) = std::pow(x(0, i), 2) + std::pow(x(1, i), 2); + } + + // Fill the output for the test + Matrix xt = (Matrix::Random(2, 1000).array() + 1.0) / 2 * 3.15; + Matrix yt = Matrix::Random(1, 1000); + + for (int i = 0; i < yt.cols(); i++) + { + yt(0, i) = std::pow(xt(0, i), 2) + std::pow(xt(1, i), 2); + } + + // Construct a network object + Network net; + // Create three layers + // Layer 1 -- FullyConnected, input size 2x200 + Layer* layer1 = new FullyConnected(2, 200); + // Layer 2 -- max FullyConnected, input size 200x200 + Layer* layer2 = new FullyConnected(200, 200); + // Layer 4 -- fully connected, input size 200x1 + Layer* layer3 = new FullyConnected(200, 1); + // Add layers to the network object + net.add_layer(layer1); + net.add_layer(layer2); + net.add_layer(layer3); + // Set output layer + net.set_output(new RegressionMSE()); + // Create optimizer object + Adam opt; + opt.m_lrate = 0.001; + // (Optional) set callback function object + VerboseCallback callback; + net.set_callback(callback); + // Initialize parameters with N(0, 0.01^2) using random seed 123 + net.init(0, 0.1, 000); + // Fit the model with a batch size of 100, running 10 epochs with random seed 123 + net.fit(opt, x, y, 500, 5000, 000); + // Obtain prediction -- each column is an observation + Matrix pred = net.predict(xt); + // Export the network to the NetFolder folder with prefix NetFile + net.export_net("./NetFolder/", "NetFile"); + // Create a new network + Network netFromFile; + // Read structure and paramaters from file + netFromFile.read_net("./NetFolder/", "NetFile"); + // Test that they give the same prediction + std::cout << net.predict(xt) - netFromFile.predict(xt) << std::endl; + // Layer objects will be freed by the network object, + // so do not manually delete them + return 0; +} \ No newline at end of file