diff --git a/examples/reinforcement_learning/vin/train_vin.py b/examples/reinforcement_learning/vin/train_vin.py index 1c823fdc..345d5edc 100644 --- a/examples/reinforcement_learning/vin/train_vin.py +++ b/examples/reinforcement_learning/vin/train_vin.py @@ -51,7 +51,7 @@ def output(self, Q, input_state_1, input_state_2, **kwargs): Q_shape = tf.shape(Q) indices = tf.stack([ - # Numer of repetitions depends on the size of + # Number of repetitions depends on the size of # the state batch tf_utils.repeat(tf.range(Q_shape[0]), n_states), diff --git a/neupy/layers/activations.py b/neupy/layers/activations.py index e1c1c229..94753e62 100644 --- a/neupy/layers/activations.py +++ b/neupy/layers/activations.py @@ -440,9 +440,9 @@ def activation_function(self, input_value): class Elu(Linear): """ - Layer with the exponensial linear unit (ELU) used as an activation + Layer with the exponential linear unit (ELU) used as an activation function. It applies linear transformation when the ``n_units`` - arameter specified and elu function after the transformation. + parameter specified and elu function after the transformation. When ``n_units`` is not specified, only elu function will be applied to the input. diff --git a/site/buildsite.sh b/site/buildsite.sh index 590bef4e..fb89a793 100755 --- a/site/buildsite.sh +++ b/site/buildsite.sh @@ -6,7 +6,7 @@ sphinx-apidoc -e \ -o apidocs \ ../neupy && \ tinker --build && \ -python search-index/build.py --for-deploy && \ +#python search-index/build.py --for-deploy && \ export IS_SUCCESS=1 if [ $IS_SUCCESS ]; then diff --git a/site/docs/layers.rst b/site/docs/layers.rst index 9b2508ac..49f6030c 100644 --- a/site/docs/layers.rst +++ b/site/docs/layers.rst @@ -12,5 +12,4 @@ These topics are relevant only to the deep learning models. layers/basics layers/create-custom-layers layers/init-methods - layers/debug - layers/with-tensorflow + layers/debug \ No newline at end of file diff --git a/site/docs/layers/basics.rst b/site/docs/layers/basics.rst index d103cad7..77cc93b3 100644 --- a/site/docs/layers/basics.rst +++ b/site/docs/layers/basics.rst @@ -308,96 +308,3 @@ and many to one .. raw:: html
- -.. _subnetworks: - -Subnetworks -=========== - -**Subnetworks** is a method that improves readability of the neural network architecture. Instead of explaining, it's much easier to show the main advantage of this technique. Here is an example of the simple convolutional network. - -.. code-block:: python - - from neupy.layers import * - - network = layers.join( - Input((1, 28, 28)), - - Convolution((32, 3, 3)), - Relu(), - BatchNorm(), - - Convolution((48, 3, 3)), - Relu(), - BatchNorm(), - MaxPooling((2, 2)), - - Convolution((64, 3, 3)), - Relu(), - BatchNorm(), - MaxPooling((2, 2)), - - Reshape(), - - Relu(1024), - BatchNorm(), - - Softmax(10), - ) - -Does it look simple to you? Most likely not. However, this is a really simple neural network. It looks a bit complicated, because it contains a lot of simple layers that usually combined into one. For instance, non-linearity like :layer:`Relu` is usually built-in inside the :layer:`Convolution` layer. So instead of combining simple layers in one complicated, in NeuPy it's better to use subnetworks. Here is an example on how to re-write network's structure from the previous example in terms of subnetworks. - -.. code-block:: python - - network = layers.join( - Input((28, 28, 1)), - - Convolution((3, 3, 32)) >> Relu() >> BatchNorm(), - Convolution((3, 3, 48)) >> Relu() >> BatchNorm(), - MaxPooling((2, 2)), - - Convolution((3, 3, 64)) >> Relu() >> BatchNorm(), - MaxPooling((2, 2)), - - Reshape(), - - Relu(1024) >> BatchNorm(), - Softmax(10), - ) - -As you can see, we use an ability to organize sequence of simple layer in one small network. Each subnetwork defines a sequence of simple operations. You can think about subnetworks as a simple way to define more complicated layers. But instead of creating redundant classes or functions, that define complex layers, we can define everything in place. In addition, it improves the readability, because now everybody can see order of these simple operations inside the subnetwork. - -Layer definitions -================= - -It's common that different papers might have different configurations for some layers, but they will refer to it in the same way. For example, saying that network uses convolutional layers, doesn't tell us much about their configurations, since convolutional layer might have some paddings or initialization for weights might be different. In order to solve this problem, NeuPy allows to customize layer's definition. - -.. code-block:: python - - from neupy import init - from neupy.layers import * - - Conv = Convolution.define( - padding='SAME', - weight=init.XavierNormal(), - bias=None, # no bias - ) - BN = BatchNorm.define( - epsilon=1e-7, - alpha=0.001, - ) - - network = join( - Input((32, 32, 3)), - - Conv((3, 3, 16)) >> Relu() >> BN(), - Conv((3, 3, 16)) >> Relu() >> BN(), - MaxPooling((2, 2)), - - Conv((3, 3, 64)) >> Relu() >> BN(), - Conv((3, 3, 64)) >> Relu() >> BN(), - MaxPooling((2, 2)), - - Reshape(), - Softmax(10), - ) diff --git a/site/docs/layers/layer-definitions.rst b/site/docs/layers/layer-definitions.rst new file mode 100644 index 00000000..6cd27c7b --- /dev/null +++ b/site/docs/layers/layer-definitions.rst @@ -0,0 +1,34 @@ +Layer definitions +================= + +It's common that different papers might have different configurations for some layers, but they will refer to it in the same way. For example, saying that network uses convolutional layers, doesn't tell us much about their configurations, since convolutional layer might have some paddings or initialization for weights might be different. In order to solve this problem, NeuPy allows to customize layer's definition. + +.. code-block:: python + + from neupy import init + from neupy.layers import * + + Conv = Convolution.define( + padding='SAME', + weight=init.XavierNormal(), + bias=None, # no bias + ) + BN = BatchNorm.define( + epsilon=1e-7, + alpha=0.001, + ) + + network = join( + Input((32, 32, 3)), + + Conv((3, 3, 16)) >> Relu() >> BN(), + Conv((3, 3, 16)) >> Relu() >> BN(), + MaxPooling((2, 2)), + + Conv((3, 3, 64)) >> Relu() >> BN(), + Conv((3, 3, 64)) >> Relu() >> BN(), + MaxPooling((2, 2)), + + Reshape(), + Softmax(10), + ) diff --git a/site/docs/layers/repeat-blocks.rst b/site/docs/layers/repeat-blocks.rst new file mode 100644 index 00000000..50bc353a --- /dev/null +++ b/site/docs/layers/repeat-blocks.rst @@ -0,0 +1,124 @@ +Replicate layers and networks +============================= + +Copy layers/networks +-------------------- + +Any layer or network could be copied. For example, we can create layer and copy it as many times as we want. + +.. code-block:: python + + >>> from neupy.layers import * + >>> layer = layers.Relu(10) + >>> layer + Relu(10, alpha=0, weight=HeNormal(gain=2), bias=Constant(0), name='relu-1') + >>> + >>> import copy + >>> copy.copy(layer) + Relu(10, alpha=0, weight=HeNormal(gain=2), bias=Constant(0), name='relu-2') + >>> + >>> copy.copy(layer) + Relu(10, alpha=0, weight=HeNormal(gain=2), bias=Constant(0), name='relu-3') + +Each new copy gets the same set of parameters. Also, notice that each new copy get new name. It works because we +didn't specify exact name of the layer. + +.. code-block:: python + + >>> layer = Relu(10, name='relu-layer') + >>> layer + Relu(10, alpha=0, weight=HeNormal(gain=2), bias=Constant(0), name='relu-layer') + >>> + >>> copy.copy(layer) + Relu(10, alpha=0, weight=HeNormal(gain=2), bias=Constant(0), name='relu-layer') + +In order to create layers with custom name, we can specify name as a string that could be formatted with a unique +index value. + +.. code-block:: python + + >>> layer = Relu(10, name='relu-layer-{}') + >>> layer + Relu(10, alpha=0, weight=HeNormal(gain=2), bias=Constant(0), name='relu-layer-1') + >>> + >>> copy.copy(layer) + Relu(10, alpha=0, weight=HeNormal(gain=2), bias=Constant(0), name='relu-layer-2') + +All the examples above also work for the networks + +.. code-block:: python + + >>> block = Conv((3, 3, 32)) >> Relu() >> BN() + >>> block + -> [... 3 layers ...] -> (?, ?, ?, 32) + >>> + >>> copy.copy(block) + -> [... 3 layers ...] -> (?, ?, ?, 32) + +Repeat the same layer/network multiple times +-------------------------------------------- + +Certain neural network architectures might have single layer or group of layers repeated multiple times in a sequence. For example: + +.. code-block:: python + + >>> from neupy.layers import * + >>> network = Input(20) >> Relu(10) >> Relu(10) >> Relu(10) >> Relu(10) + >>> network + (?, 20) -> [... 5 layers ...] -> (?, 10) + +In order to avoid redundant repetitions, NeuPy introduced function called ``repeat``. This function copies layer +multiple times and connects original and all copied layers into a sequence. We can rewrite previous example, using +``repeat`` function in order to get the exactly the same neural network architecture. + +.. code-block:: python + + >>> network = Input(20) >> repeat(Relu(10), n=4) + >>> network + (?, 20) -> [... 5 layers ...] -> (?, 10) + +And the same function will work if applied to the network. + +.. code-block:: python + + >>> block = Conv((3, 3, 32)) >> Relu() >> BN() + >>> block + -> [... 3 layers ...] -> (?, ?, ?, 32) + >>> + >>> repeat(block, n=5) + -> [... 15 layers ...] -> (?, ?, ?, 32) + +It's important to remember that input shape of the layer/network should be compatible with it's output shape. +Otherwise exception will be triggered. + +Caveats +------- + +Copying and repetition make more sense when layer hasn't been initialized yet. Let's check the following example: + +.. code-block:: python + + >>> from neupy.layers import * + >>> layer = layers.Relu(10) + >>> layer + Relu(10, alpha=0, weight=HeNormal(gain=2), bias=Constant(0), name='relu-1') + +We can see that ``weight`` and ``bias`` hasn't been generated yet. We can add layer to the network and create variables for it. + +.. code-block:: python + + >>> network = Input(20) >> layer + >>> network.create_variables() + >>> layer + Relu(10, alpha=0, weight=, bias=, name='relu-1') + +We can see that now each parameter of the layer has it's own variable. If we try to copy layer with initialized +variables that's what we will get. + +.. code-block:: python + + >>> copy.copy(layer) + Relu(10, alpha=0, weight=, bias=, name='relu-2') + +Now each parameter has it's value specified as an array, which is just a copy of the value stored in the original +variable. For this layer, variables hasn't been created yet, since it's not a part of any network. \ No newline at end of file diff --git a/site/docs/layers/subnetworks.rst b/site/docs/layers/subnetworks.rst new file mode 100644 index 00000000..ab14c1ab --- /dev/null +++ b/site/docs/layers/subnetworks.rst @@ -0,0 +1,57 @@ +.. _subnetworks: + +Subnetworks +=========== + +**Subnetworks** is a method that improves readability of the neural network architecture. Instead of explaining, it's much easier to show the main advantage of this technique. Here is an example of the simple convolutional network. + +.. code-block:: python + + from neupy.layers import * + + network = layers.join( + Input((1, 28, 28)), + + Convolution((32, 3, 3)), + Relu(), + BatchNorm(), + + Convolution((48, 3, 3)), + Relu(), + BatchNorm(), + MaxPooling((2, 2)), + + Convolution((64, 3, 3)), + Relu(), + BatchNorm(), + MaxPooling((2, 2)), + + Reshape(), + + Relu(1024), + BatchNorm(), + + Softmax(10), + ) + +Does it look simple to you? Most likely not. However, this is a really simple neural network. It looks a bit complicated, because it contains a lot of simple layers that usually combined into one. For instance, non-linearity like :layer:`Relu` is usually built-in inside the :layer:`Convolution` layer. So instead of combining simple layers in one complicated, in NeuPy it's better to use subnetworks. Here is an example on how to re-write network's structure from the previous example in terms of subnetworks. + +.. code-block:: python + + network = layers.join( + Input((28, 28, 1)), + + Convolution((3, 3, 32)) >> Relu() >> BatchNorm(), + Convolution((3, 3, 48)) >> Relu() >> BatchNorm(), + MaxPooling((2, 2)), + + Convolution((3, 3, 64)) >> Relu() >> BatchNorm(), + MaxPooling((2, 2)), + + Reshape(), + + Relu(1024) >> BatchNorm(), + Softmax(10), + ) + +As you can see, we use an ability to organize sequence of simple layer in one small network. Each subnetwork defines a sequence of simple operations. You can think about subnetworks as a simple way to define more complicated layers. But instead of creating redundant classes or functions, that define complex layers, we can define everything in place. In addition, it improves the readability, because now everybody can see order of these simple operations inside the subnetwork. diff --git a/site/docs/layers/syntax-sugar.rst b/site/docs/layers/syntax-sugar.rst new file mode 100644 index 00000000..f5ee874b --- /dev/null +++ b/site/docs/layers/syntax-sugar.rst @@ -0,0 +1,10 @@ +Syntax sugar +============ + +.. toctree:: + :glob: + :maxdepth: 2 + + subnetworks + layer-definitions + repeat-blocks \ No newline at end of file diff --git a/site/docs/layers/with-tensorflow.rst b/site/docs/layers/with-tensorflow.rst index df9fd969..72825298 100644 --- a/site/docs/layers/with-tensorflow.rst +++ b/site/docs/layers/with-tensorflow.rst @@ -1,3 +1,5 @@ +.. _with-tensorflow: + Mixing NeuPy with Tensorflow ============================ diff --git a/site/pages/cheatsheet.rst b/site/pages/cheatsheet.rst index 1867b49d..bfee62ad 100644 --- a/site/pages/cheatsheet.rst +++ b/site/pages/cheatsheet.rst @@ -163,7 +163,7 @@ Layers with activation function neupy.layers.LeakyRelu neupy.layers.Elu neupy.layers.PRelu - neupy.layers.Softplus + neupy.layers.Sofplus neupy.layers.Softmax Convolutional layers diff --git a/site/pages/documentation.rst b/site/pages/documentation.rst index ee900be6..b6597ec9 100644 --- a/site/pages/documentation.rst +++ b/site/pages/documentation.rst @@ -19,7 +19,9 @@ This part of the documentation helps to understand how to combine layers into ne :maxdepth: 2 ../docs/layers + ../docs/layers/syntax-sugar ../docs/layers/surgery + ../docs/layers/with-tensorflow Training algorithms ------------------- diff --git a/site/pages/home.rst b/site/pages/home.rst index 4de55202..46a21041 100644 --- a/site/pages/home.rst +++ b/site/pages/home.rst @@ -4,7 +4,7 @@ NeuPy NeuPy is a python library for prototyping and building neural networks. NeuPy uses Tensorflow as a computational backend for deep learning models. .. image:: ../_static/img/mnist-code-sample-home.png - :width: 100% + :width: 70% :align: center User Guide diff --git a/site/search-index/build.py b/site/search-index/build.py index 067f0108..5a05aace 100644 --- a/site/search-index/build.py +++ b/site/search-index/build.py @@ -176,7 +176,7 @@ def collect_documents(directory): filename = os.path.basename(filepath) if ignore_link(current_page_url): - logging.debug('Skip "%s", bacause file is defined in the ' + logging.debug('Skip "%s", because file is defined in the ' 'ignore list', filename) continue