diff --git a/nn/neural/Cargo.toml b/.artifacts/archive/neural/Cargo.toml similarity index 100% rename from nn/neural/Cargo.toml rename to .artifacts/archive/neural/Cargo.toml diff --git a/nn/neural/src/errors/error.rs b/.artifacts/archive/neural/src/errors/error.rs similarity index 100% rename from nn/neural/src/errors/error.rs rename to .artifacts/archive/neural/src/errors/error.rs diff --git a/nn/neural/src/errors/mod.rs b/.artifacts/archive/neural/src/errors/mod.rs similarity index 100% rename from nn/neural/src/errors/mod.rs rename to .artifacts/archive/neural/src/errors/mod.rs diff --git a/nn/neural/src/exp/layers/config.rs b/.artifacts/archive/neural/src/exp/layers/config.rs similarity index 100% rename from nn/neural/src/exp/layers/config.rs rename to .artifacts/archive/neural/src/exp/layers/config.rs diff --git a/nn/neural/src/exp/layers/mod.rs b/.artifacts/archive/neural/src/exp/layers/mod.rs similarity index 100% rename from nn/neural/src/exp/layers/mod.rs rename to .artifacts/archive/neural/src/exp/layers/mod.rs diff --git a/nn/neural/src/exp/layers/sublayer.rs b/.artifacts/archive/neural/src/exp/layers/sublayer.rs similarity index 100% rename from nn/neural/src/exp/layers/sublayer.rs rename to .artifacts/archive/neural/src/exp/layers/sublayer.rs diff --git a/nn/neural/src/exp/layers/wrapper.rs b/.artifacts/archive/neural/src/exp/layers/wrapper.rs similarity index 100% rename from nn/neural/src/exp/layers/wrapper.rs rename to .artifacts/archive/neural/src/exp/layers/wrapper.rs diff --git a/nn/neural/src/exp/mod.rs b/.artifacts/archive/neural/src/exp/mod.rs similarity index 100% rename from nn/neural/src/exp/mod.rs rename to .artifacts/archive/neural/src/exp/mod.rs diff --git a/nn/neural/src/exp/models/mod.rs b/.artifacts/archive/neural/src/exp/models/mod.rs similarity index 100% rename from nn/neural/src/exp/models/mod.rs rename to .artifacts/archive/neural/src/exp/models/mod.rs diff --git a/nn/neural/src/exp/models/modules.rs b/.artifacts/archive/neural/src/exp/models/modules.rs similarity index 100% rename from nn/neural/src/exp/models/modules.rs rename to .artifacts/archive/neural/src/exp/models/modules.rs diff --git a/nn/neural/src/exp/models/store.rs b/.artifacts/archive/neural/src/exp/models/store.rs similarity index 100% rename from nn/neural/src/exp/models/store.rs rename to .artifacts/archive/neural/src/exp/models/store.rs diff --git a/nn/neural/src/func/activate/activator.rs b/.artifacts/archive/neural/src/func/activate/activator.rs similarity index 100% rename from nn/neural/src/func/activate/activator.rs rename to .artifacts/archive/neural/src/func/activate/activator.rs diff --git a/nn/neural/src/func/activate/binary.rs b/.artifacts/archive/neural/src/func/activate/binary.rs similarity index 100% rename from nn/neural/src/func/activate/binary.rs rename to .artifacts/archive/neural/src/func/activate/binary.rs diff --git a/nn/neural/src/func/activate/linear.rs b/.artifacts/archive/neural/src/func/activate/linear.rs similarity index 100% rename from nn/neural/src/func/activate/linear.rs rename to .artifacts/archive/neural/src/func/activate/linear.rs diff --git a/nn/neural/src/func/activate/mod.rs b/.artifacts/archive/neural/src/func/activate/mod.rs similarity index 100% rename from nn/neural/src/func/activate/mod.rs rename to .artifacts/archive/neural/src/func/activate/mod.rs diff --git a/nn/neural/src/func/activate/nl/mod.rs b/.artifacts/archive/neural/src/func/activate/nl/mod.rs similarity index 100% rename from nn/neural/src/func/activate/nl/mod.rs rename to .artifacts/archive/neural/src/func/activate/nl/mod.rs diff --git a/nn/neural/src/func/activate/nl/relu.rs b/.artifacts/archive/neural/src/func/activate/nl/relu.rs similarity index 100% rename from nn/neural/src/func/activate/nl/relu.rs rename to .artifacts/archive/neural/src/func/activate/nl/relu.rs diff --git a/nn/neural/src/func/activate/nl/sigmoid.rs b/.artifacts/archive/neural/src/func/activate/nl/sigmoid.rs similarity index 100% rename from nn/neural/src/func/activate/nl/sigmoid.rs rename to .artifacts/archive/neural/src/func/activate/nl/sigmoid.rs diff --git a/nn/neural/src/func/activate/nl/softmax.rs b/.artifacts/archive/neural/src/func/activate/nl/softmax.rs similarity index 100% rename from nn/neural/src/func/activate/nl/softmax.rs rename to .artifacts/archive/neural/src/func/activate/nl/softmax.rs diff --git a/nn/neural/src/func/activate/nl/tanh.rs b/.artifacts/archive/neural/src/func/activate/nl/tanh.rs similarity index 100% rename from nn/neural/src/func/activate/nl/tanh.rs rename to .artifacts/archive/neural/src/func/activate/nl/tanh.rs diff --git a/nn/neural/src/func/loss/kinds.rs b/.artifacts/archive/neural/src/func/loss/kinds.rs similarity index 100% rename from nn/neural/src/func/loss/kinds.rs rename to .artifacts/archive/neural/src/func/loss/kinds.rs diff --git a/nn/neural/src/func/loss/mod.rs b/.artifacts/archive/neural/src/func/loss/mod.rs similarity index 100% rename from nn/neural/src/func/loss/mod.rs rename to .artifacts/archive/neural/src/func/loss/mod.rs diff --git a/nn/neural/src/func/loss/reg/huber.rs b/.artifacts/archive/neural/src/func/loss/reg/huber.rs similarity index 100% rename from nn/neural/src/func/loss/reg/huber.rs rename to .artifacts/archive/neural/src/func/loss/reg/huber.rs diff --git a/nn/neural/src/func/loss/reg/mae.rs b/.artifacts/archive/neural/src/func/loss/reg/mae.rs similarity index 100% rename from nn/neural/src/func/loss/reg/mae.rs rename to .artifacts/archive/neural/src/func/loss/reg/mae.rs diff --git a/nn/neural/src/func/loss/reg/mod.rs b/.artifacts/archive/neural/src/func/loss/reg/mod.rs similarity index 100% rename from nn/neural/src/func/loss/reg/mod.rs rename to .artifacts/archive/neural/src/func/loss/reg/mod.rs diff --git a/nn/neural/src/func/loss/reg/mse.rs b/.artifacts/archive/neural/src/func/loss/reg/mse.rs similarity index 100% rename from nn/neural/src/func/loss/reg/mse.rs rename to .artifacts/archive/neural/src/func/loss/reg/mse.rs diff --git a/nn/neural/src/func/mod.rs b/.artifacts/archive/neural/src/func/mod.rs similarity index 100% rename from nn/neural/src/func/mod.rs rename to .artifacts/archive/neural/src/func/mod.rs diff --git a/nn/neural/src/func/rms.rs b/.artifacts/archive/neural/src/func/rms.rs similarity index 100% rename from nn/neural/src/func/rms.rs rename to .artifacts/archive/neural/src/func/rms.rs diff --git a/nn/neural/src/layers/cmp/features.rs b/.artifacts/archive/neural/src/layers/cmp/features.rs similarity index 100% rename from nn/neural/src/layers/cmp/features.rs rename to .artifacts/archive/neural/src/layers/cmp/features.rs diff --git a/nn/neural/src/layers/cmp/kinds.rs b/.artifacts/archive/neural/src/layers/cmp/kinds.rs similarity index 100% rename from nn/neural/src/layers/cmp/kinds.rs rename to .artifacts/archive/neural/src/layers/cmp/kinds.rs diff --git a/nn/neural/src/layers/cmp/mod.rs b/.artifacts/archive/neural/src/layers/cmp/mod.rs similarity index 100% rename from nn/neural/src/layers/cmp/mod.rs rename to .artifacts/archive/neural/src/layers/cmp/mod.rs diff --git a/nn/neural/src/layers/layer.rs b/.artifacts/archive/neural/src/layers/layer.rs similarity index 100% rename from nn/neural/src/layers/layer.rs rename to .artifacts/archive/neural/src/layers/layer.rs diff --git a/nn/neural/src/layers/mod.rs b/.artifacts/archive/neural/src/layers/mod.rs similarity index 100% rename from nn/neural/src/layers/mod.rs rename to .artifacts/archive/neural/src/layers/mod.rs diff --git a/nn/neural/src/layers/params.rs b/.artifacts/archive/neural/src/layers/params.rs similarity index 100% rename from nn/neural/src/layers/params.rs rename to .artifacts/archive/neural/src/layers/params.rs diff --git a/nn/neural/src/layers/seq/mod.rs b/.artifacts/archive/neural/src/layers/seq/mod.rs similarity index 100% rename from nn/neural/src/layers/seq/mod.rs rename to .artifacts/archive/neural/src/layers/seq/mod.rs diff --git a/nn/neural/src/layers/seq/sequential.rs b/.artifacts/archive/neural/src/layers/seq/sequential.rs similarity index 100% rename from nn/neural/src/layers/seq/sequential.rs rename to .artifacts/archive/neural/src/layers/seq/sequential.rs diff --git a/nn/neural/src/lib.rs b/.artifacts/archive/neural/src/lib.rs similarity index 100% rename from nn/neural/src/lib.rs rename to .artifacts/archive/neural/src/lib.rs diff --git a/nn/neural/src/models/config.rs b/.artifacts/archive/neural/src/models/config.rs similarity index 100% rename from nn/neural/src/models/config.rs rename to .artifacts/archive/neural/src/models/config.rs diff --git a/nn/neural/src/models/mod.rs b/.artifacts/archive/neural/src/models/mod.rs similarity index 100% rename from nn/neural/src/models/mod.rs rename to .artifacts/archive/neural/src/models/mod.rs diff --git a/nn/neural/src/models/model.rs b/.artifacts/archive/neural/src/models/model.rs similarity index 100% rename from nn/neural/src/models/model.rs rename to .artifacts/archive/neural/src/models/model.rs diff --git a/nn/neural/src/models/modes.rs b/.artifacts/archive/neural/src/models/modes.rs similarity index 100% rename from nn/neural/src/models/modes.rs rename to .artifacts/archive/neural/src/models/modes.rs diff --git a/nn/neural/src/models/params.rs b/.artifacts/archive/neural/src/models/params.rs similarity index 100% rename from nn/neural/src/models/params.rs rename to .artifacts/archive/neural/src/models/params.rs diff --git a/nn/neural/src/neurons/mod.rs b/.artifacts/archive/neural/src/neurons/mod.rs similarity index 100% rename from nn/neural/src/neurons/mod.rs rename to .artifacts/archive/neural/src/neurons/mod.rs diff --git a/nn/neural/src/neurons/node.rs b/.artifacts/archive/neural/src/neurons/node.rs similarity index 100% rename from nn/neural/src/neurons/node.rs rename to .artifacts/archive/neural/src/neurons/node.rs diff --git a/nn/neural/src/neurons/perceptron.rs b/.artifacts/archive/neural/src/neurons/perceptron.rs similarity index 100% rename from nn/neural/src/neurons/perceptron.rs rename to .artifacts/archive/neural/src/neurons/perceptron.rs diff --git a/nn/neural/src/neurons/synapse.rs b/.artifacts/archive/neural/src/neurons/synapse.rs similarity index 100% rename from nn/neural/src/neurons/synapse.rs rename to .artifacts/archive/neural/src/neurons/synapse.rs diff --git a/nn/neural/src/nn/gnn/mod.rs b/.artifacts/archive/neural/src/nn/gnn/mod.rs similarity index 100% rename from nn/neural/src/nn/gnn/mod.rs rename to .artifacts/archive/neural/src/nn/gnn/mod.rs diff --git a/nn/neural/src/nn/gnn/model.rs b/.artifacts/archive/neural/src/nn/gnn/model.rs similarity index 100% rename from nn/neural/src/nn/gnn/model.rs rename to .artifacts/archive/neural/src/nn/gnn/model.rs diff --git a/nn/neural/src/nn/gnn/tasks.rs b/.artifacts/archive/neural/src/nn/gnn/tasks.rs similarity index 100% rename from nn/neural/src/nn/gnn/tasks.rs rename to .artifacts/archive/neural/src/nn/gnn/tasks.rs diff --git a/nn/neural/src/nn/kinds.rs b/.artifacts/archive/neural/src/nn/kinds.rs similarity index 100% rename from nn/neural/src/nn/kinds.rs rename to .artifacts/archive/neural/src/nn/kinds.rs diff --git a/nn/neural/src/nn/mod.rs b/.artifacts/archive/neural/src/nn/mod.rs similarity index 100% rename from nn/neural/src/nn/mod.rs rename to .artifacts/archive/neural/src/nn/mod.rs diff --git a/nn/neural/src/nn/position.rs b/.artifacts/archive/neural/src/nn/position.rs similarity index 100% rename from nn/neural/src/nn/position.rs rename to .artifacts/archive/neural/src/nn/position.rs diff --git a/nn/neural/src/ops/compile.rs b/.artifacts/archive/neural/src/ops/compile.rs similarity index 100% rename from nn/neural/src/ops/compile.rs rename to .artifacts/archive/neural/src/ops/compile.rs diff --git a/nn/neural/src/ops/dropout.rs b/.artifacts/archive/neural/src/ops/dropout.rs similarity index 100% rename from nn/neural/src/ops/dropout.rs rename to .artifacts/archive/neural/src/ops/dropout.rs diff --git a/nn/neural/src/ops/mod.rs b/.artifacts/archive/neural/src/ops/mod.rs similarity index 100% rename from nn/neural/src/ops/mod.rs rename to .artifacts/archive/neural/src/ops/mod.rs diff --git a/nn/neural/src/ops/norm.rs b/.artifacts/archive/neural/src/ops/norm.rs similarity index 100% rename from nn/neural/src/ops/norm.rs rename to .artifacts/archive/neural/src/ops/norm.rs diff --git a/nn/neural/src/ops/predict.rs b/.artifacts/archive/neural/src/ops/predict.rs similarity index 100% rename from nn/neural/src/ops/predict.rs rename to .artifacts/archive/neural/src/ops/predict.rs diff --git a/nn/neural/src/ops/train.rs b/.artifacts/archive/neural/src/ops/train.rs similarity index 100% rename from nn/neural/src/ops/train.rs rename to .artifacts/archive/neural/src/ops/train.rs diff --git a/nn/neural/src/params/iter.rs b/.artifacts/archive/neural/src/params/iter.rs similarity index 100% rename from nn/neural/src/params/iter.rs rename to .artifacts/archive/neural/src/params/iter.rs diff --git a/nn/neural/src/params/kinds.rs b/.artifacts/archive/neural/src/params/kinds.rs similarity index 100% rename from nn/neural/src/params/kinds.rs rename to .artifacts/archive/neural/src/params/kinds.rs diff --git a/nn/neural/src/params/masks/mask.rs b/.artifacts/archive/neural/src/params/masks/mask.rs similarity index 100% rename from nn/neural/src/params/masks/mask.rs rename to .artifacts/archive/neural/src/params/masks/mask.rs diff --git a/nn/neural/src/params/masks/mod.rs b/.artifacts/archive/neural/src/params/masks/mod.rs similarity index 100% rename from nn/neural/src/params/masks/mod.rs rename to .artifacts/archive/neural/src/params/masks/mod.rs diff --git a/nn/neural/src/params/mod.rs b/.artifacts/archive/neural/src/params/mod.rs similarity index 100% rename from nn/neural/src/params/mod.rs rename to .artifacts/archive/neural/src/params/mod.rs diff --git a/nn/neural/src/params/param.rs b/.artifacts/archive/neural/src/params/param.rs similarity index 100% rename from nn/neural/src/params/param.rs rename to .artifacts/archive/neural/src/params/param.rs diff --git a/nn/neural/src/params/store.rs b/.artifacts/archive/neural/src/params/store.rs similarity index 100% rename from nn/neural/src/params/store.rs rename to .artifacts/archive/neural/src/params/store.rs diff --git a/nn/neural/src/params/store/group.rs b/.artifacts/archive/neural/src/params/store/group.rs similarity index 100% rename from nn/neural/src/params/store/group.rs rename to .artifacts/archive/neural/src/params/store/group.rs diff --git a/nn/neural/src/params/variable.rs b/.artifacts/archive/neural/src/params/variable.rs similarity index 100% rename from nn/neural/src/params/variable.rs rename to .artifacts/archive/neural/src/params/variable.rs diff --git a/nn/neural/src/primitives.rs b/.artifacts/archive/neural/src/primitives.rs similarity index 100% rename from nn/neural/src/primitives.rs rename to .artifacts/archive/neural/src/primitives.rs diff --git a/nn/neural/src/specs.rs b/.artifacts/archive/neural/src/specs.rs similarity index 100% rename from nn/neural/src/specs.rs rename to .artifacts/archive/neural/src/specs.rs diff --git a/nn/neural/src/utils.rs b/.artifacts/archive/neural/src/utils.rs similarity index 100% rename from nn/neural/src/utils.rs rename to .artifacts/archive/neural/src/utils.rs diff --git a/nn/neural/tests/default.rs b/.artifacts/archive/neural/tests/default.rs similarity index 100% rename from nn/neural/tests/default.rs rename to .artifacts/archive/neural/tests/default.rs diff --git a/nn/nlp/Cargo.toml b/.artifacts/archive/nlp/Cargo.toml similarity index 100% rename from nn/nlp/Cargo.toml rename to .artifacts/archive/nlp/Cargo.toml diff --git a/nn/nlp/src/embed/context/mod.rs b/.artifacts/archive/nlp/src/embed/context/mod.rs similarity index 100% rename from nn/nlp/src/embed/context/mod.rs rename to .artifacts/archive/nlp/src/embed/context/mod.rs diff --git a/nn/nlp/src/embed/embedding.rs b/.artifacts/archive/nlp/src/embed/embedding.rs similarity index 100% rename from nn/nlp/src/embed/embedding.rs rename to .artifacts/archive/nlp/src/embed/embedding.rs diff --git a/nn/nlp/src/embed/mod.rs b/.artifacts/archive/nlp/src/embed/mod.rs similarity index 100% rename from nn/nlp/src/embed/mod.rs rename to .artifacts/archive/nlp/src/embed/mod.rs diff --git a/nn/nlp/src/embed/words/mod.rs b/.artifacts/archive/nlp/src/embed/words/mod.rs similarity index 100% rename from nn/nlp/src/embed/words/mod.rs rename to .artifacts/archive/nlp/src/embed/words/mod.rs diff --git a/nn/nlp/src/embed/words/word2vec.rs b/.artifacts/archive/nlp/src/embed/words/word2vec.rs similarity index 100% rename from nn/nlp/src/embed/words/word2vec.rs rename to .artifacts/archive/nlp/src/embed/words/word2vec.rs diff --git a/nn/nlp/src/encode/mod.rs b/.artifacts/archive/nlp/src/encode/mod.rs similarity index 100% rename from nn/nlp/src/encode/mod.rs rename to .artifacts/archive/nlp/src/encode/mod.rs diff --git a/nn/nlp/src/encode/positional.rs b/.artifacts/archive/nlp/src/encode/positional.rs similarity index 100% rename from nn/nlp/src/encode/positional.rs rename to .artifacts/archive/nlp/src/encode/positional.rs diff --git a/nn/nlp/src/lib.rs b/.artifacts/archive/nlp/src/lib.rs similarity index 100% rename from nn/nlp/src/lib.rs rename to .artifacts/archive/nlp/src/lib.rs diff --git a/nn/nlp/src/primitives.rs b/.artifacts/archive/nlp/src/primitives.rs similarity index 100% rename from nn/nlp/src/primitives.rs rename to .artifacts/archive/nlp/src/primitives.rs diff --git a/nn/nlp/src/specs.rs b/.artifacts/archive/nlp/src/specs.rs similarity index 100% rename from nn/nlp/src/specs.rs rename to .artifacts/archive/nlp/src/specs.rs diff --git a/nn/nlp/src/utils.rs b/.artifacts/archive/nlp/src/utils.rs similarity index 100% rename from nn/nlp/src/utils.rs rename to .artifacts/archive/nlp/src/utils.rs diff --git a/nn/nlp/tests/default.rs b/.artifacts/archive/nlp/tests/default.rs similarity index 100% rename from nn/nlp/tests/default.rs rename to .artifacts/archive/nlp/tests/default.rs diff --git a/nn/nn/Cargo.toml b/.artifacts/archive/nn/Cargo.toml similarity index 92% rename from nn/nn/Cargo.toml rename to .artifacts/archive/nn/Cargo.toml index 16d02b31..f6daa451 100644 --- a/nn/nn/Cargo.toml +++ b/.artifacts/archive/nn/Cargo.toml @@ -52,7 +52,7 @@ blas = [ "concision-nlp/blas", "concision-optim/blas", "concision-s4/blas", - "concision-transformers/blas", + "transformers?/blas", ] intel-mkl-system = [ @@ -104,13 +104,13 @@ concision-neural = { path = "../neural", version = "0.1.12" } concision-nlp = { optional = true, path = "../nlp", version = "0.1.12" } concision-optim = { optional = true, path = "../optim", version = "0.1.12" } concision-s4 = { optional = true, path = "../s4", version = "0.1.12" } -concision-transformers = { optional = true, path = "../transformers", version = "0.1.12" } +transformers = { optional = true, path = "../transformers", version = "0.1.12" } [dev-dependencies] anyhow = "1" approx = "0.5" -concision = { path = "../../concision" } +concision = { path = "../../../concision" } ndarray = { features = ["approx-0_5", "serde-1"], version = "0.15" } [package.metadata.docs.rs] diff --git a/nn/nn/examples/basic.rs b/.artifacts/archive/nn/examples/basic.rs similarity index 100% rename from nn/nn/examples/basic.rs rename to .artifacts/archive/nn/examples/basic.rs diff --git a/nn/nn/src/lib.rs b/.artifacts/archive/nn/src/lib.rs similarity index 94% rename from nn/nn/src/lib.rs rename to .artifacts/archive/nn/src/lib.rs index 54269258..5f29aacb 100644 --- a/nn/nn/src/lib.rs +++ b/.artifacts/archive/nn/src/lib.rs @@ -16,7 +16,7 @@ pub use concision_optim as optim; #[cfg(feature = "s4")] pub use concision_s4 as s4; #[cfg(feature = "transformers")] -pub use concision_transformers as transformers; +pub use transformers as transformers; pub mod prelude { pub use concision_neural::prelude::*; diff --git a/nn/nn/tests/default.rs b/.artifacts/archive/nn/tests/default.rs similarity index 100% rename from nn/nn/tests/default.rs rename to .artifacts/archive/nn/tests/default.rs diff --git a/nn/optim/Cargo.toml b/.artifacts/archive/optim/Cargo.toml similarity index 100% rename from nn/optim/Cargo.toml rename to .artifacts/archive/optim/Cargo.toml diff --git a/nn/optim/examples/norm.rs b/.artifacts/archive/optim/examples/norm.rs similarity index 100% rename from nn/optim/examples/norm.rs rename to .artifacts/archive/optim/examples/norm.rs diff --git a/nn/optim/examples/sgd.rs b/.artifacts/archive/optim/examples/sgd.rs similarity index 100% rename from nn/optim/examples/sgd.rs rename to .artifacts/archive/optim/examples/sgd.rs diff --git a/nn/optim/src/grad/adam.rs b/.artifacts/archive/optim/src/grad/adam.rs similarity index 100% rename from nn/optim/src/grad/adam.rs rename to .artifacts/archive/optim/src/grad/adam.rs diff --git a/nn/optim/src/grad/descent.rs b/.artifacts/archive/optim/src/grad/descent.rs similarity index 100% rename from nn/optim/src/grad/descent.rs rename to .artifacts/archive/optim/src/grad/descent.rs diff --git a/nn/optim/src/grad/gradient.rs b/.artifacts/archive/optim/src/grad/gradient.rs similarity index 100% rename from nn/optim/src/grad/gradient.rs rename to .artifacts/archive/optim/src/grad/gradient.rs diff --git a/nn/optim/src/grad/mod.rs b/.artifacts/archive/optim/src/grad/mod.rs similarity index 100% rename from nn/optim/src/grad/mod.rs rename to .artifacts/archive/optim/src/grad/mod.rs diff --git a/nn/optim/src/grad/modes.rs b/.artifacts/archive/optim/src/grad/modes.rs similarity index 100% rename from nn/optim/src/grad/modes.rs rename to .artifacts/archive/optim/src/grad/modes.rs diff --git a/nn/optim/src/grad/sgd.rs b/.artifacts/archive/optim/src/grad/sgd.rs similarity index 100% rename from nn/optim/src/grad/sgd.rs rename to .artifacts/archive/optim/src/grad/sgd.rs diff --git a/nn/optim/src/lib.rs b/.artifacts/archive/optim/src/lib.rs similarity index 100% rename from nn/optim/src/lib.rs rename to .artifacts/archive/optim/src/lib.rs diff --git a/nn/optim/src/norm/kinds.rs b/.artifacts/archive/optim/src/norm/kinds.rs similarity index 100% rename from nn/optim/src/norm/kinds.rs rename to .artifacts/archive/optim/src/norm/kinds.rs diff --git a/nn/optim/src/norm/mod.rs b/.artifacts/archive/optim/src/norm/mod.rs similarity index 100% rename from nn/optim/src/norm/mod.rs rename to .artifacts/archive/optim/src/norm/mod.rs diff --git a/nn/optim/src/norm/normalizer.rs b/.artifacts/archive/optim/src/norm/normalizer.rs similarity index 100% rename from nn/optim/src/norm/normalizer.rs rename to .artifacts/archive/optim/src/norm/normalizer.rs diff --git a/nn/optim/src/optimizer.rs b/.artifacts/archive/optim/src/optimizer.rs similarity index 100% rename from nn/optim/src/optimizer.rs rename to .artifacts/archive/optim/src/optimizer.rs diff --git a/nn/optim/src/params/mod.rs b/.artifacts/archive/optim/src/params/mod.rs similarity index 100% rename from nn/optim/src/params/mod.rs rename to .artifacts/archive/optim/src/params/mod.rs diff --git a/nn/optim/src/primitives.rs b/.artifacts/archive/optim/src/primitives.rs similarity index 100% rename from nn/optim/src/primitives.rs rename to .artifacts/archive/optim/src/primitives.rs diff --git a/nn/optim/src/specs.rs b/.artifacts/archive/optim/src/specs.rs similarity index 100% rename from nn/optim/src/specs.rs rename to .artifacts/archive/optim/src/specs.rs diff --git a/nn/optim/src/utils.rs b/.artifacts/archive/optim/src/utils.rs similarity index 100% rename from nn/optim/src/utils.rs rename to .artifacts/archive/optim/src/utils.rs diff --git a/nn/optim/tests/default.rs b/.artifacts/archive/optim/tests/default.rs similarity index 100% rename from nn/optim/tests/default.rs rename to .artifacts/archive/optim/tests/default.rs diff --git a/nn/s4/Cargo.toml b/.artifacts/archive/s4/Cargo.toml similarity index 100% rename from nn/s4/Cargo.toml rename to .artifacts/archive/s4/Cargo.toml diff --git a/nn/s4/examples/s4.rs b/.artifacts/archive/s4/examples/s4.rs similarity index 100% rename from nn/s4/examples/s4.rs rename to .artifacts/archive/s4/examples/s4.rs diff --git a/nn/s4/examples/ssm.rs b/.artifacts/archive/s4/examples/ssm.rs similarity index 100% rename from nn/s4/examples/ssm.rs rename to .artifacts/archive/s4/examples/ssm.rs diff --git a/nn/s4/src/cmp/cache.rs b/.artifacts/archive/s4/src/cmp/cache.rs similarity index 100% rename from nn/s4/src/cmp/cache.rs rename to .artifacts/archive/s4/src/cmp/cache.rs diff --git a/nn/s4/src/cmp/kernel.rs b/.artifacts/archive/s4/src/cmp/kernel.rs similarity index 100% rename from nn/s4/src/cmp/kernel.rs rename to .artifacts/archive/s4/src/cmp/kernel.rs diff --git a/nn/s4/src/cmp/mod.rs b/.artifacts/archive/s4/src/cmp/mod.rs similarity index 100% rename from nn/s4/src/cmp/mod.rs rename to .artifacts/archive/s4/src/cmp/mod.rs diff --git a/nn/s4/src/hippo/dplr.rs b/.artifacts/archive/s4/src/hippo/dplr.rs similarity index 100% rename from nn/s4/src/hippo/dplr.rs rename to .artifacts/archive/s4/src/hippo/dplr.rs diff --git a/nn/s4/src/hippo/hippo.rs b/.artifacts/archive/s4/src/hippo/hippo.rs similarity index 100% rename from nn/s4/src/hippo/hippo.rs rename to .artifacts/archive/s4/src/hippo/hippo.rs diff --git a/nn/s4/src/hippo/kinds.rs b/.artifacts/archive/s4/src/hippo/kinds.rs similarity index 100% rename from nn/s4/src/hippo/kinds.rs rename to .artifacts/archive/s4/src/hippo/kinds.rs diff --git a/nn/s4/src/hippo/mod.rs b/.artifacts/archive/s4/src/hippo/mod.rs similarity index 100% rename from nn/s4/src/hippo/mod.rs rename to .artifacts/archive/s4/src/hippo/mod.rs diff --git a/nn/s4/src/hippo/nplr.rs b/.artifacts/archive/s4/src/hippo/nplr.rs similarity index 100% rename from nn/s4/src/hippo/nplr.rs rename to .artifacts/archive/s4/src/hippo/nplr.rs diff --git a/nn/s4/src/lib.rs b/.artifacts/archive/s4/src/lib.rs similarity index 100% rename from nn/s4/src/lib.rs rename to .artifacts/archive/s4/src/lib.rs diff --git a/nn/s4/src/model/config.rs b/.artifacts/archive/s4/src/model/config.rs similarity index 100% rename from nn/s4/src/model/config.rs rename to .artifacts/archive/s4/src/model/config.rs diff --git a/nn/s4/src/model/mod.rs b/.artifacts/archive/s4/src/model/mod.rs similarity index 100% rename from nn/s4/src/model/mod.rs rename to .artifacts/archive/s4/src/model/mod.rs diff --git a/nn/s4/src/model/module.rs b/.artifacts/archive/s4/src/model/module.rs similarity index 100% rename from nn/s4/src/model/module.rs rename to .artifacts/archive/s4/src/model/module.rs diff --git a/nn/s4/src/model/params.rs b/.artifacts/archive/s4/src/model/params.rs similarity index 100% rename from nn/s4/src/model/params.rs rename to .artifacts/archive/s4/src/model/params.rs diff --git a/nn/s4/src/ops/convolve.rs b/.artifacts/archive/s4/src/ops/convolve.rs similarity index 100% rename from nn/s4/src/ops/convolve.rs rename to .artifacts/archive/s4/src/ops/convolve.rs diff --git a/nn/s4/src/ops/discretize.rs b/.artifacts/archive/s4/src/ops/discretize.rs similarity index 100% rename from nn/s4/src/ops/discretize.rs rename to .artifacts/archive/s4/src/ops/discretize.rs diff --git a/nn/s4/src/ops/gen.rs b/.artifacts/archive/s4/src/ops/gen.rs similarity index 100% rename from nn/s4/src/ops/gen.rs rename to .artifacts/archive/s4/src/ops/gen.rs diff --git a/nn/s4/src/ops/mod.rs b/.artifacts/archive/s4/src/ops/mod.rs similarity index 100% rename from nn/s4/src/ops/mod.rs rename to .artifacts/archive/s4/src/ops/mod.rs diff --git a/nn/s4/src/ops/scan.rs b/.artifacts/archive/s4/src/ops/scan.rs similarity index 100% rename from nn/s4/src/ops/scan.rs rename to .artifacts/archive/s4/src/ops/scan.rs diff --git a/nn/s4/src/params/dplr.rs b/.artifacts/archive/s4/src/params/dplr.rs similarity index 100% rename from nn/s4/src/params/dplr.rs rename to .artifacts/archive/s4/src/params/dplr.rs diff --git a/nn/s4/src/params/kinds.rs b/.artifacts/archive/s4/src/params/kinds.rs similarity index 100% rename from nn/s4/src/params/kinds.rs rename to .artifacts/archive/s4/src/params/kinds.rs diff --git a/nn/s4/src/params/mod.rs b/.artifacts/archive/s4/src/params/mod.rs similarity index 100% rename from nn/s4/src/params/mod.rs rename to .artifacts/archive/s4/src/params/mod.rs diff --git a/nn/s4/src/params/store.rs b/.artifacts/archive/s4/src/params/store.rs similarity index 100% rename from nn/s4/src/params/store.rs rename to .artifacts/archive/s4/src/params/store.rs diff --git a/nn/s4/src/primitives.rs b/.artifacts/archive/s4/src/primitives.rs similarity index 100% rename from nn/s4/src/primitives.rs rename to .artifacts/archive/s4/src/primitives.rs diff --git a/nn/s4/src/specs.rs b/.artifacts/archive/s4/src/specs.rs similarity index 100% rename from nn/s4/src/specs.rs rename to .artifacts/archive/s4/src/specs.rs diff --git a/nn/s4/src/ssm/config.rs b/.artifacts/archive/s4/src/ssm/config.rs similarity index 100% rename from nn/s4/src/ssm/config.rs rename to .artifacts/archive/s4/src/ssm/config.rs diff --git a/nn/s4/src/ssm/mod.rs b/.artifacts/archive/s4/src/ssm/mod.rs similarity index 100% rename from nn/s4/src/ssm/mod.rs rename to .artifacts/archive/s4/src/ssm/mod.rs diff --git a/nn/s4/src/ssm/model.rs b/.artifacts/archive/s4/src/ssm/model.rs similarity index 100% rename from nn/s4/src/ssm/model.rs rename to .artifacts/archive/s4/src/ssm/model.rs diff --git a/nn/s4/src/utils.rs b/.artifacts/archive/s4/src/utils.rs similarity index 100% rename from nn/s4/src/utils.rs rename to .artifacts/archive/s4/src/utils.rs diff --git a/nn/s4/tests/blas.rs b/.artifacts/archive/s4/tests/blas.rs similarity index 100% rename from nn/s4/tests/blas.rs rename to .artifacts/archive/s4/tests/blas.rs diff --git a/nn/s4/tests/conversion.rs b/.artifacts/archive/s4/tests/conversion.rs similarity index 100% rename from nn/s4/tests/conversion.rs rename to .artifacts/archive/s4/tests/conversion.rs diff --git a/nn/s4/tests/default.rs b/.artifacts/archive/s4/tests/default.rs similarity index 100% rename from nn/s4/tests/default.rs rename to .artifacts/archive/s4/tests/default.rs diff --git a/nn/s4/tests/dplr.rs b/.artifacts/archive/s4/tests/dplr.rs similarity index 100% rename from nn/s4/tests/dplr.rs rename to .artifacts/archive/s4/tests/dplr.rs diff --git a/nn/transformers/Cargo.toml b/.artifacts/archive/transformers/Cargo.toml similarity index 97% rename from nn/transformers/Cargo.toml rename to .artifacts/archive/transformers/Cargo.toml index e21b6d2f..4df37bdd 100644 --- a/nn/transformers/Cargo.toml +++ b/.artifacts/archive/transformers/Cargo.toml @@ -6,7 +6,7 @@ edition.workspace = true homepage.workspace = true keywords.workspace = true license.workspace = true -name = "concision-transformers" +name = "transformers" readme.workspace = true repository.workspace = true version.workspace = true diff --git a/nn/transformers/src/attention/head.rs b/.artifacts/archive/transformers/src/attention/head.rs similarity index 100% rename from nn/transformers/src/attention/head.rs rename to .artifacts/archive/transformers/src/attention/head.rs diff --git a/nn/transformers/src/attention/mod.rs b/.artifacts/archive/transformers/src/attention/mod.rs similarity index 100% rename from nn/transformers/src/attention/mod.rs rename to .artifacts/archive/transformers/src/attention/mod.rs diff --git a/nn/transformers/src/attention/multi/attention.rs b/.artifacts/archive/transformers/src/attention/multi/attention.rs similarity index 100% rename from nn/transformers/src/attention/multi/attention.rs rename to .artifacts/archive/transformers/src/attention/multi/attention.rs diff --git a/nn/transformers/src/attention/multi/mod.rs b/.artifacts/archive/transformers/src/attention/multi/mod.rs similarity index 100% rename from nn/transformers/src/attention/multi/mod.rs rename to .artifacts/archive/transformers/src/attention/multi/mod.rs diff --git a/nn/transformers/src/attention/multi/params.rs b/.artifacts/archive/transformers/src/attention/multi/params.rs similarity index 100% rename from nn/transformers/src/attention/multi/params.rs rename to .artifacts/archive/transformers/src/attention/multi/params.rs diff --git a/nn/transformers/src/attention/params/dim.rs b/.artifacts/archive/transformers/src/attention/params/dim.rs similarity index 100% rename from nn/transformers/src/attention/params/dim.rs rename to .artifacts/archive/transformers/src/attention/params/dim.rs diff --git a/nn/transformers/src/attention/params/hyperparams.rs b/.artifacts/archive/transformers/src/attention/params/hyperparams.rs similarity index 100% rename from nn/transformers/src/attention/params/hyperparams.rs rename to .artifacts/archive/transformers/src/attention/params/hyperparams.rs diff --git a/nn/transformers/src/attention/params/mod.rs b/.artifacts/archive/transformers/src/attention/params/mod.rs similarity index 100% rename from nn/transformers/src/attention/params/mod.rs rename to .artifacts/archive/transformers/src/attention/params/mod.rs diff --git a/nn/transformers/src/attention/params/qkv.rs b/.artifacts/archive/transformers/src/attention/params/qkv.rs similarity index 100% rename from nn/transformers/src/attention/params/qkv.rs rename to .artifacts/archive/transformers/src/attention/params/qkv.rs diff --git a/nn/transformers/src/attention/weights.rs b/.artifacts/archive/transformers/src/attention/weights.rs similarity index 100% rename from nn/transformers/src/attention/weights.rs rename to .artifacts/archive/transformers/src/attention/weights.rs diff --git a/nn/transformers/src/codec/decode/decoder.rs b/.artifacts/archive/transformers/src/codec/decode/decoder.rs similarity index 100% rename from nn/transformers/src/codec/decode/decoder.rs rename to .artifacts/archive/transformers/src/codec/decode/decoder.rs diff --git a/nn/transformers/src/codec/decode/mod.rs b/.artifacts/archive/transformers/src/codec/decode/mod.rs similarity index 100% rename from nn/transformers/src/codec/decode/mod.rs rename to .artifacts/archive/transformers/src/codec/decode/mod.rs diff --git a/nn/transformers/src/codec/decode/params.rs b/.artifacts/archive/transformers/src/codec/decode/params.rs similarity index 100% rename from nn/transformers/src/codec/decode/params.rs rename to .artifacts/archive/transformers/src/codec/decode/params.rs diff --git a/nn/transformers/src/codec/encode/encoder.rs b/.artifacts/archive/transformers/src/codec/encode/encoder.rs similarity index 100% rename from nn/transformers/src/codec/encode/encoder.rs rename to .artifacts/archive/transformers/src/codec/encode/encoder.rs diff --git a/nn/transformers/src/codec/encode/mod.rs b/.artifacts/archive/transformers/src/codec/encode/mod.rs similarity index 100% rename from nn/transformers/src/codec/encode/mod.rs rename to .artifacts/archive/transformers/src/codec/encode/mod.rs diff --git a/nn/transformers/src/codec/encode/params.rs b/.artifacts/archive/transformers/src/codec/encode/params.rs similarity index 100% rename from nn/transformers/src/codec/encode/params.rs rename to .artifacts/archive/transformers/src/codec/encode/params.rs diff --git a/nn/transformers/src/codec/encode/stack.rs b/.artifacts/archive/transformers/src/codec/encode/stack.rs similarity index 100% rename from nn/transformers/src/codec/encode/stack.rs rename to .artifacts/archive/transformers/src/codec/encode/stack.rs diff --git a/nn/transformers/src/codec/mod.rs b/.artifacts/archive/transformers/src/codec/mod.rs similarity index 100% rename from nn/transformers/src/codec/mod.rs rename to .artifacts/archive/transformers/src/codec/mod.rs diff --git a/nn/transformers/src/ffn/mod.rs b/.artifacts/archive/transformers/src/ffn/mod.rs similarity index 100% rename from nn/transformers/src/ffn/mod.rs rename to .artifacts/archive/transformers/src/ffn/mod.rs diff --git a/nn/transformers/src/ffn/network.rs b/.artifacts/archive/transformers/src/ffn/network.rs similarity index 100% rename from nn/transformers/src/ffn/network.rs rename to .artifacts/archive/transformers/src/ffn/network.rs diff --git a/nn/transformers/src/ffn/params.rs b/.artifacts/archive/transformers/src/ffn/params.rs similarity index 100% rename from nn/transformers/src/ffn/params.rs rename to .artifacts/archive/transformers/src/ffn/params.rs diff --git a/nn/transformers/src/lib.rs b/.artifacts/archive/transformers/src/lib.rs similarity index 100% rename from nn/transformers/src/lib.rs rename to .artifacts/archive/transformers/src/lib.rs diff --git a/nn/transformers/src/ops/merge.rs b/.artifacts/archive/transformers/src/ops/merge.rs similarity index 100% rename from nn/transformers/src/ops/merge.rs rename to .artifacts/archive/transformers/src/ops/merge.rs diff --git a/nn/transformers/src/ops/mod.rs b/.artifacts/archive/transformers/src/ops/mod.rs similarity index 100% rename from nn/transformers/src/ops/mod.rs rename to .artifacts/archive/transformers/src/ops/mod.rs diff --git a/nn/transformers/src/ops/split.rs b/.artifacts/archive/transformers/src/ops/split.rs similarity index 100% rename from nn/transformers/src/ops/split.rs rename to .artifacts/archive/transformers/src/ops/split.rs diff --git a/nn/transformers/src/primitives.rs b/.artifacts/archive/transformers/src/primitives.rs similarity index 100% rename from nn/transformers/src/primitives.rs rename to .artifacts/archive/transformers/src/primitives.rs diff --git a/nn/transformers/src/specs.rs b/.artifacts/archive/transformers/src/specs.rs similarity index 100% rename from nn/transformers/src/specs.rs rename to .artifacts/archive/transformers/src/specs.rs diff --git a/nn/transformers/src/transform/config.rs b/.artifacts/archive/transformers/src/transform/config.rs similarity index 100% rename from nn/transformers/src/transform/config.rs rename to .artifacts/archive/transformers/src/transform/config.rs diff --git a/nn/transformers/src/transform/mod.rs b/.artifacts/archive/transformers/src/transform/mod.rs similarity index 100% rename from nn/transformers/src/transform/mod.rs rename to .artifacts/archive/transformers/src/transform/mod.rs diff --git a/nn/transformers/src/transform/transformer.rs b/.artifacts/archive/transformers/src/transform/transformer.rs similarity index 100% rename from nn/transformers/src/transform/transformer.rs rename to .artifacts/archive/transformers/src/transform/transformer.rs diff --git a/nn/transformers/src/utils.rs b/.artifacts/archive/transformers/src/utils.rs similarity index 100% rename from nn/transformers/src/utils.rs rename to .artifacts/archive/transformers/src/utils.rs diff --git a/nn/transformers/tests/default.rs b/.artifacts/archive/transformers/tests/default.rs similarity index 100% rename from nn/transformers/tests/default.rs rename to .artifacts/archive/transformers/tests/default.rs diff --git a/.artifacts/license/APACHE.SCSYS.LICENSE b/.artifacts/license/APACHE.SCSYS.LICENSE index 86685f9a..622e1d93 100644 --- a/.artifacts/license/APACHE.SCSYS.LICENSE +++ b/.artifacts/license/APACHE.SCSYS.LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2023 Scattered-Systems, DAO LLC + Copyright 2024 Scattered-Systems, LLC Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/.artifacts/license/MIT.LICENSE b/.artifacts/license/MIT.LICENSE index 4cea9a3c..94d45d58 100644 --- a/.artifacts/license/MIT.LICENSE +++ b/.artifacts/license/MIT.LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2023 Joe McCain III +Copyright (c) 2024 Joe McCain III Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/.github/ISSUE_TEMPLATE/issue.md b/.github/ISSUE_TEMPLATE/issue.md new file mode 100644 index 00000000..925cfede --- /dev/null +++ b/.github/ISSUE_TEMPLATE/issue.md @@ -0,0 +1,17 @@ +--- +about: A generic issue template +assignees: + - FL03 +labels: [] +projects: ['@FL03/concision:features'] +name: Generic Issue +title: '' +--- + +**Describe the proposal or feature that this issue is tracking.** + +## Issues + +- [] + +## Pull Requests diff --git a/.github/ISSUE_TEMPLATE/proposal.md b/.github/ISSUE_TEMPLATE/proposal.md index 1d5e2cb0..d8370f22 100644 --- a/.github/ISSUE_TEMPLATE/proposal.md +++ b/.github/ISSUE_TEMPLATE/proposal.md @@ -1,12 +1,14 @@ --- -name: Proposal -about: A proposal for a new feature or change -title: 'Proposal:' -labels: ['proposal'] -projects: ['@FL03/concision:features', '@FL03/concision:roadmap'] +about: A formal proposal discussing any new features, changes, or improvements to the project. assignees: - FL03 - +labels: ['proposal'] +name: Improvement Proposal +projects: ['@FL03/concision:features', '@FL03/concision:roadmap'] +title: 'CNC-0000:' --- +### Resources + +- [company](https://github.com/scattered-systems) diff --git a/.github/ISSUE_TEMPLATE/tracking.md b/.github/ISSUE_TEMPLATE/tracking.md new file mode 100644 index 00000000..0139c486 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/tracking.md @@ -0,0 +1,17 @@ +--- +about: Create a new tracking issue to track the progress of a proposal or feature. +assignees: + - FL03 +labels: ['tracking'] +projects: ['@FL03/concision:features'] +name: Tracking Issue +title: 'Tracking Issue:' +--- + +**Describe the proposal or feature that this issue is tracking.** + +## Issues + +- [] + +## Pull Requests diff --git a/.github/workflows/clippy.yml b/.github/workflows/clippy.yml index 8ad78abc..39d91bd1 100644 --- a/.github/workflows/clippy.yml +++ b/.github/workflows/clippy.yml @@ -1,4 +1,4 @@ -name: Clippy +name: clippy on: pull_request: diff --git a/.github/workflows/crates.yml b/.github/workflows/crates.yml index 467e3a1c..b7c226d0 100644 --- a/.github/workflows/crates.yml +++ b/.github/workflows/crates.yml @@ -1,4 +1,4 @@ -name: crates.io +name: crates concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 2ed6c641..afdf6cee 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -1,4 +1,4 @@ -name: Rust +name: rust concurrency: cancel-in-progress: false @@ -6,6 +6,7 @@ concurrency: env: CARGO_TERM_COLOR: always + CRATE_BASENAME: ${{ github.event.repository.name }} on: pull_request: @@ -53,12 +54,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: setup (langspace) + - name: rustup run: | rustup default nightly rustup update - - name: Bench - run: cargo bench --all -v + - run: cargo bench --all -v + test: name: Test strategy: @@ -67,14 +68,13 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: setup (langspace) - run: | - rustup default ${{ matrix.toolchain }} - rustup update - - name: Test - run: cargo test --features full -v --workspace + - run: rustup default ${{ matrix.toolchain }} && rustup update + - run: cargo test --features full -v --workspace + blas: continue-on-error: true + env: + PACKAGE_NAME: ${{ github.event.repository.name }}-${{ matrix.crate }} name: Test (blas) strategy: matrix: @@ -83,9 +83,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: rustup - run: | - rustup default ${{ matrix.toolchain }} - rustup update + - run: rustup default ${{ matrix.toolchain }} && rustup update - name: test - run: cargo test --features blas -v --package ${{ github.event.repository.name }}-${{ matrix.crate }} + run: | + cargo clean + cargo test --features blas -v -p ${{ env.PACKAGE_NAME }} diff --git a/Cargo.toml b/Cargo.toml index 51d4adf4..32f3103c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,22 +8,20 @@ keywords = ["data-science", "scsys", "toolkit"] license = "Apache-2.0" readme = "README.md" repository = "https://github.com/FL03/concision" -version = "0.1.13" +version = "0.1.14" [workspace.dependencies] # acme = { features = ["full"], branch = "v0.3.2", git = "https://github.com/FL03/acme", version = "0.3.2" } # ndtensor = { features = ["full"], branch = "v0.1.1", git = "https://github.com/FL03/ndtensor", version = "0.1" } -# scsys = { features = ["full"], branch = "v0.2.2", git = "https://github.com/scattered-systems/scsys", version = "0.2" } +scsys = { default-features = false, branch = "v0.2.3", features = ["derive"], git = "https://github.com/scattered-systems/scsys.git", version = "0.2" } approx = "0.5" -itertools = "0.12" +itertools = "0.13" lazy_static = "1" ndarray = { default-features = false, version = "0.15" } -ndarray-rand = "0.14" ndarray-stats = "0.5" num = { default-features = false, version = "0.4" } -# serde = { features = ["derive"], version = "1" } -# serde_json = "1" +paste = "1" smart-default = "0.7" strum = { default-features = false, features = ["derive"], version = "0.26" } @@ -33,7 +31,7 @@ default-members = [ ] exclude = [ - "nn/*", + ".artifacts/archive/*", ] members = [ @@ -44,7 +42,8 @@ members = [ "macros", "models/linear", "models/gnn", - "models/kan", + "models/kan", + "models/transformers", ] resolver = "2" diff --git a/LICENSE b/LICENSE index f25b2b18..ea3895e4 100644 --- a/LICENSE +++ b/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2023 Joe McCain III + Copyright 2024 Joe McCain III Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/README.md b/README.md index 3642afb8..fb5e3434 100644 --- a/README.md +++ b/README.md @@ -4,12 +4,15 @@ [![clippy](https://github.com/FL03/concision/actions/workflows/clippy.yml/badge.svg)](https://github.com/FL03/concision/actions/workflows/clippy.yml) [![rust](https://github.com/FL03/concision/actions/workflows/rust.yml/badge.svg)](https://github.com/FL03/concision/actions/workflows/rust.yml) -[![crates.io](https://github.com/FL03/concision/actions/workflows/crates.yml/badge.svg)](https://github.com/FL03/concision/actions/workflows/crates.yml) - *** -Concision is designed to be a complete toolkit for building machine learning models in Rust. +### _The library is currently in the early stages of development and is not yet ready for production use._ + +Concision is designed to be a complete toolkit for building machine learning models in Rust. + +Concision is a machine learning library for building powerful models in Rust prioritizing ease-of-use, efficiency, and flexability. The library is built to make use of the +both the upcoming `autodiff` experimental feature and increased support for generics in the 2024 edition of Rust. ## Getting Started @@ -28,28 +31,29 @@ cargo build --features full -r --workspace ## Usage +### Example: Linear Model (biased) + ```rust extern crate concision as cnc; - use cnc::func::Sigmoid; - use cnc::linear::{Config, Features, Linear}; - use cnc::{linarr, Predict, Result}; + use cnc::prelude::{linarr, Linear, Result, Sigmoid}; use ndarray::Ix2; fn main() -> Result<()> { tracing_subscriber::fmt::init(); tracing::info!("Starting linear model example"); - let (samples, dmodel, features) = (20, 5, 3); - let features = Features::new(3, 5); - let config = Config::new("example", features).biased(); - let data = linarr::((samples, dmodel)).unwrap(); + let (samples, d_in, d_out) = (20, 5, 3); + let data = linarr::((samples, d_in)).unwrap(); + + let model = Linear::::from_features(d_in, d_out).uniform(); + // let model = Linear::::from_features(d_in, d_out).uniform(); + + assert!(model.is_biased()); - let model: Linear = Linear::std(config).uniform(); - // `.activate(*data, *activation)` runs the forward pass and applies the activation function to the result let y = model.activate(&data, Sigmoid::sigmoid).unwrap(); - assert_eq!(y.dim(), (samples, features)); - println!("Predictions: {:?}", y); + assert_eq!(y.dim(), (samples, d_out)); + println!("Predictions:\n{:?}", &y); Ok(()) } diff --git a/concision/Cargo.toml b/concision/Cargo.toml index bf337937..1cd286ac 100644 --- a/concision/Cargo.toml +++ b/concision/Cargo.toml @@ -21,22 +21,10 @@ full = [ "default", "approx", "derive", + "models", "rand", "serde", "tracing", - "linear", - "gnn", -] - -alloc = [ - "concision-core/alloc", - "concision-data/alloc", - "concision-linear/alloc", - "concision-gnn/alloc", -] - -approx = [ - "concision-core/approx", ] data = [ @@ -48,6 +36,18 @@ derive = [ "macros", ] +macros = [ + "dep:concision-macros" +] + +# ********* [FF] Models(s) ********* +models = [ + "gnn", + "kan", + "linear", + "transformer", +] + gnn = [ "dep:concision-gnn", ] @@ -57,74 +57,97 @@ kan = [ ] linear = [ - "dep:concision-linear" + "dep:concision-linear", ] -macros = [ - "dep:concision-macros" +transformer = [ + "dep:concision-transformers", ] -models = [ - "gnn", - "kan", - "linear", + +# ********* [FF] Dependencies ********* +alloc = [ + "concision-core/alloc", + "concision-data?/alloc", + "concision-gnn?/alloc", + "concision-kan?/alloc", + "concision-linear?/alloc", + "concision-transformers?/alloc", +] + +approx = [ + "concision-core/approx", + "concision-data?/approx", + "concision-gnn?/approx", + "concision-kan?/approx", + "concision-linear?/approx", + "concision-transformers?/approx", ] rand = [ "concision-core/rand", - "concision-data/rand", - - "concision-gnn/rand", - "concision-kan/rand", - "concision-linear/rand", + "concision-data?/rand", + "concision-gnn?/rand", + "concision-kan?/rand", + "concision-linear?/rand", + "concision-transformers?/rand", ] serde = [ "concision-core/serde", - "concision-data/serde", - "concision-gnn/serde", - "concision-kan/serde", - "concision-linear/serde", -] - -std = [ - "concision-core/std", - "concision-data/std", - "concision-gnn/std", - "concision-kan/std", - "concision-linear/std", + "concision-data?/serde", + "concision-gnn?/serde", + "concision-kan?/serde", + "concision-linear?/serde", + "concision-transformers?/serde", ] tracing = [ "concision-core/tracing", - "concision-data/tracing", - "concision-gnn/tracing", - "concision-kan/tracing", - "concision-linear/tracing", + "concision-data?/tracing", + "concision-gnn?/tracing", + "concision-kan?/tracing", + "concision-linear?/tracing", + "concision-transformers?/tracing", +] + +# ********* [FF] Environment(s) ********* + +std = [ + "concision-core/std", + "concision-data?/std", + "concision-gnn?/std", + "concision-kan?/std", + "concision-linear?/std", + "concision-transformers?/std", ] wasm = [ "concision-core/wasm", - "concision-data/wasm", - "concision-gnn/wasm", - "concision-kan/wasm", - "concision-linear/wasm", + "concision-data?/wasm", + "concision-gnn?/wasm", + "concision-kan?/wasm", + "concision-linear?/wasm", + "concision-transformers?/wasm", ] wasi = [ "concision-core/wasi", - "concision-data/wasi", - "concision-gnn/wasi", - "concision-kan/wasi", - "concision-linear/wasi", + "concision-data?/wasi", + "concision-gnn?/wasi", + "concision-kan?/wasi", + "concision-linear?/wasi", + "concision-transformers?/wasi", ] +# ********* [FF] Blas ********* blas = [ "concision-core/blas", - "concision-data/blas", - "concision-gnn/blas", - "concision-kan/blas", - "concision-linear/blas", + "concision-data?/blas", + "concision-gnn?/blas", + "concision-kan?/blas", + "concision-linear?/blas", + "concision-transformers?/blas", ] intel-mkl-system = [ @@ -161,46 +184,56 @@ test = true name = "linear" required-features = ["linear", "rand", "serde", "tracing"] +[[example]] +name = "transformer" +required-features = ["transformer", "rand", "serde", "tracing"] + [build-dependencies] [dependencies.concision-core] path = "../core" -version = "0.1.13" +version = "0.1.14" [dependencies.concision-data] optional = true path = "../data" -version = "0.1.13" +version = "0.1.14" [dependencies.concision-derive] optional = true path = "../derive" -version = "0.1.13" +version = "0.1.14" [dependencies.concision-macros] optional = true path = "../macros" -version = "0.1.13" +version = "0.1.14" # *************** Models *************** [dependencies.concision-gnn] optional = true path = "../models/gnn" -version = "0.1.13" +version = "0.1.14" [dependencies.concision-kan] optional = true path = "../models/kan" -version = "0.1.13" +version = "0.1.14" [dependencies.concision-linear] optional = true path = "../models/linear" -version = "0.1.13" +version = "0.1.14" + +[dependencies.concision-transformers] +optional = true +path = "../models/transformers" +version = "0.1.14" [dev-dependencies] anyhow = "1" +approx.workspace = true lazy_static.workspace = true ndarray.workspace = true num = { features = ["rand", "serde"], version = "0.4" } diff --git a/concision/benches/default.rs b/concision/benches/default.rs index 937f2387..9e07b340 100644 --- a/concision/benches/default.rs +++ b/concision/benches/default.rs @@ -3,50 +3,97 @@ extern crate test; -use std::mem::replace; use test::Bencher; // bench: find the `BENCH_SIZE` first terms of the fibonacci sequence -static BENCH_SIZE: usize = 20; - -// recursive fibonacci -fn fibonacci(n: usize) -> u32 { - if n < 2 { - 1 - } else { - fibonacci(n - 1) + fibonacci(n - 2) - } -} - -// iterative fibonacci -struct Fibonacci { - curr: u32, - next: u32, -} - -impl Iterator for Fibonacci { - type Item = u32; - fn next(&mut self) -> Option { - let new_next = self.curr + self.next; - let new_curr = replace(&mut self.next, new_next); +const BENCH_SIZE: u32 = 20; - Some(replace(&mut self.curr, new_curr)) - } +#[bench] +fn fibonacci(b: &mut Bencher) { + // exact code to benchmark must be passed as a closure to the iter + // method of Bencher + b.iter(|| (0..BENCH_SIZE).map(fib::fibonacci).collect::>()) } -fn fibonacci_sequence() -> Fibonacci { - Fibonacci { curr: 1, next: 1 } +#[bench] +fn iter_fibonacci(b: &mut Bencher) { + b.iter(|| { + fib::Fibonacci::new() + .take(BENCH_SIZE as usize) + .collect::>() + }) } -// function to benchmark must be annotated with `#[bench]` #[bench] fn recursive_fibonacci(b: &mut Bencher) { // exact code to benchmark must be passed as a closure to the iter // method of Bencher - b.iter(|| (0..BENCH_SIZE).map(fibonacci).collect::>()) + b.iter(|| { + (0..BENCH_SIZE) + .map(fib::recursive_fibonacci) + .collect::>() + }) } -#[bench] -fn iterative_fibonacci(b: &mut Bencher) { - b.iter(|| fibonacci_sequence().take(BENCH_SIZE).collect::>()) +mod fib { + /// fibonacci(n) returns the nth fibonacci number + /// This function uses the definition of Fibonacci where: + /// F(0) = F(1) = 1 and F(n+1) = F(n) + F(n-1) for n>0 + /// + /// Warning: This will overflow the 128-bit unsigned integer at n=186 + pub fn fibonacci(n: u32) -> u128 { + // Use a and b to store the previous two values in the sequence + let mut a = 0; + let mut b = 1; + for _i in 0..n { + // As we iterate through, move b's value into a and the new computed + // value into b. + let c = a + b; + a = b; + b = c; + } + b + } + + /// fibonacci(n) returns the nth fibonacci number + /// This function uses the definition of Fibonacci where: + /// F(0) = F(1) = 1 and F(n+1) = F(n) + F(n-1) for n>0 + /// + /// Warning: This will overflow the 128-bit unsigned integer at n=186 + pub fn recursive_fibonacci(n: u32) -> u128 { + // Call the actual tail recursive implementation, with the extra + // arguments set up. + _recursive_fibonacci(n, 0, 1) + } + + fn _recursive_fibonacci(n: u32, previous: u128, current: u128) -> u128 { + if n == 0 { + current + } else { + _recursive_fibonacci(n - 1, current, current + previous) + } + } + + pub struct Fibonacci { + curr: u32, + next: u32, + } + + impl Fibonacci { + pub fn new() -> Fibonacci { + Fibonacci { curr: 0, next: 1 } + } + } + + impl Iterator for Fibonacci { + type Item = u32; + + fn next(&mut self) -> Option { + use core::mem::replace; + let new_next = self.curr + self.next; + let new_curr = replace(&mut self.next, new_next); + + Some(replace(&mut self.curr, new_curr)) + } + } } diff --git a/concision/examples/linear.rs b/concision/examples/linear.rs index 284325f8..64d77b4c 100644 --- a/concision/examples/linear.rs +++ b/concision/examples/linear.rs @@ -4,8 +4,8 @@ */ extern crate concision as cnc; -use cnc::linear::{Biased, Linear}; -use cnc::prelude::{linarr, Result, Sigmoid}; +use cnc::linear::Features; +use cnc::prelude::{linarr, InitializeExt, Linear, Result, Sigmoid}; use ndarray::Ix2; fn tracing() { @@ -24,16 +24,17 @@ fn tracing() { fn main() -> Result<()> { tracing(); tracing::info!("Starting linear model example"); + let samples = 20; + let (dm, dn) = (5, 3); + let features = Features::new(dn, dm); + let data = linarr::((samples, dm)).unwrap(); - let (samples, d_in, d_out) = (20, 5, 3); - let data = linarr::((samples, d_in)).unwrap(); - - let model: Linear = Linear::from_features(d_in, d_out).uniform(); + let model = Linear::::lecun_normal(features, dm); assert!(model.is_biased()); let y = model.activate(&data, Sigmoid::sigmoid).unwrap(); - assert_eq!(y.dim(), (samples, d_out)); - println!("Predictions:\n{:?}", &y); + assert_eq!(y.dim(), (samples, dn)); + println!("Predictions:\n{:#?}", &y); Ok(()) } diff --git a/concision/examples/transformer.rs b/concision/examples/transformer.rs new file mode 100644 index 00000000..cc24879b --- /dev/null +++ b/concision/examples/transformer.rs @@ -0,0 +1,38 @@ +/* + Appellation: transformer + Contrib: FL03 +*/ +extern crate concision as cnc; + +use approx::AbsDiffEq; +use cnc::prelude::Result; +use cnc::transformer::AttentionHead; +use ndarray::Array2; + +fn tracing() { + use tracing::Level; + use tracing_subscriber::fmt::time; + + tracing_subscriber::fmt() + .compact() + .with_ansi(true) + .with_max_level(Level::DEBUG) + .with_target(false) + .with_timer(time::uptime()) + .init(); +} + +fn main() -> Result<()> { + tracing(); + tracing::info!("Starting up the transformer model example..."); + + let shape = (3, 3); + let head = AttentionHead::::ones(shape); + let score = head.attention(); + assert!(score + .attention() + .abs_diff_eq(&Array2::from_elem(shape, 1f64 / 3f64), 1e-6)); + println!("{:?}", score); + + Ok(()) +} diff --git a/concision/src/lib.rs b/concision/src/lib.rs index f4739902..ae868246 100644 --- a/concision/src/lib.rs +++ b/concision/src/lib.rs @@ -27,6 +27,9 @@ pub use concision_kan as kan; pub use concision_linear as linear; #[cfg(feature = "macros")] pub use concision_macros::*; +#[cfg(feature = "transformer")] +#[doc(inline)] +pub use concision_transformers as transformer; pub mod prelude { pub use concision_core::prelude::*; @@ -42,4 +45,6 @@ pub mod prelude { pub use concision_linear::prelude::*; #[cfg(feature = "macros")] pub use concision_macros::*; + #[cfg(feature = "transformer")] + pub use concision_transformers::prelude::*; } diff --git a/core/Cargo.toml b/core/Cargo.toml index c8d8e060..c8f85779 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -30,6 +30,7 @@ alloc = [ "num/alloc", "rand?/alloc", "rand_distr?/alloc", + "scsys/alloc", "serde?/alloc", ] @@ -55,7 +56,7 @@ rand-ext = [ "uuid/v4", ] -rng_std = [ +std-rng = [ "rand?/std", "rand?/std_rng", ] @@ -66,6 +67,7 @@ serde = [ "num/serde", "rand?/serde1", "rand_distr?/serde1", + "scsys/serde", "uuid/serde" ] @@ -80,15 +82,18 @@ tracing = [ # ********* [FF] Environments ********* std = [ "alloc", + "std-rng", "ndarray/std", "num/std", - "rng_std", + "scsys/std", "serde/std", "strum/std", "uuid/std" ] -wasm = [] +wasm = [ + "getrandom/js", +] wasi = [] @@ -98,15 +103,29 @@ crate-type = ["lib"] doctest = false test = true + + [[test]] name = "fft" required-features = ["approx"] +[[test]] +name = "init" +required-features = ["rand", "std"] + +[[test]] +name = "nn" + [build-dependencies] +[dev-dependencies] +lazy_static.workspace = true + [dependencies] ndarray.workspace = true num.workspace = true +paste.workspace = true +scsys.workspace = true smart-default.workspace = true strum.workspace = true @@ -143,13 +162,11 @@ default-features = false features = ["v5", "v8"] version = "1" -[dev-dependencies] -lazy_static = "1" - [package.metadata.docs.rs] all-features = true rustc-args = ["--cfg", "docsrs"] -[target.wasm32-unknown-unknown] +[target.wasm32-unknown-unknown.dependencies] +getrandom = "0.2" [target.wasm32-wasi] diff --git a/core/src/error/kinds/external.rs b/core/src/error/kinds/external.rs index 89bdc0e0..eb5d289f 100644 --- a/core/src/error/kinds/external.rs +++ b/core/src/error/kinds/external.rs @@ -67,4 +67,4 @@ impl From> for ExternalError { } } -error_from!(ExternalError::Error<&str, String>); +from_variant!(ExternalError::Error {<&str>.to_string(), .to_string()}); diff --git a/core/src/error/kinds/predict.rs b/core/src/error/kinds/predict.rs index 114657c1..de39d1d8 100644 --- a/core/src/error/kinds/predict.rs +++ b/core/src/error/kinds/predict.rs @@ -2,6 +2,7 @@ Appellation: error Contrib: FL03 */ +use scsys::VariantConstructors; use smart_default::SmartDefault; use strum::{AsRefStr, Display, EnumCount, EnumIs, EnumIter, EnumString, VariantNames}; @@ -20,6 +21,7 @@ use strum::{AsRefStr, Display, EnumCount, EnumIs, EnumIter, EnumString, VariantN PartialEq, PartialOrd, SmartDefault, + VariantConstructors, VariantNames, )] #[cfg_attr( @@ -34,11 +36,3 @@ pub enum PredictError { ShapeMismatch, TypeError, } - -impl PredictError { - variant_constructor!( - ArithmeticError.arithmetic_error, - ShapeMismatch.shape_mismatch, - TypeError.type_error - ); -} diff --git a/core/src/error/kinds/shape.rs b/core/src/error/kinds/shape.rs index 9e493334..d7c2c831 100644 --- a/core/src/error/kinds/shape.rs +++ b/core/src/error/kinds/shape.rs @@ -27,10 +27,9 @@ use strum::{AsRefStr, Display, EnumCount, EnumIs, EnumIter, EnumString, VariantN )] #[strum(serialize_all = "snake_case")] pub enum ShapeError { - LayoutError, + IncompatibleLayout, + IncompatibleRank, ShapeMismatch, - RankMismatch, SizeMismatch, - Unknown, } diff --git a/core/src/func/activate.rs b/core/src/func/activate.rs deleted file mode 100644 index 5ab8d035..00000000 --- a/core/src/func/activate.rs +++ /dev/null @@ -1,38 +0,0 @@ -/* - Appellation: activate - Contrib: FL03 -*/ -pub use self::{binary::*, nl::*}; - -pub mod binary; -pub mod nl; - -pub fn linear(x: &T) -> T -where - T: Clone, -{ - x.clone() -} - -build_unary_trait!(LinearActivation.linear); - -impl LinearActivation for T -where - T: Clone, -{ - type Output = T; - - fn linear(&self) -> Self::Output { - linear(self) - } -} - -pub(crate) mod prelude { - pub use super::binary::*; - pub use super::nl::*; - pub use super::{linear, LinearActivation}; -} - -pub trait Activator { - type Output; -} diff --git a/core/src/func/activate/binary.rs b/core/src/func/activate/binary.rs index b45a0588..5b417dc0 100644 --- a/core/src/func/activate/binary.rs +++ b/core/src/func/activate/binary.rs @@ -6,18 +6,18 @@ use nd::{Array, ArrayBase, Data, Dimension}; use num::{One, Zero}; /// -pub fn heavyside(x: &T) -> T +pub fn heavyside(x: T) -> T where T: One + PartialOrd + Zero, { - if x > &T::zero() { + if x > T::zero() { T::one() } else { T::zero() } } -build_unary_trait!(Heavyside.heavyside,); +unary!(Heavyside::heavyside(self),); macro_rules! impl_heavyside { ($($ty:ty),* $(,)*) => { @@ -27,7 +27,7 @@ macro_rules! impl_heavyside { impl Heavyside for $ty { type Output = $ty; - fn heavyside(&self) -> Self::Output { + fn heavyside(self) -> Self::Output { heavyside(self) } } @@ -36,15 +36,28 @@ macro_rules! impl_heavyside { impl_heavyside!(f32, f64, i8, i16, i32, i64, i128, isize, u8, u16, u32, u64, u128, usize,); -impl Heavyside for ArrayBase +impl Heavyside for ArrayBase where - A: Heavyside, + A: Clone + Heavyside, D: Dimension, S: Data, { - type Output = Array<::Output, D>; + type Output = Array; - fn heavyside(&self) -> Self::Output { - self.map(Heavyside::heavyside) + fn heavyside(self) -> Self::Output { + self.mapv(Heavyside::heavyside) + } +} + +impl<'a, A, B, S, D> Heavyside for &'a ArrayBase +where + A: Clone + Heavyside, + D: Dimension, + S: Data, +{ + type Output = Array; + + fn heavyside(self) -> Self::Output { + self.mapv(Heavyside::heavyside) } } diff --git a/core/src/func/activate/linear.rs b/core/src/func/activate/linear.rs new file mode 100644 index 00000000..d4c9dc7e --- /dev/null +++ b/core/src/func/activate/linear.rs @@ -0,0 +1,21 @@ +/* + Appellation: linear + Contrib: FL03 +*/ + +pub fn linear(x: T) -> T { + x +} + +unary!(LinearActivation::linear(self)); + +impl<'a, T> LinearActivation for &'a T +where + T: Clone, +{ + type Output = T; + + fn linear(self) -> Self::Output { + self.clone() + } +} diff --git a/core/src/func/activate/mod.rs b/core/src/func/activate/mod.rs new file mode 100644 index 00000000..cd1c2f0b --- /dev/null +++ b/core/src/func/activate/mod.rs @@ -0,0 +1,32 @@ +/* + Appellation: activate + Contrib: FL03 +*/ +pub use self::{binary::*, linear::*, nl::*}; + +pub mod binary; +pub mod linear; +pub mod nl; + +pub(crate) mod prelude { + pub use super::binary::*; + pub use super::linear::*; + pub use super::nl::*; + pub use super::{Activate, Evaluate}; +} + +#[doc(hidden)] +pub trait Activate { + type Output; + + fn activate(&self, args: &T) -> Self::Output; +} + +#[doc(hidden)] +pub trait Evaluate { + type Output; + + fn eval(&self, args: T) -> Self::Output; +} + +activator!(LinearActor::(T::clone) where T: Clone); diff --git a/core/src/func/activate/nl.rs b/core/src/func/activate/nl.rs index 93341e6f..d9e70fed 100644 --- a/core/src/func/activate/nl.rs +++ b/core/src/func/activate/nl.rs @@ -2,59 +2,60 @@ Appellation: sigmoid Contrib: FL03 */ +use crate::math::Exp; use ndarray::*; use num::complex::{Complex, ComplexFloat}; -use num::{Float, Zero}; +use num::traits::Zero; -pub fn relu(args: &T) -> T +fn _relu(args: T) -> T where - T: Clone + PartialOrd + Zero, + T: PartialOrd + Zero, { - if args > &T::zero() { - return args.clone(); + if args > T::zero() { + return args; } T::zero() } -pub fn sigmoid(args: &T) -> T +fn _sigmoid(args: T) -> T where T: ComplexFloat, { - (T::one() + (*args).neg().exp()).recip() + (T::one() + args.neg().exp()).recip() } -pub fn softmax(args: &Array) -> Array +fn _softmax(args: &ArrayBase) -> Array where + A: ComplexFloat + ScalarOperand, D: Dimension, - T: Float, + S: Data, { - let denom = args.mapv(|x| x.exp()).sum(); - args.mapv(|x| x.exp() / denom) + let e = args.exp(); + &e / e.sum() } -pub fn softmax_axis(args: &Array, axis: Option) -> Array -where - D: Dimension + RemoveAxis, - T: NdFloat, -{ - let exp = args.mapv(|x| x.exp()); - if let Some(axis) = axis { - let denom = exp.sum_axis(Axis(axis)); - exp / denom - } else { - let denom = exp.sum(); - exp / denom - } -} +// fn __softmax(args: &I) -> I +// where +// I: Clone + core::ops::Div + Exp, T: Exp + core::iter::Sum , +// for<'a> I: IntoIterator, +// { +// let e = args.exp(); +// e.clone() / e.into_iter().sum::() +// } -pub fn tanh(args: &T) -> T +fn _tanh(args: T) -> T where T: ComplexFloat, { args.tanh() } -build_unary_trait!(ReLU.relu, Sigmoid.sigmoid, Softmax.softmax, Tanh.tanh,); +unary!( + ReLU::relu(self), + Sigmoid::sigmoid(self), + Softmax::softmax(self), + Tanh::tanh(self), +); /* ********** Implementations ********** @@ -73,22 +74,25 @@ macro_rules! nonlinear { nonlinear!(@arr $rho::$call); }; (@impl $rho:ident::$call:ident<$T:ty>) => { - impl $rho for $T { - type Output = $T; + paste::paste! { + impl $rho for $T { + type Output = $T; - fn $call(&self) -> Self::Output { - $call(self) + fn $call(self) -> Self::Output { + [<_ $call>](self) + } } - } - impl<'a> $rho for &'a $T { - type Output = $T; + impl<'a> $rho for &'a $T { + type Output = $T; - fn $call(&self) -> Self::Output { - $call(*self) + fn $call(self) -> Self::Output { + [<_ $call>](*self) + } } } + }; (@arr $name:ident::$call:ident) => { impl $name for ArrayBase @@ -99,12 +103,24 @@ macro_rules! nonlinear { { type Output = Array<::Output, D>; - fn $call(&self) -> Self::Output { - self.map($name::$call) + fn $call(self) -> Self::Output { + self.mapv($name::$call) } } - }; + impl<'a, A, S, D> $name for &'a ArrayBase + where + A: Clone + $name, + D: Dimension, + S: Data + { + type Output = Array<::Output, D>; + + fn $call(self) -> Self::Output { + self.mapv($name::$call) + } + } + }; } nonlinear!( @@ -134,6 +150,32 @@ nonlinear!( f32, f64, Complex, - Complex < f64 > + Complex ]>, ); + +impl Softmax for ArrayBase +where + A: ComplexFloat + ScalarOperand, + D: Dimension, + S: Data, +{ + type Output = Array; + + fn softmax(self) -> Self::Output { + _softmax(&self) + } +} + +impl<'a, A, S, D> Softmax for &'a ArrayBase +where + A: ComplexFloat + ScalarOperand, + D: Dimension, + S: Data, +{ + type Output = Array; + + fn softmax(self) -> Self::Output { + _softmax(self) + } +} diff --git a/core/src/func/loss.rs b/core/src/func/loss.rs deleted file mode 100644 index aed784ab..00000000 --- a/core/src/func/loss.rs +++ /dev/null @@ -1,14 +0,0 @@ -/* - Appellation: loss - Contrib: FL03 -*/ - -pub(crate) mod prelude { - pub use super::Loss; -} - -pub trait Loss { - type Output; - - fn loss(&self, cmp: &T) -> Self::Output; -} diff --git a/core/src/func/loss/entropy.rs b/core/src/func/loss/entropy.rs new file mode 100644 index 00000000..5e966b72 --- /dev/null +++ b/core/src/func/loss/entropy.rs @@ -0,0 +1,12 @@ +/* + Appellation: entropy + Contrib: FL03 +*/ + +pub trait Entropy { + type Output; + + fn cross_entropy(&self, target: &T) -> Self::Output; +} + +pub struct CrossEntropy; diff --git a/core/src/func/loss/mod.rs b/core/src/func/loss/mod.rs new file mode 100644 index 00000000..71694cbb --- /dev/null +++ b/core/src/func/loss/mod.rs @@ -0,0 +1,23 @@ +/* + Appellation: loss + Contrib: FL03 +*/ +pub use self::reg::prelude::*; +pub use self::{entropy::*, utils::*}; + +pub(crate) mod utils; + +pub mod entropy; +pub mod reg; + +pub(crate) mod prelude { + pub use super::reg::prelude::*; + pub use super::utils::*; + pub use super::Loss; +} + +pub trait Loss { + type Output; + + fn loss(&self, a: &A, cmp: &B) -> Self::Output; +} diff --git a/core/src/func/loss/reg.rs b/core/src/func/loss/reg.rs new file mode 100644 index 00000000..679d8a6d --- /dev/null +++ b/core/src/func/loss/reg.rs @@ -0,0 +1,13 @@ +/* + Appellation: reg + Contrib: FL03 +*/ +//! # Regressive Loss Functions +//! +//! + +pub mod avg; + +pub(crate) mod prelude { + pub use super::avg::*; +} diff --git a/core/src/func/loss/reg/avg.rs b/core/src/func/loss/reg/avg.rs new file mode 100644 index 00000000..25d80beb --- /dev/null +++ b/core/src/func/loss/reg/avg.rs @@ -0,0 +1,65 @@ +/* + Appellation: avg + Contrib: FL03 +*/ +use crate::math::{Abs, Squared}; +use nd::prelude::*; +use nd::{Data, ScalarOperand}; +use num::traits::{FromPrimitive, Num, Pow, Signed}; + +pub trait MeanAbsoluteError { + type Output; + + fn mae(&self, target: &Rhs) -> Self::Output; +} + +pub trait MeanSquaredError { + type Output; + + fn mse(&self, target: &Rhs) -> Self::Output; +} + +losses! { + impl MSE::, ArrayBase, Output = Option>(MeanSquaredError::mse) + where + A: FromPrimitive + Num + Pow + ScalarOperand, + D: Dimension, + S: Data, +} + +losses! { + impl MAE::, ArrayBase, Output = Option>(MeanAbsoluteError::mae) + where + A: FromPrimitive + Num + ScalarOperand + Signed, + D: Dimension, + S: Data, +} + +/* + ************* Implementations ************* +*/ +impl MeanAbsoluteError> for ArrayBase +where + A: FromPrimitive + Num + ScalarOperand + Signed, + D: Dimension, + S: Data, +{ + type Output = Option; + + fn mae(&self, target: &ArrayBase) -> Self::Output { + (target - self).abs().mean() + } +} + +impl MeanSquaredError> for ArrayBase +where + A: FromPrimitive + Num + Pow + ScalarOperand, + D: Dimension, + S: Data, +{ + type Output = Option; + + fn mse(&self, target: &ArrayBase) -> Self::Output { + (target - self).sqrd().mean() + } +} diff --git a/core/src/func/loss/utils.rs b/core/src/func/loss/utils.rs new file mode 100644 index 00000000..9f61779b --- /dev/null +++ b/core/src/func/loss/utils.rs @@ -0,0 +1,29 @@ +/* + Appellation: utils + Contrib: FL03 +*/ +use crate::math::{Abs, Squared}; +use nd::prelude::*; +use nd::{Data, ScalarOperand}; +use num::traits::{FromPrimitive, Num, Pow, Signed}; + +/// A functional implementation of the mean absolute error loss function which compares two similar +/// [arrays](ndarray::ArrayBase) +pub fn mae(pred: &ArrayBase, target: &ArrayBase) -> Option +where + A: FromPrimitive + Num + ScalarOperand + Signed, + D: Dimension, + S: Data, +{ + (pred - target).abs().mean() +} +/// A functional implementation of the mean squared error loss function that compares two similar +/// [arrays](ndarray::ArrayBase) +pub fn mse(pred: &ArrayBase, target: &ArrayBase) -> Option +where + A: FromPrimitive + Num + Pow + ScalarOperand, + D: Dimension, + S: Data, +{ + (pred - target).sqrd().mean() +} diff --git a/core/src/func/mod.rs b/core/src/func/mod.rs index 266ce461..96513d96 100644 --- a/core/src/func/mod.rs +++ b/core/src/func/mod.rs @@ -5,6 +5,7 @@ //! Functional pub use self::prelude::*; +#[macro_use] pub mod activate; pub mod loss; diff --git a/core/src/init/distr/lecun.rs b/core/src/init/distr/lecun.rs new file mode 100644 index 00000000..0c4763c5 --- /dev/null +++ b/core/src/init/distr/lecun.rs @@ -0,0 +1,54 @@ +/* + Appellation: lecun + Contrib: FL03 +*/ +use crate::init::distr::TruncatedNormal; +use num::Float; +use rand::Rng; +use rand_distr::{Distribution, NormalError, StandardNormal}; + +/// [LecunNormal] is a truncated [normal](rand_distr::Normal) distribution centered at 0 +/// with a standard deviation that is calculated as `σ = sqrt(1/n_in)` +/// where `n_in` is the number of input units. +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct LecunNormal { + n: usize, +} + +impl LecunNormal { + pub fn new(n: usize) -> Self { + Self { n } + } + /// Create a [truncated normal](TruncatedNormal) [distribution](Distribution) centered at 0; + /// See [Self::std_dev] for the standard deviation calculations. + pub fn distr(&self) -> Result, NormalError> + where + F: Float, + StandardNormal: Distribution, + { + TruncatedNormal::new(F::zero(), self.std_dev()) + } + /// Calculate the standard deviation (`σ`) of the distribution. + /// This is done by computing the root of the reciprocal of the number of inputs + /// + /// Symbolically: `σ = sqrt(1/n)` + pub fn std_dev(&self) -> F + where + F: Float, + { + F::from(self.n).unwrap().recip().sqrt() + } +} + +impl Distribution for LecunNormal +where + F: Float, + StandardNormal: Distribution, +{ + fn sample(&self, rng: &mut R) -> F + where + R: Rng + ?Sized, + { + self.distr().expect("NormalError").sample(rng) + } +} diff --git a/core/src/init/distr/trunc.rs b/core/src/init/distr/trunc.rs new file mode 100644 index 00000000..fc94f0b9 --- /dev/null +++ b/core/src/init/distr/trunc.rs @@ -0,0 +1,81 @@ +/* + Appellation: trunc + Contrib: FL03 +*/ +use num::traits::Float; +use rand::Rng; +use rand_distr::{Distribution, Normal, NormalError, StandardNormal}; + +/// A truncated normal distribution is similar to a [normal](rand_distr::Normal) [distribution](rand_distr::Distribution), however, +/// any generated value over two standard deviations from the mean is discarded and re-generated. +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct TruncatedNormal +where + StandardNormal: Distribution, +{ + mean: F, + std: F, +} + +impl TruncatedNormal +where + F: Float, + StandardNormal: Distribution, +{ + /// Create a new truncated normal distribution with a given mean and standard deviation + pub fn new(mean: F, std: F) -> Result { + Ok(Self { mean, std }) + } + + pub(crate) fn boundary(&self) -> F { + self.mean() + self.std_dev() * F::from(2).unwrap() + } + + pub(crate) fn score(&self, x: F) -> F { + self.mean() - self.std_dev() * x + } + + pub fn distr(&self) -> Normal { + Normal::new(self.mean(), self.std_dev()).unwrap() + } + + pub fn mean(&self) -> F { + self.mean + } + + pub fn std_dev(&self) -> F { + self.std + } +} + +impl Distribution for TruncatedNormal +where + F: Float, + StandardNormal: Distribution, +{ + fn sample(&self, rng: &mut R) -> F + where + R: Rng + ?Sized, + { + let bnd = self.boundary(); + let mut x = self.score(rng.sample(StandardNormal)); + // if x is outside of the boundary, re-sample + while x < -bnd || x > bnd { + x = self.score(rng.sample(StandardNormal)); + } + x + } +} + +impl From> for TruncatedNormal +where + F: Float, + StandardNormal: Distribution, +{ + fn from(normal: Normal) -> Self { + Self { + mean: normal.mean(), + std: normal.std_dev(), + } + } +} diff --git a/core/src/init/distr/xavier.rs b/core/src/init/distr/xavier.rs new file mode 100644 index 00000000..ecc1ee0c --- /dev/null +++ b/core/src/init/distr/xavier.rs @@ -0,0 +1,117 @@ +/* + Appellation: xavier + Contrib: FL03 +*/ +//! # Xavier +//! +//! Xavier initialization techniques were developed in 2010 by Xavier Glorot. +//! These methods are designed to initialize the weights of a neural network in a way that +//! prevents the vanishing and exploding gradient problems. The initialization technique +//! manifests into two distributions: [XavierNormal] and [XavierUniform]. +// #76 +use num::Float; +use rand::Rng; +use rand_distr::uniform::{SampleUniform, Uniform}; +use rand_distr::{Distribution, Normal, NormalError, StandardNormal}; + +pub(crate) fn std_dev(inputs: usize, outputs: usize) -> F +where + F: Float, +{ + (F::from(2).unwrap() / F::from(inputs + outputs).unwrap()).sqrt() +} + +pub(crate) fn boundary(inputs: usize, outputs: usize) -> F +where + F: Float, +{ + (F::from(6).unwrap() / F::from(inputs + outputs).unwrap()).sqrt() +} +/// Normal Xavier initializers leverage a normal distribution with a mean of 0 and a standard deviation (`σ`) +/// computed by the formula: `σ = sqrt(2/(d_in + d_out))` +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct XavierNormal +where + F: Float, + StandardNormal: Distribution, +{ + std: F, +} + +impl XavierNormal +where + F: Float, + StandardNormal: Distribution, +{ + pub fn new(inputs: usize, outputs: usize) -> Self { + Self { + std: std_dev(inputs, outputs), + } + } + + pub fn distr(&self) -> Result, NormalError> { + Normal::new(F::zero(), self.std_dev()) + } + + pub fn std_dev(&self) -> F { + self.std + } +} + +impl Distribution for XavierNormal +where + F: Float, + StandardNormal: Distribution, +{ + fn sample(&self, rng: &mut R) -> F + where + R: Rng + ?Sized, + { + self.distr().unwrap().sample(rng) + } +} + +/// Uniform Xavier initializers use a uniform distribution to initialize the weights of a neural network +/// within a given range. +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct XavierUniform +where + X: SampleUniform, +{ + boundary: X, +} + +impl XavierUniform +where + X: Float + SampleUniform, +{ + pub fn new(inputs: usize, outputs: usize) -> Self { + Self { + boundary: boundary(inputs, outputs), + } + } + + pub fn boundary(&self) -> X { + self.boundary + } + + pub fn distr(&self) -> Uniform + where + X: Float, + { + let bnd = self.boundary(); + Uniform::new(-bnd, bnd) + } +} + +impl Distribution for XavierUniform +where + X: Float + SampleUniform, +{ + fn sample(&self, rng: &mut R) -> X + where + R: Rng + ?Sized, + { + self.distr().sample(rng) + } +} diff --git a/core/src/init/initializer.rs b/core/src/init/initializer.rs new file mode 100644 index 00000000..2de38df9 --- /dev/null +++ b/core/src/init/initializer.rs @@ -0,0 +1,40 @@ +/* + Appellation: initializer + Contrib: FL03 +*/ +use super::Initialize; +use core::marker::PhantomData; +use nd::prelude::*; +use nd::DataOwned; +use rand_distr::{Distribution, StandardNormal}; + +pub struct InitializerBase +where + D: Dimension, + Dst: Clone + Distribution, +{ + pub(crate) dim: D, + pub(crate) distr: Dst, + pub(crate) _dtype: PhantomData, +} + +impl InitializerBase +where + D: Dimension, + Dst: Clone + Distribution, +{ + pub fn new(dim: D, distr: Dst) -> Self { + Self { + dim, + distr, + _dtype: PhantomData::, + } + } + + pub fn init(self) -> ArrayBase + where + S: DataOwned, + { + ArrayBase::rand(self.dim, self.distr) + } +} diff --git a/core/src/init/mod.rs b/core/src/init/mod.rs new file mode 100644 index 00000000..7f5bc4b4 --- /dev/null +++ b/core/src/init/mod.rs @@ -0,0 +1,48 @@ +/* + Appellation: init + Contrib: FL03 +*/ +//! # Initialization +//! +//! This module implements several initialization primitives for generating tensors using +//! various distributions and strategies. The module is designed to be used in conjuction with +//! the `rand` and `rand_distr` libraries. While `ndarray_rand` provides a `RandomExt` trait, +//! we provide an alternative [Initialize] trait which is designed to be more flexible and +//! better suited for machine-learning workloads. +#![cfg(feature = "rand")] + +pub use self::distr::prelude::*; +pub use self::traits::*; +pub use self::utils::*; + +pub(crate) mod traits; +pub(crate) mod utils; + +pub mod initializer; + +pub mod distr { + pub use self::prelude::*; + + pub mod lecun; + pub mod trunc; + pub mod xavier; + + pub(crate) mod prelude { + pub use super::lecun::*; + pub use super::trunc::*; + pub use super::xavier::*; + } +} + +#[doc(no_inline)] +pub use ndarray_rand as ndrand; +#[doc(no_inline)] +pub use rand; +#[doc(no_inline)] +pub use rand_distr; + +pub(crate) mod prelude { + pub use super::distr::prelude::*; + pub use super::traits::{Initialize, InitializeExt}; + pub use super::utils::*; +} diff --git a/core/src/init/traits.rs b/core/src/init/traits.rs new file mode 100644 index 00000000..a01ca7d9 --- /dev/null +++ b/core/src/init/traits.rs @@ -0,0 +1,234 @@ +/* + Appellation: initialize + Contrib: FL03 +*/ +use crate::init::distr::*; + +use core::ops::Neg; +use nd::{ArrayBase, DataOwned, Dimension, RawData, ShapeBuilder}; +use ndrand::RandomExt; +use num::complex::ComplexDistribution; +use num::traits::Float; +use rand::rngs::StdRng; +use rand::{Rng, SeedableRng}; +use rand_distr::uniform::{SampleUniform, Uniform}; +use rand_distr::{Bernoulli, BernoulliError, Distribution, Normal, NormalError, StandardNormal}; + +/// This trait provides the base methods required for initializing an [ndarray](ndarray::ArrayBase) with random values. +/// [Initialize] is similar to [RandomExt](ndarray_rand::RandomExt), however, it focuses on flexibility while implementing additional +/// features geared towards machine-learning models; such as lecun_normal initialization. +pub trait Initialize +where + D: Dimension, +{ + type Data: RawData; + /// Generate a random array using the given distribution + fn rand(shape: Sh, distr: Ds) -> Self + where + Ds: Clone + Distribution, + Sh: ShapeBuilder, + Self::Data: DataOwned; + /// Generate a random array using the given distribution and random number generator + fn rand_with(shape: Sh, distr: Ds, rng: &mut R) -> Self + where + R: Rng + ?Sized, + Ds: Clone + Distribution, + Sh: ShapeBuilder, + Self::Data: DataOwned; + /// Initialize an array with random values using the given distribution and current shape + fn init_rand(self, distr: Ds) -> Self + where + Ds: Clone + Distribution, + Self: Sized, + Self::Data: DataOwned; + /// Initialize an array with random values from the current shape using the given distribution and random number generator + fn init_rand_with(self, distr: Ds, rng: &mut R) -> Self + where + R: Rng + ?Sized, + Ds: Clone + Distribution, + Self::Data: DataOwned; +} + +/// This trait extends the [Initialize] trait with methods for generating random arrays from various distributions. +pub trait InitializeExt: Initialize + Sized +where + A: Clone, + D: Dimension, + S: RawData, +{ + fn bernoulli(shape: Sh, p: f64) -> Result + where + S: DataOwned, + Sh: ShapeBuilder, + Bernoulli: Distribution, + { + let dist = Bernoulli::new(p)?; + Ok(Self::rand(shape, dist)) + } + /// Initialize the object according to the Lecun Initialization scheme. + /// LecunNormal distributions are truncated [Normal](rand_distr::Normal) + /// distributions centered at 0 with a standard deviation equal to the + /// square root of the reciprocal of the number of inputs. + fn lecun_normal(shape: Sh, n: usize) -> Self + where + A: Float, + S: DataOwned, + Sh: ShapeBuilder, + StandardNormal: Distribution, + { + let distr = LecunNormal::new(n); + Self::rand(shape, distr) + } + /// Given a shape, mean, and standard deviation generate a new object using the [Normal](rand_distr::Normal) distribution + fn normal(shape: Sh, mean: A, std: A) -> Result + where + A: Float, + S: DataOwned, + Sh: ShapeBuilder, + StandardNormal: Distribution, + { + let distr = Normal::new(mean, std)?; + Ok(Self::rand(shape, distr)) + } + + fn randc(shape: Sh, re: A, im: A) -> Self + where + S: DataOwned, + Sh: ShapeBuilder, + ComplexDistribution: Distribution, + { + let distr = ComplexDistribution::new(re, im); + Self::rand(shape, distr) + } + /// Generate a random array using the [StandardNormal](rand_distr::StandardNormal) distribution + fn stdnorm(shape: Sh) -> Self + where + S: DataOwned, + Sh: ShapeBuilder, + StandardNormal: Distribution, + { + Self::rand(shape, StandardNormal) + } + /// Generate a random array using the [StandardNormal](rand_distr::StandardNormal) distribution with a given seed + fn stdnorm_from_seed(shape: Sh, seed: u64) -> Self + where + S: DataOwned, + Sh: ShapeBuilder, + StandardNormal: Distribution, + { + Self::rand_with(shape, StandardNormal, &mut StdRng::seed_from_u64(seed)) + } + /// Initialize the object using the [TruncatedNormal](crate::init::distr::TruncatedNormal) distribution + fn truncnorm(shape: Sh, mean: A, std: A) -> Result + where + A: Float, + S: DataOwned, + Sh: ShapeBuilder, + StandardNormal: Distribution, + { + let distr = TruncatedNormal::new(mean, std)?; + Ok(Self::rand(shape, distr)) + } + /// A [uniform](rand_distr::uniform::Uniform) generator with values between u(-dk, dk) + fn uniform(shape: Sh, dk: A) -> Self + where + A: Neg + SampleUniform, + S: DataOwned, + Sh: ShapeBuilder, + ::Sampler: Clone, + { + Self::rand(shape, Uniform::new(dk.clone().neg(), dk)) + } + + fn uniform_from_seed(shape: Sh, start: A, stop: A, key: u64) -> Self + where + A: SampleUniform, + S: DataOwned, + Sh: ShapeBuilder, + ::Sampler: Clone, + { + Self::rand_with( + shape, + Uniform::new(start, stop), + &mut StdRng::seed_from_u64(key), + ) + } + /// Generate a random array with values between u(-a, a) where a is the reciprocal of the value at the given axis + fn uniform_along(shape: Sh, axis: usize) -> Self + where + A: Copy + Float + SampleUniform, + S: DataOwned, + Sh: ShapeBuilder, + ::Sampler: Clone, + { + let dim = shape.into_shape().raw_dim().clone(); + let dk = A::from(dim[axis]).unwrap().recip(); + Self::uniform(dim, dk) + } + /// A [uniform](rand_distr::uniform::Uniform) generator with values between u(-dk, dk) + fn uniform_between(shape: Sh, a: A, b: A) -> Self + where + A: SampleUniform, + S: DataOwned, + Sh: ShapeBuilder, + ::Sampler: Clone, + { + Self::rand(shape, Uniform::new(a, b)) + } +} +/* + ************ Implementations ************ +*/ +impl Initialize for ArrayBase +where + D: Dimension, + S: RawData, + ArrayBase: RandomExt, +{ + type Data = S; + + fn rand(shape: Sh, distr: Ds) -> ArrayBase + where + S: DataOwned, + Ds: Clone + Distribution, + Sh: ShapeBuilder, + { + Self::random(shape, distr) + } + + fn rand_with(shape: Sh, distr: Ds, rng: &mut R) -> ArrayBase + where + R: Rng + ?Sized, + S: DataOwned, + Ds: Clone + Distribution, + Sh: ShapeBuilder, + { + Self::random_using(shape, distr, rng) + } + + fn init_rand(self, distr: Ds) -> ArrayBase + where + S: DataOwned, + Ds: Clone + Distribution, + { + Self::rand(self.dim(), distr) + } + + fn init_rand_with(self, distr: Ds, rng: &mut R) -> ArrayBase + where + R: Rng + ?Sized, + S: DataOwned, + Ds: Clone + Distribution, + { + Self::rand_with(self.dim(), distr, rng) + } +} + +impl InitializeExt for U +where + A: Clone, + D: Dimension, + S: RawData, + U: Initialize, +{ +} diff --git a/core/src/init/utils.rs b/core/src/init/utils.rs new file mode 100644 index 00000000..dacb3df2 --- /dev/null +++ b/core/src/init/utils.rs @@ -0,0 +1,62 @@ +/* + Appellation: utils + Contrib: FL03 +*/ +use ndarray::*; +use ndrand::RandomExt; +use num::complex::{Complex, ComplexDistribution}; +use num::Num; +use rand::distributions::uniform::{SampleUniform, Uniform}; +use rand::rngs::StdRng; +use rand::{rngs, SeedableRng}; +use rand_distr::{Distribution, StandardNormal}; + +/// Generate a random array of complex numbers with real and imaginary parts in the range [0, 1) +pub fn randc(shape: impl IntoDimension) -> ArrayBase +where + A: Clone + Num, + D: Dimension, + S: DataOwned>, + ComplexDistribution: Distribution>, +{ + let distr = ComplexDistribution::::new(A::one(), A::one()); + ArrayBase::random(shape, distr) +} + +/// Given a shape, generate a random array using the StandardNormal distribution +pub fn stdnorm(shape: Sh) -> ArrayBase +where + D: Dimension, + S: DataOwned, + Sh: ShapeBuilder, + StandardNormal: Distribution, +{ + ArrayBase::random(shape, StandardNormal) +} + +pub fn stdnorm_from_seed(shape: Sh, seed: u64) -> ArrayBase +where + D: Dimension, + S: DataOwned, + Sh: ShapeBuilder, + StandardNormal: Distribution, +{ + ArrayBase::random_using(shape, StandardNormal, &mut StdRng::seed_from_u64(seed)) +} +/// Creates a random array from a uniform distribution using a given key +pub fn uniform_from_seed( + key: u64, + start: T, + stop: T, + shape: impl IntoDimension, +) -> Array +where + D: Dimension, + T: SampleUniform, +{ + Array::random_using( + shape, + Uniform::new(start, stop), + &mut rngs::StdRng::seed_from_u64(key), + ) +} diff --git a/core/src/lib.rs b/core/src/lib.rs index b7ee82c7..a09d48e1 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -16,7 +16,7 @@ pub use self::nn::Module; pub use self::{primitives::*, traits::prelude::*, types::prelude::*, utils::prelude::*}; #[cfg(feature = "rand")] -pub use self::rand::{GenerateRandom, RandomExt}; +pub use self::init::{Initialize, InitializeExt}; #[macro_use] pub(crate) mod macros; @@ -24,26 +24,27 @@ pub(crate) mod primitives; pub mod error; pub mod func; +pub mod init; +pub mod math; pub mod nn; pub mod ops; -pub mod params; -#[cfg(feature = "rand")] -pub mod rand; + pub mod traits; pub mod types; pub mod utils; pub mod prelude { + #[allow(unused_imports)] pub(crate) use super::primitives::rust::*; pub use super::error::prelude::*; pub use super::func::prelude::*; + #[cfg(feature = "rand")] + pub use super::init::prelude::*; + pub use super::math::prelude::*; pub use super::nn::prelude::*; pub use super::ops::prelude::*; - pub use super::params::prelude::*; pub use super::primitives::*; - #[cfg(feature = "rand")] - pub use super::rand::prelude::*; pub use super::traits::prelude::*; pub use super::types::prelude::*; pub use super::utils::prelude::*; diff --git a/core/src/macros.rs b/core/src/macros.rs index b447975f..aaf6ceb6 100644 --- a/core/src/macros.rs +++ b/core/src/macros.rs @@ -2,115 +2,78 @@ Appellation: macros Contrib: FL03 */ -#![allow(unused_macros)] +#[macro_use] +mod activate; +#[macro_use] +mod builder; +#[macro_use] +mod enums; +#[macro_use] +mod getters; +#[macro_use] +mod ops; +#[macro_use] +mod toggle; -macro_rules! error_from { - ($base:ident::$variant:ident<$($err:ty),* $(,)?>) => { - error_from!(@loop $base::$variant<$($err),*>); - }; - ($base:ident::$variant:ident<$err:ty>$($rest:tt)*) => { - error_from!(@loop $base::$variant<$($err),*>$($rest)*); - }; - (@loop $base:ident::$variant:ident<$($err:ty),* $(,)?>) => { - $( - error_from!(@impl $base::$variant<$err>); - )* - }; - (@impl $base:ident::$variant:ident<$err:ty>) => { - impl From<$err> for $base { - fn from(err: $err) -> Self { - Self::$variant(err.to_string()) - } - } - }; - (@impl $base:ident::$variant:ident<$err:ty>.$method:ident) => { - impl From<$err> for $base { - fn from(err: $err) -> Self { - Self::$variant(err.$method()) - } - } - }; -} +/// AS +#[macro_export] +macro_rules! dimensional { -macro_rules! nested_constructor { - ($variant:ident<$inner:ident>, $method:ident, [$($call:ident),*]) => { - nested_constructor!(@loop $variant<$inner>, $method, [$($call),*]); - }; - (@loop $variant:ident<$inner:ident>, $method:ident, [$($call:ident),*]) => { - pub fn $method(inner:$inner) -> Self { - Self::$variant(inner) + (dim: $name:ident$(())?) => { + /// Returns a reference to the current dimension, as a slice. + pub fn as_slice(&self) -> &[usize] { + self.$name$(())?.shape() } - $( - pub fn $call() -> Self { - Self::$method($inner::$call()) - } - )* - - }; -} - -macro_rules! variant_constructor { - ($($rest:tt),* $(,)?) => { - $( - variant_constructor!(@loop $($rest),*); - )* - }; - ($($variant:ident.$method:ident$(($call:expr))?),* $(,)?) => { - $( - variant_constructor!(@loop $variant.$method$(($call))?); - )* - }; - - (@loop $variant:ident.$method:ident$(($call:expr))?) => { - pub fn $method() -> Self { - Self::$variant$(($call))? + pub fn into_pattern(self) -> D::Pattern { + self.$name$(())?.into_pattern() } - }; -} -macro_rules! impl_unary { - ($name:ident.$call:ident<$T:ty>($f:expr) $($rest:tt)*) => { - impl_unary!(@impl $name.$call<$T>($f) $($rest)*); - }; - (@impl $name:ident.$call:ident<$T:ty>($f:expr)) => { - impl $name for $T { - type Output = $T; + pub fn ndim(&self) -> usize { + self.$name$(())?.ndim() + } - fn $call(&self) -> Self::Output { - $f(self) - } + pub fn raw_dim(&self) -> D { + self.$name$(())?.dim().clone() } }; -} -macro_rules! build_unary_trait { - ($($name:ident.$call:ident),* $(,)?) => { - $( - build_unary_trait!(@impl $name.$call); - )* - }; - (@impl $name:ident.$call:ident) => { - pub trait $name { - type Output; - fn $call(&self) -> Self::Output; + ($name:ident) => { + /// Return the [pattern](ndarray::Dimension::Pattern) of the dimension + pub fn dim(&self) -> D::Pattern { + self.$name.dim() + } + /// Returns rank (ndim) of the dimension + pub fn ndim(&self) -> usize { + self.$name.ndim() + } + /// Returns the raw dimension [D](ndarray::Dimension) + pub fn raw_dim(&self) -> D { + self.$name.dim() + } + /// Returns a reference to the current dimension, as a slice. + pub fn shape(&self) -> &[usize] { + self.$name.shape() } }; -} -macro_rules! linspace { - (start: $start:expr, end: $end:expr, n: $n:expr, dtype: $T:ty) => { - ndarray::Array1::<$T>::linspace($start, $end, $n) - }; - (end: $end:expr, dtype: $T:ty) => { - let n = ($end - $T::one()).to_usize().unwrap(); - ndarray::Array1::<$T>::linspace($T::zero(), $end, $end.to_usize().unwrap()) + ($name:ident()) => { + /// Return the [pattern](ndarray::Dimension::Pattern) of the dimension + pub fn dim(&self) -> D::Pattern { + self.$name().dim() + } + /// Returns rank (ndim) of the dimension + pub fn ndim(&self) -> usize { + self.$name().ndim() + } + /// Returns the raw dimension [D](ndarray::Dimension) + pub fn raw_dim(&self) -> D { + self.$name().raw_dim() + } + /// Returns a reference to the current dimension, as a slice. + pub fn shape(&self) -> &[usize] { + self.$name().shape() + } }; - (dim: $dim:expr, dtype: $T:ty) => {{ - let dim = $dim.into_dimension(); - let n = dim.size(); - ndarray::Array1::<$T>::linspace(<$T>::zero(), <$T>::from(n - 1).unwrap(), n) - .into_shape($dim) - }}; } diff --git a/core/src/macros/activate.rs b/core/src/macros/activate.rs new file mode 100644 index 00000000..44e6eeb3 --- /dev/null +++ b/core/src/macros/activate.rs @@ -0,0 +1,36 @@ +/* + Appellation: activate + Contrib: FL03 +*/ + +macro_rules! activator { + ($name:ident::<$out:ty>($rho:expr) $($rest:tt)*) => { + #[derive(Clone, Copy, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] + #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] + pub struct $name; + + impl $crate::func::activate::Activate for $name $($rest)* { + type Output = $out; + + fn activate(&self, args: &T) -> Self::Output { + $rho(args) + } + } + }; +} + +macro_rules! losses { + (impl<$($T:ident),* $(,)?> $name:ident::<$lhs:ty, $rhs:ty, Output = $out:ty>($loss:expr) $($rest:tt)*) => { + #[derive(Clone, Copy, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] + #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] + pub struct $name; + + impl<$($T),*> $crate::func::Loss<$lhs, $rhs> for $name $($rest)* { + type Output = $out; + + fn loss(&self, a: &$lhs, b: &$rhs) -> Self::Output { + $loss(a, b) + } + } + }; +} diff --git a/core/src/macros/builder.rs b/core/src/macros/builder.rs new file mode 100644 index 00000000..6e545c9f --- /dev/null +++ b/core/src/macros/builder.rs @@ -0,0 +1,44 @@ +/* + Appellation: builder + Contrib: FL03 +*/ + +#[macro_export] +macro_rules! builder { + ($(#[derive($($d:ident),+)])? $name:ident($inner:ty) {$($k:ident: $v:ty),* $(,)?}) => { + $crate::builder!(@loop builder: $name, derive: [$($($d),+)?], inner: $inner {$($k: $v),*}); + }; + (@loop builder: $name:ident, derive: [$($d:ident),* $(,)?], inner: $inner:ty {$($k:ident: $v:ty),* $(,)?}) => { + + #[derive(Default, $($d),*)] + pub struct $name { + inner: $inner, + } + + $crate::builder!(@impl builder: $name, inner: $inner {$($k: $v),*}); + }; + (@impl builder: $name:ident, inner: $inner:ty {$($k:ident: $v:ty),* $(,)?}) => { + impl $name { + pub fn new() -> Self { + Self { + inner: Default::default() + } + } + + pub fn from_inner(inner: $inner) -> Self { + Self { inner } + } + + pub fn build(self) -> $inner { + self.inner + } + + $( + pub fn $k(mut self, $k: $v) -> Self { + self.inner.$k = $k; + self + } + )* + } + }; +} diff --git a/core/src/macros/enums.rs b/core/src/macros/enums.rs new file mode 100644 index 00000000..ce08a85a --- /dev/null +++ b/core/src/macros/enums.rs @@ -0,0 +1,44 @@ +/* + Appellation: enums + Contrib: FL03 +*/ + +macro_rules! from_variant { + ($base:ident::$variant:ident $($rest:tt)*) => { + from_variant!(@branch $base::$variant $($rest)*); + }; + (@branch $base:ident::$variant:ident($from:ty)$(.$method:ident())*) => { + from_variant!(@impl $base::$variant($from)$(.$method())*); + }; + (@branch $base:ident::$variant:ident{$(<$err:ty>$(.$method:ident())*),* $(,)?}) => { + $( + from_variant!(@impl $base::$variant($err)$(.$method())*); + )* + }; + (@impl $base:ident::$variant:ident($from:ty)$(.$method:ident())*) => { + impl From<$from> for $base { + fn from(val: $from) -> Self { + Self::$variant(val$(.$method())*) + } + } + }; +} + +#[allow(unused_macros)] +macro_rules! nested_enum_constructor { + ($variant:ident<$inner:ident>, $method:ident, [$($call:ident),*]) => { + nested_enum_constructor!(@loop $variant<$inner>, $method, [$($call),*]); + }; + (@loop $variant:ident<$inner:ident>, $method:ident, [$($call:ident),*]) => { + pub fn $method(inner:$inner) -> Self { + Self::$variant(inner) + } + + $( + pub fn $call() -> Self { + Self::$method($inner::$call()) + } + )* + + }; +} diff --git a/core/src/macros/getters.rs b/core/src/macros/getters.rs new file mode 100644 index 00000000..86d232b4 --- /dev/null +++ b/core/src/macros/getters.rs @@ -0,0 +1,47 @@ +/* + Appellation: getters + Contrib: FL03 +*/ + +#[macro_export] +macro_rules! getters { + ($($call:ident$(.$field:ident)?<$out:ty>),* $(,)?) => { + $($crate::getters!(@impl $call$(.$field)?<$out>);)* + }; + ($via:ident::<[$($call:ident$(.$field:ident)?<$out:ty>),* $(,)?]>) => { + $($crate::getters!(@impl $via::$call$(.$field)?<$out>);)* + }; + ($($call:ident$(.$field:ident)?),* $(,)? => $out:ty) => { + $($crate::getters!(@impl $call$(.$field)?<$out>);)* + }; + ($via:ident::<[$($call:ident$(.$field:ident)?),* $(,)?]> => $out:ty) => { + $crate::getters!($via::<[$($call$(.$field)?<$out>),*]>); + }; + + (@impl $call:ident<$out:ty>) => { + $crate::getters!(@impl $call.$call<$out>); + }; + (@impl $via:ident::$call:ident<$out:ty>) => { + $crate::getters!(@impl $via::$call.$call<$out>); + }; + (@impl $call:ident.$field:ident<$out:ty>) => { + pub fn $call(&self) -> &$out { + &self.$field + } + paste::paste! { + pub fn [< $call _mut>](&mut self) -> &mut $out { + &mut self.$field + } + } + }; + (@impl $via:ident::$call:ident.$field:ident<$out:ty>) => { + pub fn $call(&self) -> &$out { + &self.$via.$field + } + paste::paste! { + pub fn [< $call _mut>](&mut self) -> &mut $out { + &mut self.$via.$field + } + } + }; +} diff --git a/core/src/macros/ops.rs b/core/src/macros/ops.rs new file mode 100644 index 00000000..f37089b0 --- /dev/null +++ b/core/src/macros/ops.rs @@ -0,0 +1,36 @@ +/* + Appellation: ops + Contrib: FL03 +*/ + +macro_rules! unary { + ($($name:ident::$call:ident),* $(,)?) => { + $( + unary!(@impl $name::$call(self)); + )* + }; + ($($name:ident::$call:ident(self)),* $(,)?) => { + $( + unary!(@impl $name::$call(self)); + )* + }; + ($($name:ident::$call:ident(&self)),* $(,)?) => { + $( + unary!(@impl $name::$call(&self)); + )* + }; + (@impl $name:ident::$call:ident(self)) => { + pub trait $name { + type Output; + + fn $call(self) -> Self::Output; + } + }; + (@impl $name:ident::$call:ident(&self)) => { + pub trait $name { + type Output; + + fn $call(&self) -> Self::Output; + } + }; +} diff --git a/core/src/macros/toggle.rs b/core/src/macros/toggle.rs new file mode 100644 index 00000000..908f6abf --- /dev/null +++ b/core/src/macros/toggle.rs @@ -0,0 +1,19 @@ +/* + Appellation: toggle + Contrib: FL03 +*/ + +#[macro_export] +macro_rules! toggle { + (enum $($name:ident),* $(,)?) => { + $(toggle!(@enum $name);)* + }; + + (@enum $name:ident) => { + #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] + #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] + pub enum $name {} + + impl $crate::traits::misc::toggle::Toggle for $name {} + }; +} diff --git a/core/src/math/arith.rs b/core/src/math/arith.rs new file mode 100644 index 00000000..04c88c1a --- /dev/null +++ b/core/src/math/arith.rs @@ -0,0 +1,68 @@ +/* + Appellation: arith + Contrib: FL03 +*/ +use num::integer::Roots; +use num::traits::FromPrimitive; + +pub trait Root { + type Output; + + fn nth_root(&self, n: u32) -> Self::Output; + + fn sqrt(&self) -> Self::Output { + self.nth_root(2) + } + + fn cbrt(&self) -> Self::Output { + self.nth_root(3) + } +} + +macro_rules! impl_root { + (float $($T:ty),* $(,)?) => { + $( + impl_root!(@float $T); + )* + }; + ($($T:ty),* $(,)?) => { + $( + impl_root!(@impl $T); + )* + }; + + (@impl $T:ty) => { + impl Root for $T { + type Output = $T; + + fn nth_root(&self, n: u32) -> Self::Output { + Roots::nth_root(self, n) + } + } + }; + (@float $T:ty) => { + impl Root for $T { + type Output = $T; + + fn nth_root(&self, n: u32) -> Self::Output { + self.powf(<$T>::from_u32(n).unwrap().recip()) + } + } + }; +} + +impl_root!(float f32, f64); +impl_root! { + i8, + i16, + i32, + i64, + i128, + isize, + u8, + u16, + u32, + u64, + u128, + usize, +} diff --git a/core/src/math/mod.rs b/core/src/math/mod.rs new file mode 100644 index 00000000..da193f09 --- /dev/null +++ b/core/src/math/mod.rs @@ -0,0 +1,19 @@ +/* + Appellation: math + Contrib: FL03 +*/ +//! # Mathematics +//! +//! This module focuses on providing the mathematical foundation for the library. +//! Any defined operation is designed to extend the functionality of the basic primitives +//! as well as the `ndarray` crate. +pub use self::traits::*; + +pub mod arith; +pub mod stats; +pub mod traits; + +pub(crate) mod prelude { + pub use super::stats::prelude::*; + pub use super::traits::*; +} diff --git a/core/src/math/stats/mod.rs b/core/src/math/stats/mod.rs new file mode 100644 index 00000000..7a0a3892 --- /dev/null +++ b/core/src/math/stats/mod.rs @@ -0,0 +1,13 @@ +/* + Appellation: stats + Contrib: FL03 +*/ +//! # Statistics +//! +pub use self::summary::*; + +mod summary; + +pub(crate) mod prelude { + pub use super::summary::*; +} diff --git a/core/src/math/stats/summary.rs b/core/src/math/stats/summary.rs new file mode 100644 index 00000000..35b5821d --- /dev/null +++ b/core/src/math/stats/summary.rs @@ -0,0 +1,151 @@ +/* + Appellation: summary + Contrib: FL03 +*/ +use crate::math::arith::Root; +use core::iter::{Product, Sum}; +use nd::{ArrayBase, Data, Dimension}; +use num::traits::{FromPrimitive, Num, NumOps, Pow}; + +/// This trait describes the fundamental methods of summary statistics. +/// These include the mean, standard deviation, variance, and more. +pub trait SummaryStatistics +where + Self::Item: FromPrimitive, + Self::Output: NumOps, +{ + type Item; + type Output; + + fn elems(&self) -> Self::Item { + Self::Item::from_usize(self.len()).unwrap() + } + + fn len(&self) -> usize; + + fn mean(&self) -> Self::Output { + self.sum() / self.elems() + } + + fn product(&self) -> Self::Output; + + fn sum(&self) -> Self::Output; + + fn std(&self) -> Self::Output; + + fn var(&self) -> Self::Output; +} + +/* + ************* Implementations ************* +*/ +impl<'a, T, I> SummaryStatistics for &'a I +where + I: Clone + ExactSizeIterator, + T: Copy + FromPrimitive + Num + Pow + Product + Root + Sum, +{ + type Item = T; + type Output = T; + + fn len(&self) -> usize { + ExactSizeIterator::len(*self) + } + + fn product(&self) -> Self::Output { + (*self).clone().product() + } + + fn sum(&self) -> Self::Output { + (*self).clone().sum() + } + + fn std(&self) -> Self::Output { + let mean = self.mean(); + let sum = (*self).clone().map(|x| (x - mean).pow(2)).sum::(); + (sum / self.elems()).sqrt() + } + + fn var(&self) -> Self::Output { + let mean = self.mean(); + let sum = (*self).clone().map(|x| (x - mean).pow(2)).sum::(); + sum / self.elems() + } +} + +macro_rules! impl_summary { + ($($T:ty),* $(,)?) => { + $( + impl_summary!(@impl $T); + )* + }; + (@impl $T:ty) => { + + impl SummaryStatistics for $T + where + T: Copy + FromPrimitive + Num + Pow + Product + Root + Sum, + { + type Item = T; + type Output = T; + + fn len(&self) -> usize { + self.len() + } + + fn product(&self) -> Self::Output { + self.iter().copied().product::() + } + + fn sum(&self) -> Self::Output { + self.iter().copied().sum::() + } + + fn std(&self) -> Self::Output { + let mean = self.mean(); + let sum = self.iter().copied().map(|x| (x - mean).pow(2)).sum::(); + (sum / self.elems()).sqrt() + } + + fn var(&self) -> Self::Output { + let mean = self.mean(); + let sum = self.iter().copied().map(|x| (x - mean).pow(2)).sum::(); + sum / self.elems() + } + } + }; +} + +impl_summary!(Vec, [T]); + +impl SummaryStatistics for ArrayBase +where + A: Copy + FromPrimitive + Num + Pow + Product + Root + Sum, + D: Dimension, + S: Data, +{ + type Item = A; + type Output = A; + + fn len(&self) -> usize { + self.len() + } + + fn product(&self) -> Self::Output { + self.iter().copied().product::() + } + + fn sum(&self) -> Self::Output { + self.iter().copied().sum::() + } + + fn std(&self) -> Self::Output { + let mean = self.mean().unwrap_or_else(A::zero); + let sum = self.iter().copied().map(|x| (x - mean).pow(2)).sum::(); + (sum / self.elems()).sqrt() + } + + fn var(&self) -> Self::Output { + let mean = self.mean().unwrap_or_else(A::zero); + let sum = self.iter().copied().map(|x| (x - mean).pow(2)).sum::(); + sum / self.elems() + } +} diff --git a/core/src/math/traits.rs b/core/src/math/traits.rs new file mode 100644 index 00000000..d71d433d --- /dev/null +++ b/core/src/math/traits.rs @@ -0,0 +1,152 @@ +/* + Appellation: traits + Contrib: FL03 +*/ +use nd::{Array, ArrayBase, Data, Dimension}; +use num::complex::{Complex, ComplexFloat}; +use num::traits::Signed; + +unary!( + Abs::abs(self), + Cos::cos(self), + Cosh::cosh(self), + Exp::exp(self), + Sine::sin(self), + Sinh::sinh(self), + Squared::sqrd(self), + SquareRoot::sqrt(self) +); + +/* + ********* Implementations ********* +*/ + +macro_rules! unary_impl { + ($name:ident::$method:ident<[$($T:ty),* $(,)?]>) => { + unary_impl!(@loop $name::$method<[$($T),*]>); + }; + ($($name:ident::$method:ident<$T:ty$(, Output = $O:ty)?>),* $(,)?) => { + $(unary_impl!(@impl $name::$method<$T$(, Output = $O>)?);)* + }; + ($($name:ident::$method:ident<$T:ty, Output = $O:ty>),* $(,)?) => { + $(unary_impl!(@impl $name::$method<$T, Output = $O>);)* + }; + (@loop $name:ident::$method:ident<[$($T:ty),* $(,)?]>) => { + $(unary_impl!(@impl $name::$method<$T>);)* + }; + (@impl $name:ident::$method:ident<$T:ty>) => { + unary_impl!(@impl $name::$method<$T, Output = $T>); + }; + (@impl $name:ident::$method:ident<$T:ty, Output = $O:ty>) => { + impl $name for $T { + type Output = $O; + + fn $method(self) -> Self::Output { + <$T>::$method(self) + } + } + }; +} + +macro_rules! unary_impls { + ($($name:ident::$method:ident<[$($T:ty),* $(,)?]>),* $(,)?) => { + $(unary_impl!(@loop $name::$method<[$($T),*]>);)* + }; +} + +unary_impls!( + Abs::abs<[f32, f64]>, + Cosh::cosh<[f32, f64, Complex, Complex]>, + Cos::cos<[f32, f64, Complex, Complex]>, + Exp::exp<[f32, f64, Complex, Complex]>, + Sinh::sinh<[f32, f64, Complex, Complex]>, + Sine::sin<[f32, f64, Complex, Complex]>, + SquareRoot::sqrt<[f32, f64]> +); + +impl Abs for ArrayBase +where + A: Clone + Signed, + D: Dimension, + S: Data, +{ + type Output = Array; + + fn abs(self) -> Self::Output { + self.mapv(|x| x.abs()) + } +} + +impl<'a, A, S, D> Abs for &'a ArrayBase +where + A: Clone + Signed, + D: Dimension, + S: Data, +{ + type Output = Array; + + fn abs(self) -> Self::Output { + self.mapv(|x| x.abs()) + } +} + +impl Squared for A +where + A: Clone + core::ops::Mul, +{ + type Output = A; + + fn sqrd(self) -> Self::Output { + self.clone() * self + } +} + +impl SquareRoot for Complex +where + Complex: ComplexFloat, +{ + type Output = Self; + + fn sqrt(self) -> Self::Output { + ComplexFloat::sqrt(self) + } +} + +impl SquareRoot for ArrayBase +where + A: Clone + SquareRoot, + D: Dimension, + S: Data, +{ + type Output = Array; + + fn sqrt(self) -> Self::Output { + self.mapv(|x| x.sqrt()) + } +} + +impl Exp for ArrayBase +where + A: Clone + Exp, + D: Dimension, + S: Data, +{ + type Output = Array; + + fn exp(self) -> Self::Output { + self.mapv(|x| x.exp()) + } +} + +impl<'a, A, S, D> Exp for &'a ArrayBase +where + A: Clone + ComplexFloat, + D: Dimension, + S: Data, +{ + type Output = Array; + + fn exp(self) -> Self::Output { + self.mapv(|x| x.exp()) + } +} diff --git a/core/src/nn/dropout.rs b/core/src/nn/dropout.rs new file mode 100644 index 00000000..19acdbc0 --- /dev/null +++ b/core/src/nn/dropout.rs @@ -0,0 +1,96 @@ +/* + Appellation: dropout + Contrib: FL03 +*/ +#![allow(unused_imports)] +use crate::Forward; +use nd::prelude::*; +use nd::{DataOwned, ScalarOperand}; +#[cfg(feature = "rand")] +use ndrand::{rand_distr::Bernoulli, RandomExt}; +use num::traits::Num; + +#[cfg(feature = "rand")] +pub fn dropout(array: &ArrayBase, p: f64) -> Array +where + A: Num + ScalarOperand, + D: Dimension, + S: DataOwned, +{ + // Create a Bernoulli distribution for dropout + let distribution = Bernoulli::new(p).unwrap(); + + // Create a mask of the same shape as the input array + let mask: Array = Array::random(array.dim(), distribution); + let mask = mask.mapv(|x| if x { A::zero() } else { A::one() }); + + // Element-wise multiplication to apply dropout + array * mask +} + +/// [Dropout] randomly zeroizes elements with a given probability (`p`). +pub trait Dropout { + type Output; + + fn dropout(&self, p: f64) -> Self::Output; +} + +/// The [DropoutLayer] layer is randomly zeroizes inputs with a given probability (`p`). +/// This regularization technique is often used to prevent overfitting. +/// +/// +/// ### Config +/// +/// - (p) Probability of dropping an element +#[derive(Clone, Copy, Debug, PartialEq, PartialOrd)] +#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] +pub struct DropoutLayer { + pub(crate) p: f64, +} + +/* + ************* Implementations ************* +*/ +#[cfg(feature = "rand")] +impl Dropout for ArrayBase +where + A: Num + ScalarOperand, + D: Dimension, + S: DataOwned, +{ + type Output = Array; + + fn dropout(&self, p: f64) -> Self::Output { + dropout(self, p) + } +} + +impl DropoutLayer { + pub fn new(p: f64) -> Self { + Self { p } + } + + pub fn scale(&self) -> f64 { + (1f64 - self.p).recip() + } +} + +impl Default for DropoutLayer { + fn default() -> Self { + Self::new(0.5) + } +} + +#[cfg(feature = "rand")] +impl Forward> for DropoutLayer +where + A: Num + ScalarOperand, + D: Dimension, + S: DataOwned, +{ + type Output = Array; + + fn forward(&self, input: &ArrayBase) -> Self::Output { + input.dropout(self.p) + } +} diff --git a/core/src/nn/mask/mask.rs b/core/src/nn/mask/mask.rs new file mode 100644 index 00000000..94da711a --- /dev/null +++ b/core/src/nn/mask/mask.rs @@ -0,0 +1,236 @@ +/* + Appellation: mask + Contrib: FL03 +*/ +use nd::iter::{Iter, IterMut}; +use nd::prelude::*; +use nd::{Data, DataMut, OwnedRepr, RawData, RawDataClone}; + +pub struct Mask, D = Ix2>(ArrayBase) +where + D: Dimension, + S: RawData; + +impl Mask +where + D: Dimension, + S: RawData, +{ + pub fn from_arr(data: ArrayBase) -> Self { + Self(data) + } + + pub fn apply(&mut self, data: &ArrayBase, fill: A) -> ArrayBase + where + A: Clone, + S: Data, + T: DataMut + RawDataClone, + { + let mut res = data.clone(); + res.zip_mut_with(self.as_mut(), |x, &m| { + if m { + *x = fill.clone(); + } + }); + res + } + + pub fn mask_inplace<'a, A, T, F>( + &mut self, + data: &'a mut ArrayBase, + fill: A, + ) -> &'a mut ArrayBase + where + A: Clone, + S: Data, + T: DataMut, + { + data.zip_mut_with(&mut self.0, |x, &m| { + if m { + *x = fill.clone(); + } + }); + data + } + + pub fn as_slice(&self) -> &[bool] + where + S: Data, + { + self.get().as_slice().unwrap() + } + + pub fn as_mut_slice(&mut self) -> &mut [bool] + where + S: DataMut, + { + self.get_mut().as_slice_mut().unwrap() + } + + pub fn dim(&self) -> D::Pattern { + self.get().dim() + } + + pub fn iter(&self) -> Iter<'_, bool, D> + where + S: Data, + { + self.get().iter() + } + + pub fn iter_mut(&mut self) -> IterMut<'_, bool, D> + where + S: DataMut, + { + self.get_mut().iter_mut() + } + + pub fn get(&self) -> &ArrayBase { + &self.0 + } + + pub fn get_mut(&mut self) -> &mut ArrayBase { + &mut self.0 + } + + pub fn into_inner(self) -> ArrayBase { + self.0 + } + + pub fn ndim(&self) -> usize { + self.get().ndim() + } + + pub fn raw_dim(&self) -> D { + self.get().raw_dim() + } + + pub fn set(&mut self, data: ArrayBase) { + self.0 = data; + } + + pub fn shape(&self) -> D { + self.get().raw_dim() + } +} + +/* + ************* Implementations ************* +*/ +mod impls { + use super::Mask; + use core::borrow::{Borrow, BorrowMut}; + use core::ops::{Deref, DerefMut, Index, IndexMut}; + use nd::{ArrayBase, Data, DataMut, Dimension, NdIndex, RawData}; + + impl AsRef> for Mask + where + D: Dimension, + S: RawData, + { + fn as_ref(&self) -> &ArrayBase { + &self.0 + } + } + + impl AsMut> for Mask + where + D: Dimension, + S: RawData, + { + fn as_mut(&mut self) -> &mut ArrayBase { + &mut self.0 + } + } + + impl Borrow> for Mask + where + D: Dimension, + S: RawData, + { + fn borrow(&self) -> &ArrayBase { + &self.0 + } + } + + impl BorrowMut> for Mask + where + D: Dimension, + S: RawData, + { + fn borrow_mut(&mut self) -> &mut ArrayBase { + &mut self.0 + } + } + + impl Deref for Mask + where + D: Dimension, + S: RawData, + { + type Target = ArrayBase; + + fn deref(&self) -> &Self::Target { + &self.0 + } + } + + impl DerefMut for Mask + where + D: Dimension, + S: RawData, + { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } + } + + impl Index for Mask + where + D: Dimension, + I: NdIndex, + S: Data, + { + type Output = as Index>::Output; + + fn index(&self, index: I) -> &Self::Output { + &self.0[index] + } + } + + impl IndexMut for Mask + where + D: Dimension, + I: NdIndex, + S: DataMut, + { + fn index_mut(&mut self, index: I) -> &mut Self::Output { + &mut self.0[index] + } + } +} + +mod impl_from { + use super::Mask; + use nd::{ArrayBase, Dimension, RawData}; + + impl From> for Mask + where + D: Dimension, + S: RawData, + { + fn from(mask: ArrayBase) -> Self { + Mask(mask) + } + } + + impl From> for ArrayBase + where + D: Dimension, + S: RawData, + { + fn from(mask: Mask) -> Self { + mask.0 + } + } +} diff --git a/core/src/nn/mask/mod.rs b/core/src/nn/mask/mod.rs new file mode 100644 index 00000000..5a3aaa2b --- /dev/null +++ b/core/src/nn/mask/mod.rs @@ -0,0 +1,29 @@ +/* + Appellation: mask + Contrib: FL03 +*/ +pub use self::mask::*; + +pub(crate) mod mask; + +pub(crate) mod prelude { + pub use super::mask::Mask; + pub use super::NdMask; +} + +use nd::{ArrayBase, Dimension, Ix2, RawData}; + +pub trait NdMask +where + D: Dimension, +{ + type Data: RawData; +} + +impl NdMask for ArrayBase +where + D: Dimension, + S: RawData, +{ + type Data = S; +} diff --git a/core/src/nn/mod.rs b/core/src/nn/mod.rs index b82d7727..172b7e7d 100644 --- a/core/src/nn/mod.rs +++ b/core/src/nn/mod.rs @@ -2,14 +2,28 @@ Appellation: nn Contrib: FL03 */ -pub use self::{error::ModelError, models::Module}; +#[cfg(any(feature = "alloc", feature = "std"))] +pub use self::types::*; +pub use self::{dropout::*, error::ModelError, model::prelude::*}; +pub mod dropout; pub mod error; -pub mod models; +pub mod mask; +pub mod model; pub(crate) mod prelude { - pub use super::error::ModelError; - pub use super::models::prelude::*; + pub use super::dropout::*; + pub use super::error::*; + pub use super::mask::prelude::*; + pub use super::model::prelude::*; +} + +#[cfg(any(feature = "alloc", feature = "std"))] +mod types { + use crate::rust::Box; + use nd::prelude::Array2; + + pub type ForwardDyn, O = T> = Box>; } #[cfg(test)] diff --git a/core/src/nn/model.rs b/core/src/nn/model.rs new file mode 100644 index 00000000..316d03e7 --- /dev/null +++ b/core/src/nn/model.rs @@ -0,0 +1,59 @@ +/* + Appellation: model + Contrib: FL03 +*/ +pub use self::module::*; + +pub mod config; +pub mod module; +#[doc(hidden)] +pub mod repo; + +pub(crate) mod prelude { + pub use super::config::*; + pub use super::module::*; + pub use super::Model; +} + +use crate::traits::Forward; + +pub trait Model: Module +where + Self: Forward, +{ + type Ctx; + type Data; + + fn children(&self) -> Vec>; + + fn context(&self) -> Self::Ctx; +} + +/// This trait describes any neural networks or models that +/// adhears to the deep netural network architecture. +/// This design considers a single input and output layer, while +/// allowing for any number of hidden layers to be persisted. +/// +/// The `HIDDEN` constant is used to specify the number of hidden layers +/// and is used to compute the total number of layers (HIDDEN + 2) +pub trait DeepNeuralNetwork: Forward { + const HIDDEN: Option = None; + + type Input: Forward; + type Hidden: Forward; // The type of `hidden` layers; all hidden layers implement the same activation function + type Out: Forward; + + fn input(&self) -> &Self::Input; + + fn hidden(&self) -> &[Self::Hidden]; + + fn output(&self) -> &Self::Out; + + fn nlayers(&self) -> usize { + self.nhidden() + 2 + } + + fn nhidden(&self) -> usize { + Self::HIDDEN.unwrap_or_else(|| self.hidden().len()) + } +} diff --git a/core/src/nn/model/config.rs b/core/src/nn/model/config.rs new file mode 100644 index 00000000..7596c36b --- /dev/null +++ b/core/src/nn/model/config.rs @@ -0,0 +1,18 @@ +/* + Appellation: config + Contrib: FL03 +*/ +use crate::traits::Config; + +pub struct ModelConfig { + pub name: String, + _children: Vec>, +} + +impl Config for ModelConfig {} + +#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize,))] +pub struct ConfigBase { + pub id: usize, + pub name: &'static str, +} diff --git a/core/src/nn/models/module.rs b/core/src/nn/model/module.rs similarity index 72% rename from core/src/nn/models/module.rs rename to core/src/nn/model/module.rs index af182f25..8d8ee23b 100644 --- a/core/src/nn/models/module.rs +++ b/core/src/nn/model/module.rs @@ -2,13 +2,17 @@ Appellation: modules Contrib: FL03 */ -use crate::Predict; +use crate::{Config, Predict}; + +pub type ModuleDyn = Box>; +pub type DynModuleExt = Box>; +pub type Stack = Vec>>; /// A `Module` defines any object that may be used as a layer in a neural network. /// [Config](Module::Config) is a type that defines the configuration of the module; including any and all hyperparameters. /// [Params](Module::Params) is a type that defines the parameters of the module; typically references a Linear set of parameters { weights, bias } pub trait Module { - type Config; + type Config: Config; type Params; fn config(&self) -> &Self::Config; @@ -20,4 +24,4 @@ pub trait Module { pub trait ModuleExt: Module + Predict {} -pub type Stack = Vec>>; +impl ModuleExt for M where M: Module + Predict {} diff --git a/core/src/nn/model/repo.rs b/core/src/nn/model/repo.rs new file mode 100644 index 00000000..affd401a --- /dev/null +++ b/core/src/nn/model/repo.rs @@ -0,0 +1,10 @@ +/* + Appellation: repo + Contrib: FL03 +*/ +#![allow(unused)] + +pub struct ModelRepo { + pub name: String, + pub(crate) store: String, +} diff --git a/core/src/nn/models/mod.rs b/core/src/nn/models/mod.rs deleted file mode 100644 index 37e9ebe7..00000000 --- a/core/src/nn/models/mod.rs +++ /dev/null @@ -1,13 +0,0 @@ -/* - Appellation: models - Contrib: FL03 -*/ -pub use self::{model::*, module::*}; - -pub mod model; -pub mod module; - -pub(crate) mod prelude { - pub use super::model::*; - pub use super::module::*; -} diff --git a/core/src/nn/models/model.rs b/core/src/nn/models/model.rs deleted file mode 100644 index 16687d4b..00000000 --- a/core/src/nn/models/model.rs +++ /dev/null @@ -1,6 +0,0 @@ -/* - Appellation: model - Contrib: FL03 -*/ - -pub trait Model {} diff --git a/core/src/ops/fft/cmp.rs b/core/src/ops/fft/cmp/direction.rs similarity index 92% rename from core/src/ops/fft/cmp.rs rename to core/src/ops/fft/cmp/direction.rs index b22eb815..5daff00a 100644 --- a/core/src/ops/fft/cmp.rs +++ b/core/src/ops/fft/cmp/direction.rs @@ -1,7 +1,8 @@ /* - Appellation: cmp - Contrib: FL03 + Appellation: direction + Contrib: FL03 */ +use scsys::VariantConstructors; use strum::{ AsRefStr, Display, EnumCount, EnumIs, EnumIter, EnumString, VariantArray, VariantNames, }; @@ -81,6 +82,7 @@ impl From for usize { PartialEq, PartialOrd, VariantArray, + VariantConstructors, VariantNames, )] #[cfg_attr( diff --git a/core/src/ops/fft/cmp/mode.rs b/core/src/ops/fft/cmp/mode.rs new file mode 100644 index 00000000..7d538eae --- /dev/null +++ b/core/src/ops/fft/cmp/mode.rs @@ -0,0 +1,44 @@ +/* + Appellation: mode + Contrib: FL03 +*/ +use scsys::VariantConstructors; +use strum::{ + AsRefStr, Display, EnumCount, EnumIs, EnumIter, EnumString, VariantArray, VariantNames, +}; + +toggle!(enum C, R); + +/// +#[derive( + AsRefStr, + Clone, + Copy, + Debug, + Default, + Display, + EnumCount, + EnumIs, + EnumIter, + EnumString, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + VariantArray, + VariantConstructors, + VariantNames, +)] +#[cfg_attr( + feature = "serde", + derive(serde::Deserialize, serde::Serialize), + serde(rename_all = "lowercase", untagged) +)] +#[repr(usize)] +#[strum(serialize_all = "lowercase")] +pub enum FftMode { + #[default] + Complex, + Real, +} diff --git a/core/src/ops/fft/plan.rs b/core/src/ops/fft/cmp/plan.rs similarity index 53% rename from core/src/ops/fft/plan.rs rename to core/src/ops/fft/cmp/plan.rs index 6b1c42dd..d6809acb 100644 --- a/core/src/ops/fft/plan.rs +++ b/core/src/ops/fft/cmp/plan.rs @@ -2,45 +2,62 @@ Appellation: plan Contrib: FL03 */ -#[cfg(no_std)] -use alloc::vec::{self, Vec}; use core::slice; -#[cfg(not(no_std))] -use std::vec; + +use crate::ops::prelude::fft_permutation; #[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] pub struct FftPlan { + len: usize, plan: Vec, } impl FftPlan { - pub fn new(n: usize) -> Self { - let mut plan = Vec::with_capacity(n); - plan.extend(0..n); - - let mut rev = 0; // reverse - let mut pos = 1; // position - while pos < n { - let mut bit = n >> 1; - while bit & rev != 0 { - rev ^= bit; - bit >>= 1; - } - rev ^= bit; - // This is equivalent to adding 1 to a reversed number - if pos < rev { - // Only swap each element once - plan.swap(pos, rev); - } - pos += 1; + pub fn new(len: usize) -> Self { + Self { + len, + plan: Vec::with_capacity(len), } - Self { plan } + } + + pub fn build(self) -> Self { + let plan = fft_permutation(self.len); + Self { plan, ..self } + } + + pub fn clear(&mut self) { + self.len = 0; + self.plan.clear(); + } + + pub fn get(&self, index: usize) -> Option<&usize> { + self.plan().get(index) + } + + pub fn iter(&self) -> slice::Iter { + self.plan().iter() + } + + pub fn len(&self) -> usize { + self.len } pub fn plan(&self) -> &[usize] { &self.plan } + + pub fn set(&mut self, len: usize) { + self.len = len; + self.plan = Vec::with_capacity(len); + } + + pub fn with(self, len: usize) -> Self { + Self { + len, + plan: Vec::with_capacity(len), + } + } } impl AsRef<[usize]> for FftPlan { @@ -63,15 +80,18 @@ impl Extend for FftPlan { impl FromIterator for FftPlan { fn from_iter>(iter: T) -> Self { + let plan = Vec::from_iter(iter); Self { - plan: Vec::from_iter(iter), + len: plan.len(), + plan, } } } +#[cfg(any(feature = "alloc", feature = "std"))] impl IntoIterator for FftPlan { type Item = usize; - type IntoIter = vec::IntoIter; + type IntoIter = crate::rust::vec::IntoIter; fn into_iter(self) -> Self::IntoIter { self.plan.into_iter() diff --git a/core/src/ops/fft/fft.rs b/core/src/ops/fft/fft.rs index 65dfa1c6..d493c24d 100644 --- a/core/src/ops/fft/fft.rs +++ b/core/src/ops/fft/fft.rs @@ -3,27 +3,22 @@ Contrib: FL03 */ use super::{FftDirection, FftPlan}; -// use crate::prelude::AsComplex; -// use num::complex::{Complex, ComplexFloat}; -// use num::traits::{Float, FloatConst, NumAssignOps, NumOps}; -// use num::traits::real::Real; -// use std::ops::Neg; -pub struct FastFourierTransform { +pub struct Fft { direction: FftDirection, plan: FftPlan, } -impl FastFourierTransform { +impl Fft { pub fn new(direction: FftDirection, plan: FftPlan) -> Self { Self { direction, plan } } - pub fn direction(&self) -> FftDirection { + pub const fn direction(&self) -> FftDirection { self.direction } - pub fn plan(&self) -> &FftPlan { + pub const fn plan(&self) -> &FftPlan { &self.plan } } diff --git a/core/src/ops/fft/mod.rs b/core/src/ops/fft/mod.rs index a0d766ab..3306c3b5 100644 --- a/core/src/ops/fft/mod.rs +++ b/core/src/ops/fft/mod.rs @@ -4,26 +4,39 @@ */ //! # Fast Fourier Transform //! -//! +//! The `fft` module provides an implementation of the Fast Fourier Transform (FFT) algorithm. +//! The Fast Fourier Transform is an efficient algorithm for computing the Discrete Fourier Transform (DFT). pub use self::prelude::*; pub(crate) mod fft; pub(crate) mod utils; -pub mod cmp; -pub mod plan; +pub mod cmp { + pub use self::prelude::*; + + pub mod direction; + pub mod mode; + pub mod plan; + + pub(crate) mod prelude { + pub use super::direction::FftDirection; + pub use super::mode::FftMode; + pub use super::plan::FftPlan; + } +} + +/// Trait for computing the Discrete Fourier Transform (DFT) of a sequence. +pub trait DFT { + type Output; -pub trait Fft { - fn fft(&self) -> Vec; - fn ifft(&self) -> Vec; + fn dft(&self) -> Self::Output; } pub(crate) mod prelude { - pub use super::cmp::*; + pub use super::cmp::prelude::*; pub use super::fft::*; - pub use super::plan::*; pub use super::utils::*; - pub use super::Fft; + pub use super::DFT; } #[cfg(test)] diff --git a/core/src/ops/fft/utils.rs b/core/src/ops/fft/utils.rs index fd1243f4..c3990202 100644 --- a/core/src/ops/fft/utils.rs +++ b/core/src/ops/fft/utils.rs @@ -169,3 +169,30 @@ where let scale = T::from(n).unwrap().recip(); result.iter().map(|x| x.re() * scale).collect() } + +#[doc(hidden)] +/// Generates a permutation for the Fast Fourier Transform. +pub fn fft_permutation(length: usize) -> Vec { + let mut result = Vec::new(); + result.reserve_exact(length); + for i in 0..length { + result.push(i); + } + let mut reverse = 0_usize; + let mut position = 1_usize; + while position < length { + let mut bit = length >> 1; + while bit & reverse != 0 { + reverse ^= bit; + bit >>= 1; + } + reverse ^= bit; + // This is equivalent to adding 1 to a reversed number + if position < reverse { + // Only swap each element once + result.swap(position, reverse); + } + position += 1; + } + result +} diff --git a/core/src/ops/pad.rs b/core/src/ops/pad.rs index d1257c99..e6962770 100644 --- a/core/src/ops/pad.rs +++ b/core/src/ops/pad.rs @@ -2,134 +2,83 @@ Appellation: pad Contrib: FL03 */ -pub use self::utils::*; -use num::Zero; -use strum::{AsRefStr, Display, EnumCount, EnumIs, EnumIter, VariantNames}; +pub use self::{action::PadAction, mode::PadMode, utils::*}; -pub trait PadItem { +pub(crate) mod action; +pub(crate) mod mode; + +use nd::{Array, ArrayBase, DataOwned, Dimension}; +use num::traits::{FromPrimitive, Num}; + +pub trait Pad { type Output; - fn pad(&self, pad: usize) -> Self::Output; + fn pad(&self, mode: PadMode, pad: &[[usize; 2]]) -> Self::Output; } -// impl Pad for Array -// where -// T: Clone + Num, -// D: Dimension, -// { -// fn pad(&self, pad: usize) -> Self { -// self.pad_with(pad, T::zero()) -// } - -// fn pad_with(&self, pad: usize, value: T) -> Self { -// let mut pad = vec![value; pad]; -// pad.extend_from_slice(self); -// pad.extend_from_slice(&vec![value; pad.len()]); -// Array::from_vec(pad) -// } -// } - -#[derive( - AsRefStr, - Clone, - Copy, - Debug, - Default, - Display, - EnumCount, - EnumIs, - EnumIter, - Eq, - Hash, - Ord, - PartialEq, - PartialOrd, - VariantNames, -)] -#[repr(u8)] -#[cfg_attr( - feature = "serde", - derive(serde::Deserialize, serde::Serialize), - serde(rename_all = "snake_case", untagged) -)] -#[strum(serialize_all = "snake_case")] -pub enum PadAction { - Clipping, - Lane, - Reflecting, - #[default] - StopAfterCopy, - Wrapping, -} +impl Pad for ArrayBase +where + A: Copy + FromPrimitive + Num, + D: Dimension, + S: DataOwned, +{ + type Output = Array; -#[derive(Clone, Copy, Debug, Display, EnumCount, EnumIs, Eq, Hash, Ord, PartialEq, PartialOrd)] -#[cfg_attr( - feature = "serde", - derive(serde::Deserialize, serde::Serialize), - serde(rename_all = "lowercase", untagged) -)] -pub enum PadMode { - Constant(T), - Edge, - Maximum, - Mean, - Median, - Minimum, - Mode, - Reflect, - Symmetric, - Wrap, + fn pad(&self, mode: PadMode, pad: &[[usize; 2]]) -> Self::Output { + self::utils::pad(self, pad, mode) + } } -impl From for PadMode { - fn from(value: T) -> Self { - PadMode::Constant(value) - } +pub struct Padding { + pub(crate) action: PadAction, + pub(crate) mode: PadMode, + pub(crate) pad: Vec<[usize; 2]>, + pub(crate) padding: usize, } -impl PadMode { - pub(crate) fn action(&self) -> PadAction { - match self { - PadMode::Constant(_) => PadAction::StopAfterCopy, - PadMode::Edge => PadAction::Clipping, - PadMode::Maximum => PadAction::Clipping, - PadMode::Mean => PadAction::Clipping, - PadMode::Median => PadAction::Clipping, - PadMode::Minimum => PadAction::Clipping, - PadMode::Mode => PadAction::Clipping, - PadMode::Reflect => PadAction::Reflecting, - PadMode::Symmetric => PadAction::Reflecting, - PadMode::Wrap => PadAction::Wrapping, +impl Padding { + pub fn new() -> Self { + Self { + action: PadAction::default(), + mode: PadMode::default(), + pad: Vec::new(), + padding: 0, } } - pub fn init(&self) -> T - where - T: Copy + Zero, - { - match *self { - PadMode::Constant(v) => v, - _ => T::zero(), - } + + pub fn pad(&self) -> &[[usize; 2]] { + &self.pad } -} -pub struct Padding { - pub mode: PadMode, - pub pad: usize, + pub fn with_action(mut self, action: PadAction) -> Self { + self.action = action; + self + } + + pub fn with_mode(mut self, mode: PadMode) -> Self { + self.mode = mode; + self + } + + pub fn with_padding(mut self, padding: usize) -> Self { + self.padding = padding; + self + } } mod utils { + #![cfg(any(feature = "std", feature = "alloc"))] use super::{PadAction, PadMode}; use crate::traits::ArrayLike; - use ndarray::{Array, ArrayBase, AxisDescription, Data, DataOwned, Dimension, Slice}; + use nd::{Array, ArrayBase, AxisDescription, Data, DataOwned, Dimension, Slice}; use num::{FromPrimitive, Num}; - #[cfg(no_std)] + #[cfg(all(feature = "alloc", no_std))] use alloc::borrow::Cow; #[cfg(feature = "std")] use std::borrow::Cow; - fn read_pad(nb_dim: usize, pad: &[[usize; 2]]) -> Cow<[[usize; 2]]> { + fn reader(nb_dim: usize, pad: &[[usize; 2]]) -> Cow<[[usize; 2]]> { if pad.len() == 1 && pad.len() < nb_dim { // The user provided a single padding for all dimensions Cow::from(vec![pad[0]; nb_dim]) @@ -146,7 +95,7 @@ mod utils { D: Dimension, S: DataOwned, { - let pad = read_pad(data.ndim(), pad); + let pad = reader(data.ndim(), pad); let mut new_dim = data.raw_dim(); for (ax, (&ax_len, pad)) in data.shape().iter().zip(pad.iter()).enumerate() { new_dim[ax] = ax_len + pad[0] + pad[1]; @@ -168,7 +117,7 @@ mod utils { D: Dimension, S: Data, { - let pad = read_pad(data.ndim(), pad); + let pad = reader(data.ndim(), pad); // Select portion of padded array that needs to be copied from the original array. output diff --git a/core/src/ops/pad/action.rs b/core/src/ops/pad/action.rs new file mode 100644 index 00000000..15325ddc --- /dev/null +++ b/core/src/ops/pad/action.rs @@ -0,0 +1,41 @@ +/* + Appellation: action + Contrib: FL03 +*/ +use scsys::VariantConstructors; +use strum::{AsRefStr, Display, EnumCount, EnumIs, EnumIter, EnumString, VariantNames}; + +#[derive( + AsRefStr, + Clone, + Copy, + Debug, + Default, + Display, + EnumCount, + EnumIs, + EnumIter, + EnumString, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + VariantConstructors, + VariantNames, +)] +#[cfg_attr( + feature = "serde", + derive(serde::Deserialize, serde::Serialize), + serde(rename_all = "snake_case", untagged) +)] +#[repr(u8)] +#[strum(serialize_all = "snake_case")] +pub enum PadAction { + Clipping, + Lane, + Reflecting, + #[default] + StopAfterCopy, + Wrapping, +} diff --git a/core/src/ops/pad/mode.rs b/core/src/ops/pad/mode.rs new file mode 100644 index 00000000..ac37f1ab --- /dev/null +++ b/core/src/ops/pad/mode.rs @@ -0,0 +1,77 @@ +/* + Appellation: mode + Contrib: FL03 +*/ +use crate::ops::pad::PadAction; +use num::Zero; +use smart_default::SmartDefault; +use strum::{AsRefStr, Display, EnumCount, EnumIs, VariantNames}; + +#[derive( + AsRefStr, + Clone, + Copy, + Debug, + Display, + EnumCount, + EnumIs, + Eq, + Hash, + Ord, + PartialEq, + PartialOrd, + SmartDefault, + VariantNames, +)] +#[cfg_attr( + feature = "serde", + derive(serde::Deserialize, serde::Serialize), + serde(rename_all = "lowercase", untagged) +)] +#[repr(C)] +#[strum(serialize_all = "snake_case")] +pub enum PadMode { + Constant(T), + Edge, + Maximum, + Mean, + Median, + Minimum, + Mode, + Reflect, + Symmetric, + #[default] + Wrap, +} + +impl From for PadMode { + fn from(value: T) -> Self { + PadMode::Constant(value) + } +} + +impl PadMode { + pub(crate) fn action(&self) -> PadAction { + match self { + PadMode::Constant(_) => PadAction::StopAfterCopy, + PadMode::Edge => PadAction::Clipping, + PadMode::Maximum => PadAction::Clipping, + PadMode::Mean => PadAction::Clipping, + PadMode::Median => PadAction::Clipping, + PadMode::Minimum => PadAction::Clipping, + PadMode::Mode => PadAction::Clipping, + PadMode::Reflect => PadAction::Reflecting, + PadMode::Symmetric => PadAction::Reflecting, + PadMode::Wrap => PadAction::Wrapping, + } + } + pub fn init(&self) -> T + where + T: Copy + Zero, + { + match *self { + PadMode::Constant(v) => v, + _ => T::zero(), + } + } +} diff --git a/core/src/params/impls/impl_rand.rs b/core/src/params/impls/impl_rand.rs deleted file mode 100644 index f8681da9..00000000 --- a/core/src/params/impls/impl_rand.rs +++ /dev/null @@ -1,23 +0,0 @@ -/* - Appellation: impl_rand - Contrib: FL03 -*/ -use crate::params::Parameter; -use crate::rand::GenerateRandom; -use ndarray::{Array, Dimension}; -use ndrand::rand_distr::uniform::SampleUniform; -use ndrand::rand_distr::{Distribution, StandardNormal}; -use num::Float; - -impl Parameter -where - D: Dimension, - T: Float + SampleUniform, - StandardNormal: Distribution, -{ - pub fn init_uniform(mut self, dk: T) -> Self { - let dim = self.value.dim(); - self.value = Array::uniform_between(dk, dim); - self - } -} diff --git a/core/src/primitives.rs b/core/src/primitives.rs index 7f881602..c2755083 100644 --- a/core/src/primitives.rs +++ b/core/src/primitives.rs @@ -5,21 +5,20 @@ pub use consts::*; pub mod consts { - - pub const DEFAULT_MODEL_SIZE: usize = 2048; - pub const EPSILON: f64 = 1e-8; + /// The default model size for any given model + pub const D_MODEL: usize = 512; + /// The default epsilon value for floating point operations + pub const EPSILON: f64 = 1e-5; } -#[allow(unused_imports)] +#[allow(unused)] pub(crate) mod rust { - pub(crate) use core::*; - - #[cfg(no_std)] + #[cfg(all(feature = "alloc", no_std))] pub(crate) use self::no_std::*; #[cfg(feature = "std")] pub(crate) use self::with_std::*; - - #[cfg(no_std)] + pub(crate) use core::*; + #[cfg(all(feature = "alloc", no_std))] mod no_std { pub use alloc::borrow::Cow; pub use alloc::boxed::{self, Box}; @@ -31,12 +30,12 @@ pub(crate) mod rust { pub use std::borrow::Cow; pub use std::boxed::{self, Box}; pub use std::collections::{self, BTreeMap, BTreeSet, BinaryHeap, VecDeque}; - pub(crate) use std::sync::Arc; + pub use std::sync::Arc; pub use std::vec::{self, Vec}; } - #[cfg(no_std)] - pub type Map = collections::BTreeMap; + #[cfg(all(feature = "alloc", no_std))] + pub type Map = alloc::collections::BTreeMap; #[cfg(feature = "std")] - pub type Map = collections::HashMap; + pub type Map = std::collections::HashMap; } diff --git a/core/src/rand/generate.rs b/core/src/rand/generate.rs deleted file mode 100644 index 9078df1b..00000000 --- a/core/src/rand/generate.rs +++ /dev/null @@ -1,96 +0,0 @@ -/* - Appellation: generate - Contrib: FL03 -*/ -use core::ops::Neg; -use ndarray::*; -use ndrand::rand::rngs::StdRng; -use ndrand::rand::{Rng, SeedableRng}; -use ndrand::rand_distr::uniform::{SampleUniform, Uniform}; -use ndrand::rand_distr::{Bernoulli, BernoulliError, Distribution, StandardNormal}; -use ndrand::RandomExt; -use num::traits::real::Real; -use num::traits::Float; - -pub trait GenerateRandom: Sized -where - D: Dimension, -{ - fn rand(dim: Sh, distr: IdS) -> Self - where - IdS: Distribution, - Sh: ShapeBuilder; - - fn rand_using(dim: Sh, distr: IdS, rng: &mut R) -> Self - where - IdS: Distribution, - R: Rng, - Sh: ShapeBuilder; - - fn bernoulli(dim: impl IntoDimension, p: Option) -> Result - where - Bernoulli: Distribution, - { - let dist = Bernoulli::new(p.unwrap_or(0.5))?; - Ok(Self::rand(dim.into_dimension(), dist)) - } - - fn stdnorm(dim: impl IntoDimension) -> Self - where - StandardNormal: Distribution, - { - Self::rand(dim, StandardNormal) - } - - fn normal_from_key(key: u64, dim: impl IntoDimension) -> Self - where - StandardNormal: Distribution, - R: Rng, - { - Self::rand_using( - dim.into_dimension(), - StandardNormal, - &mut StdRng::seed_from_u64(key), - ) - } - - fn uniform(axis: usize, dim: impl IntoDimension) -> Self - where - T: Real + SampleUniform, - { - let dim = dim.into_dimension(); - let dk = T::from(dim[axis]).unwrap().recip().sqrt(); - Self::uniform_between(dk, dim) - } - - fn uniform_between(dk: T, dim: impl IntoDimension) -> Self - where - T: Copy + Neg + SampleUniform, - { - Self::rand(dim, Uniform::new(-dk, dk)) - } -} - -impl GenerateRandom for Array -where - T: Float + SampleUniform, - D: Dimension, - StandardNormal: Distribution, -{ - fn rand(dim: Sh, distr: Dtr) -> Self - where - Dtr: Distribution, - Sh: ShapeBuilder, - { - Self::random(dim, distr) - } - - fn rand_using(dim: Sh, distr: Dtr, rng: &mut R) -> Self - where - Dtr: Distribution, - R: Rng + ?Sized, - Sh: ShapeBuilder, - { - Self::random_using(dim, distr, rng) - } -} diff --git a/core/src/rand/mod.rs b/core/src/rand/mod.rs deleted file mode 100644 index 7bc578d8..00000000 --- a/core/src/rand/mod.rs +++ /dev/null @@ -1,27 +0,0 @@ -/* - Appellation: rand - Contrib: FL03 -*/ -#![cfg(feature = "rand")] - -pub use self::prelude::*; - -pub(crate) mod generate; -pub(crate) mod utils; - -#[doc(no_inline)] -pub use ndarray_rand as ndrand; -#[doc(no_inline)] -pub use ndarray_rand::{RandomExt, SamplingStrategy}; -#[doc(no_inline)] -pub use rand; -#[doc(no_inline)] -pub use rand_distr; - -pub(crate) mod prelude { - #[doc(no_inline)] - pub use ndarray_rand::RandomExt; - - pub use super::generate::*; - pub use super::utils::*; -} diff --git a/core/src/rand/utils.rs b/core/src/rand/utils.rs deleted file mode 100644 index cfcad3c4..00000000 --- a/core/src/rand/utils.rs +++ /dev/null @@ -1,113 +0,0 @@ -/* - Appellation: utils - Contrib: FL03 -*/ -use ndarray::*; -use ndrand::rand::rngs::StdRng; -use ndrand::rand::SeedableRng; -use ndrand::rand_distr::{Distribution, StandardNormal}; -use ndrand::RandomExt; -use num::complex::{Complex, ComplexDistribution}; -use num::traits::real::Real; -use num::Num; -use rand::distributions::uniform::{SampleUniform, Uniform}; - -pub fn lecun_normal(shape: impl IntoDimension) -> Array -where - D: Dimension, - T: Real + ScalarOperand, - StandardNormal: Distribution, -{ - let dim = shape.into_dimension(); - let n = dim.size(); - let scale = T::from(n).unwrap().recip().sqrt(); - Array::random(dim, StandardNormal) * scale -} - -pub fn lecun_normal_seeded(shape: impl IntoDimension, seed: u64) -> Array -where - D: Dimension, - T: Real + ScalarOperand, - StandardNormal: Distribution, -{ - let dim = shape.into_dimension(); - let n = dim.size(); - let scale = T::from(n).unwrap().recip().sqrt(); - Array::random_using(dim, StandardNormal, &mut StdRng::seed_from_u64(seed)) * scale -} - -/// Generate a random array of complex numbers with real and imaginary parts in the range [0, 1) -pub fn randc(shape: impl IntoDimension) -> Array, D> -where - D: Dimension, - T: Clone + Num, - ComplexDistribution: Distribution>, -{ - let distr = ComplexDistribution::::new(T::one(), T::one()); - Array::random(shape, distr) -} -/// -pub fn randcomplex(shape: impl IntoDimension) -> Array, D> -where - D: Dimension, - T: Copy + Num, - StandardNormal: Distribution, -{ - let dim = shape.into_dimension(); - let re = Array::random(dim.clone(), StandardNormal); - let im = Array::random(dim.clone(), StandardNormal); - let mut res = Array::zeros(dim); - ndarray::azip!((re in &re, im in &im, res in &mut res) { - *res = Complex::new(*re, *im); - }); - res -} -/// Creates a random array from a uniform distribution using a given key -pub fn seeded_uniform( - key: u64, - start: T, - stop: T, - shape: impl IntoDimension, -) -> Array -where - D: Dimension, - T: SampleUniform, -{ - Array::random_using( - shape, - Uniform::new(start, stop), - &mut StdRng::seed_from_u64(key), - ) -} -/// -pub fn seeded_stdnorm(key: u64, shape: impl IntoDimension) -> Array -where - D: Dimension, - StandardNormal: Distribution, -{ - Array::random_using(shape, StandardNormal, &mut StdRng::seed_from_u64(key)) -} -/// -pub fn randc_normal(key: u64, shape: impl IntoDimension) -> Array, D> -where - D: Dimension, - T: Copy + Num, - StandardNormal: Distribution, -{ - let dim = shape.into_dimension(); - let re = seeded_stdnorm(key, dim.clone()); - let im = seeded_stdnorm(key, dim.clone()); - let mut res = Array::zeros(dim); - ndarray::azip!((re in &re, im in &im, res in &mut res) { - *res = Complex::new(*re, *im); - }); - res -} -/// Given a shape, generate a random array using the StandardNormal distribution -pub fn stdnorm(shape: impl IntoDimension) -> Array -where - D: Dimension, - StandardNormal: Distribution, -{ - Array::random(shape, StandardNormal) -} diff --git a/core/src/traits/arr/create.rs b/core/src/traits/arr/create.rs index 918cb6a3..8c45d927 100644 --- a/core/src/traits/arr/create.rs +++ b/core/src/traits/arr/create.rs @@ -2,10 +2,10 @@ Appellation: create Contrib: FL03 */ -use nd::{ArrayBase, DataOwned, Dimension, RawData, ShapeBuilder}; +use nd::{ArrayBase, DataOwned, Dimension, Ix2, ShapeBuilder}; use num::traits::Num; -pub trait TensorConstructor +pub trait NdLike where Self: DefaultLike + FillLike @@ -14,63 +14,38 @@ where { } -pub trait ArrayLike +pub trait ArrayLike where D: Dimension, - S: RawData, { - fn array_like(&self, shape: Sh, elem: A) -> ArrayBase - where - Sh: ShapeBuilder; -} - -impl ArrayLike for ArrayBase -where - A: Clone, - D: Dimension, - S: nd::DataOwned, -{ - fn array_like(&self, shape: Sh, elem: A) -> ArrayBase - where - Sh: ShapeBuilder, - { - if self.is_standard_layout() { - ArrayBase::from_elem(shape, elem) - } else { - ArrayBase::from_elem(shape.f(), elem) - } - } -} - -pub trait DefaultLike { type Output; - fn default_like(&self) -> Self::Output; + fn array_like(&self, shape: Sh, elem: A) -> Self::Output + where + Sh: ShapeBuilder; } -pub trait FillLike { - type Output; - - fn fill_like(&self, elem: T) -> Self::Output; -} +macro_rules! ndlike { + ($($name:ident::$(<$($T:ident),*>::)?$method:ident $(($($field:ident:$ft:ty),*))?),* $(,)?) => { + $(ndlike!(@impl $name::$(<$($T),*>::)?$method$(($($field:$ft),*))?);)* + }; + (@impl $name:ident::$(<$($T:ident),*>::)?$method:ident$(($($field:ident: $ft:ty),*))?) => { + pub trait $name$(<$($T),*>)? { + type Output; -pub trait OnesLike { - type Output; + fn $method(&self $(, $($field:$ft),*)?) -> Self::Output; + } + }; - fn ones_like(&self) -> Self::Output; } -pub trait ZerosLike { - type Output; - - fn zeros_like(&self) -> Self::Output; -} +ndlike!(DefaultLike::default_like, OnesLike::ones_like, ZerosLike::zeros_like, FillLike::::fill_like(elem: T)); /* ******** implementations ******** */ -impl TensorConstructor> for ArrayBase +impl NdLike> for ArrayBase where A: Clone + Default + Num, D: Dimension, @@ -78,6 +53,26 @@ where { } +impl ArrayLike for ArrayBase +where + A: Clone, + D: Dimension, + S: nd::DataOwned, +{ + type Output = ArrayBase; + + fn array_like(&self, shape: Sh, elem: A) -> Self::Output + where + Sh: ShapeBuilder, + { + if self.is_standard_layout() { + ArrayBase::from_elem(shape, elem) + } else { + ArrayBase::from_elem(shape.f(), elem) + } + } +} + impl FillLike for ArrayBase where A: Clone, @@ -91,11 +86,9 @@ where } } -macro_rules! impl_like { +macro_rules! impl_ndlike { + ($name:ident::$method:ident.$call:ident: $($p:tt)*) => { - impl_like!(@impl $name::$method.$call: $($p)*); - }; - (@impl $name:ident::$method:ident.$call:ident: $($p:tt)*) => { impl $name for ArrayBase where A: $($p)*, @@ -111,6 +104,6 @@ macro_rules! impl_like { }; } -impl_like!(DefaultLike::default_like.default: Default); -impl_like!(OnesLike::ones_like.ones: Clone + num::One); -impl_like!(ZerosLike::zeros_like.zeros: Clone + num::Zero); +impl_ndlike!(DefaultLike::default_like.default: Default); +impl_ndlike!(OnesLike::ones_like.ones: Clone + num::One); +impl_ndlike!(ZerosLike::zeros_like.zeros: Clone + num::Zero); diff --git a/core/src/traits/arr/misc.rs b/core/src/traits/arr/misc.rs index 1c5ac006..40857596 100644 --- a/core/src/traits/arr/misc.rs +++ b/core/src/traits/arr/misc.rs @@ -2,8 +2,19 @@ Appellation: convert Contrib: FL03 */ -use nd::Axis; -use nd::{ArrayBase, Dimension, RawData}; +use nd::prelude::*; +use nd::{DataMut, RawData}; + +/// This trait is used to fill an array with a value based on a mask. +/// The mask is a boolean array of the same shape as the array. +pub trait MaskFill +where + D: Dimension, +{ + type Output; + + fn masked_fill(&self, mask: &Array, value: A) -> Self::Output; +} pub trait IntoAxis { fn into_axis(self) -> Axis; @@ -16,6 +27,27 @@ pub trait IsSquare { /* ******** implementations ******** */ + +impl MaskFill for ArrayBase +where + A: Clone, + D: Dimension, + S: DataMut, + Self: Clone, +{ + type Output = ArrayBase; + + fn masked_fill(&self, mask: &Array, value: A) -> Self::Output { + let mut arr = self.clone(); + arr.zip_mut_with(&mask, |x, &m| { + if m { + *x = value.clone(); + } + }); + arr + } +} + impl IntoAxis for S where S: AsRef, @@ -31,6 +63,7 @@ where S: RawData, { fn is_square(&self) -> bool { - self.shape().iter().all(|&x| x == self.shape()[0]) + let first = self.shape().first().unwrap(); + self.shape().iter().all(|x| x == first) } } diff --git a/core/src/traits/arr/ops.rs b/core/src/traits/arr/ops.rs index 3a2601eb..5fc97c9e 100644 --- a/core/src/traits/arr/ops.rs +++ b/core/src/traits/arr/ops.rs @@ -1,6 +1,6 @@ /* - Appellation: arr - Contrib: FL03 + Appellation: ops + Contrib: FL03 */ use nd::linalg::Dot; use nd::*; @@ -21,7 +21,7 @@ pub trait Inverse { pub trait Matmul { type Output; - fn matmul(&self, rhs: Rhs) -> Self::Output; + fn matmul(&self, rhs: &Rhs) -> Self::Output; } pub trait Matpow { @@ -56,14 +56,14 @@ where } } -impl Matmul for S +impl Matmul for S where S: Dot, { type Output = Y; - fn matmul(&self, rhs: X) -> Self::Output { - self.dot(&rhs) + fn matmul(&self, rhs: &X) -> Self::Output { + self.dot(rhs) } } diff --git a/core/src/traits/arr/reshape.rs b/core/src/traits/arr/reshape.rs new file mode 100644 index 00000000..7079f130 --- /dev/null +++ b/core/src/traits/arr/reshape.rs @@ -0,0 +1,40 @@ +/* + Appellation: reshape [traits::arr] + Contrib: FL03 +*/ +use nd::prelude::*; +use nd::{RawData, RawDataClone}; + +pub trait Unsqueeze { + type Output; + + fn unsqueeze(self, axis: usize) -> Self::Output; +} + +/* + ************* Implementations ************* +*/ + +impl Unsqueeze for ArrayBase +where + D: Dimension, + S: RawData, +{ + type Output = ArrayBase; + + fn unsqueeze(self, axis: usize) -> Self::Output { + self.insert_axis(Axis(axis)) + } +} + +impl<'a, A, S, D> Unsqueeze for &'a ArrayBase +where + D: Dimension, + S: RawDataClone, +{ + type Output = ArrayBase; + + fn unsqueeze(self, axis: usize) -> Self::Output { + self.clone().insert_axis(Axis(axis)) + } +} diff --git a/core/src/traits/misc/sequential.rs b/core/src/traits/misc/sequential.rs new file mode 100644 index 00000000..8e92192e --- /dev/null +++ b/core/src/traits/misc/sequential.rs @@ -0,0 +1,63 @@ +/* + Appellation: sequential [traits::misc] + Contrib: FL03 +*/ +use num::traits::FromPrimitive; + +/// A trait for sequential data structures; +/// This trait is implemented for iterators that have a known length. +pub trait Sequence { + const LENGTH: Option = None; + + fn len(&self) -> usize; + + fn is_empty(&self) -> bool { + self.len() == 0 + } + + fn elems(&self) -> T + where + T: FromPrimitive, + { + T::from_usize(self.len()).unwrap() + } +} + +pub trait SequenceIter { + type Item; + + fn len(&self) -> usize; +} +/* + ************* Implementations ************* +*/ +impl SequenceIter for I +where + I: ExactSizeIterator, +{ + type Item = T; + + fn len(&self) -> usize { + self.len() + } +} + +impl Sequence for Vec { + fn len(&self) -> usize { + self.len() + } +} + +impl Sequence for [T] { + fn len(&self) -> usize { + self.len() + } +} + +impl Sequence for [T; N] { + const LENGTH: Option = Some(N); + + fn len(&self) -> usize { + N + } +} diff --git a/core/src/traits/misc/setup.rs b/core/src/traits/misc/setup.rs deleted file mode 100644 index 1b7856f9..00000000 --- a/core/src/traits/misc/setup.rs +++ /dev/null @@ -1,18 +0,0 @@ -/* - Appellation: setup - Contrib: FL03 -*/ - -pub trait Init { - fn init(self) -> Self; -} - -pub trait InitInplace { - fn init(&mut self); -} - -pub trait Setup { - type Config; - - fn setup(&mut self, config: Self::Config); -} diff --git a/core/src/traits/misc/toggle.rs b/core/src/traits/misc/toggle.rs new file mode 100644 index 00000000..4865d33f --- /dev/null +++ b/core/src/traits/misc/toggle.rs @@ -0,0 +1,48 @@ +/* + Appellation: toggle + Contrib: FL03 +*/ + +pub trait Toggle: 'static {} + +pub trait OfType { + fn of() -> bool + where + T: 'static, + Self: 'static, + { + core::any::TypeId::of::() == core::any::TypeId::of::() + } +} + +/* + ************* Implementations ************* +*/ +impl OfType for T {} + +macro_rules! impl_toggle { + ($($scope:ident$(<$T:ident>)?),* $(,)?) => { + $(impl_toggle!(@impl $scope$(<$T>)?);)* + }; + (@impl $scope:ident$(<$T:ident>)?) => { + impl$(<$T>)? Toggle for $scope$(<$T> where $T: 'static)? {} + }; +} + +impl_toggle!( + bool, + char, + i8, + i16, + i32, + i64, + i128, + isize, + u8, + u16, + u32, + u64, + u128, + usize, + Option +); diff --git a/core/src/traits/mod.rs b/core/src/traits/mod.rs index 809054cc..9dc12247 100644 --- a/core/src/traits/mod.rs +++ b/core/src/traits/mod.rs @@ -4,52 +4,52 @@ */ pub use self::prelude::*; -pub mod math; +pub mod num; +pub mod ops; pub mod predict; +pub mod setup; pub mod train; pub mod arr { pub use self::prelude::*; - pub(crate) mod create; - pub(crate) mod misc; - pub(crate) mod ops; + mod create; + mod misc; + mod ops; + mod reshape; pub(crate) mod prelude { pub use super::create::*; pub use super::misc::*; pub use super::ops::*; + pub use super::reshape::*; } } -pub(crate) mod misc { - pub mod adjust; - pub mod setup; - pub mod store; +pub mod misc { + pub use self::prelude::*; + + pub(crate) mod adjust; + #[doc(hidden)] + pub(crate) mod sequential; + #[doc(hidden)] + pub(crate) mod store; + pub(crate) mod toggle; pub(crate) mod prelude { pub use super::adjust::*; - pub use super::setup::*; + pub use super::sequential::*; pub use super::store::*; + pub use super::toggle::*; } } -pub trait Transform { - type Output; - - fn transform(&self, args: &T) -> Self::Output; -} - pub(crate) mod prelude { - pub use super::Transform; - - pub use super::math::*; - pub use super::predict::*; - pub use super::train::*; - pub use super::arr::prelude::*; pub use super::misc::prelude::*; + pub use super::num::*; + pub use super::ops::*; + pub use super::predict::*; + pub use super::setup::*; + pub use super::train::*; } - -#[cfg(test)] -mod tests {} diff --git a/core/src/traits/math.rs b/core/src/traits/num.rs similarity index 85% rename from core/src/traits/math.rs rename to core/src/traits/num.rs index 1d499a72..d21e77f4 100644 --- a/core/src/traits/math.rs +++ b/core/src/traits/num.rs @@ -52,10 +52,6 @@ pub trait RoundTo { fn round_to(&self, places: usize) -> Self; } -pub trait SquareRoot { - fn sqrt(self) -> Self; -} - /* ********* Implementations ********* */ @@ -151,34 +147,3 @@ where crate::round_to(*self, places) } } - -impl SquareRoot for f32 { - fn sqrt(self) -> Self { - f32::sqrt(self) - } -} - -impl SquareRoot for f64 { - fn sqrt(self) -> Self { - f64::sqrt(self) - } -} - -impl SquareRoot for Complex -where - T: Float, -{ - fn sqrt(self) -> Self { - Complex::::sqrt(self) - } -} - -impl SquareRoot for Array -where - D: Dimension, - T: Float, -{ - fn sqrt(self) -> Self { - self.mapv(|x| x.sqrt()) - } -} diff --git a/core/src/traits/ops.rs b/core/src/traits/ops.rs new file mode 100644 index 00000000..48e1cf3a --- /dev/null +++ b/core/src/traits/ops.rs @@ -0,0 +1,47 @@ +/* + Appellation: ops + Contrib: FL03 +*/ +/// A trait for applying a function to a type +pub trait Apply { + type Output; + + fn apply(&self, f: F) -> Self::Output + where + F: Fn(T) -> U; + + fn apply_mut(&mut self, f: F) -> Self::Output + where + F: FnMut(T) -> U; +} + +pub trait ApplyOn { + type Output; + + fn apply(self, f: F) -> Self::Output + where + F: FnMut(T) -> U; +} + +pub trait Transform { + type Output; + + fn transform(&self, args: &T) -> Self::Output; +} + +/* + ************* Implementations ************* +*/ +impl ApplyOn for S +where + S: Iterator, +{ + type Output = core::iter::Map; + + fn apply(self, f: F) -> Self::Output + where + F: FnMut(T) -> U, + { + self.map(f) + } +} diff --git a/core/src/traits/predict.rs b/core/src/traits/predict.rs index fd294757..632d05b1 100644 --- a/core/src/traits/predict.rs +++ b/core/src/traits/predict.rs @@ -4,16 +4,6 @@ */ use crate::error::PredictError; -#[doc(hidden)] -pub trait Activate: Forward -where - F: Fn(&Self::Output) -> Self::Output, -{ - fn activate(&self, args: &T, f: F) -> Self::Output { - f(&self.forward(args)) - } -} - /// [Forward] describes an object capable of forward propagation. pub trait Forward { type Output; @@ -39,17 +29,17 @@ pub trait Predict { /* ********* Implementations ********* */ -impl Forward for Option +impl Forward for S where - S: Forward, - T: Clone, + S: Predict, { - type Output = T; + type Output = Y; - fn forward(&self, args: &T) -> Self::Output { - match self { - Some(s) => s.forward(args), - None => args.clone(), + fn forward(&self, args: &X) -> Self::Output { + if let Ok(y) = self.predict(args) { + y + } else { + panic!("Error in forward propagation") } } } diff --git a/core/src/traits/setup.rs b/core/src/traits/setup.rs new file mode 100644 index 00000000..d6f2f611 --- /dev/null +++ b/core/src/traits/setup.rs @@ -0,0 +1,71 @@ +/* + Appellation: setup + Contrib: FL03 +*/ +use core::borrow::{Borrow, BorrowMut}; + +/// A trait used to denote objects that may be used for configuring various items +pub trait Config {} + +/// [Configuration] describes composite configuration objects; +/// A configuration object is allowed to inherit from another configuration object +pub trait Configuration +where + C: Config, + Self::Config: Borrow, +{ + type Config: Config; + + fn root(&self) -> &C; + + fn set(&mut self, config: Self::Config); + + fn set_root(&mut self, config: C); +} + +pub trait Init { + fn init(self) -> Self; +} + +pub trait InitInplace { + fn init(&mut self); +} + +pub trait Setup { + type Config; + + fn setup(&mut self, config: Self::Config); +} + +pub trait Context +where + C: Config, +{ + type Cnf: Configuration; + + fn config(&self) -> Self::Cnf; +} + +/* + ************* Implementations ************* +*/ + +impl Configuration for D +where + C: Config, + D: Config + BorrowMut, +{ + type Config = D; + + fn root(&self) -> &C { + self.borrow() + } + + fn set(&mut self, config: Self::Config) { + *self = config; + } + + fn set_root(&mut self, config: C) { + *self.borrow_mut() = config; + } +} diff --git a/core/src/types/mod.rs b/core/src/types/mod.rs index 655b3673..d6347e49 100644 --- a/core/src/types/mod.rs +++ b/core/src/types/mod.rs @@ -6,25 +6,28 @@ pub use self::prelude::*; #[cfg(feature = "std")] pub use self::std_types::*; -pub mod direction; +pub mod propagate; +pub mod shape; +pub type NdResult = core::result::Result; /// A type alias for a [Result](core::result::Result) with the crate's [Error](crate::error::Error) type. /// Defaults to `Result<(), Error>` pub type Result = core::result::Result; #[cfg(feature = "std")] mod std_types { - /// + /// A type alias for a boxed [Error](std::error::Error) type that is `Send`, `Sync`, and `'static`. pub type BoxError = Box; - /// + /// A type alias for a boxed [Result](core::result::Result) which returns some object, `T`, and uses a [BoxError] as the error type. pub type BoxResult = core::result::Result; } pub(crate) mod prelude { - pub use super::direction::Direction; + pub use super::propagate::Propagate; + pub use super::shape::ModelShape; #[cfg(feature = "std")] pub use super::std_types::*; - pub use super::Result; + pub use super::{NdResult, Result}; } #[cfg(test)] diff --git a/core/src/types/direction.rs b/core/src/types/propagate.rs similarity index 87% rename from core/src/types/direction.rs rename to core/src/types/propagate.rs index 2a86ada7..d765ccc5 100644 --- a/core/src/types/direction.rs +++ b/core/src/types/propagate.rs @@ -29,13 +29,13 @@ use strum::{AsRefStr, Display, EnumCount, EnumIs, EnumIter, EnumString, VariantN serde(rename_all = "lowercase") )] #[strum(serialize_all = "lowercase")] -pub enum Direction { +pub enum Propagate { Backward = 0, #[default] Forward = 1, } -impl Direction { +impl Propagate { /// A functional alias for [Direction::Backward]. pub fn backward() -> Self { Self::Backward @@ -46,13 +46,13 @@ impl Direction { } } -impl From for usize { - fn from(direction: Direction) -> Self { +impl From for usize { + fn from(direction: Propagate) -> Self { direction as usize } } -impl From for Direction { +impl From for Propagate { fn from(index: usize) -> Self { match index % Self::COUNT { 0 => Self::Backward, diff --git a/core/src/types/shape.rs b/core/src/types/shape.rs new file mode 100644 index 00000000..37e75047 --- /dev/null +++ b/core/src/types/shape.rs @@ -0,0 +1,166 @@ +/* + Appellation: shape + Contrib: FL03 +*/ +use nd::prelude::{Ix1, Ix2}; +use nd::{Dimension, ErrorKind, IntoDimension, RemoveAxis, ShapeBuilder, ShapeError}; + +pub(crate) fn _from_dim(dim: D) -> Result +where + D: Dimension, +{ + if dim.ndim() == 1 { + Ok(Features::new(dim[0], 1)) + } else if dim.ndim() >= 2 { + Ok(Features::new(dim[1], dim[0])) + } else { + Err(ShapeError::from_kind(ErrorKind::IncompatibleShape)) + } +} + +#[derive(Clone, Copy, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] +#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] +pub struct ModelShape { + pub(crate) features: Features, + pub(crate) network: usize, +} + +impl ModelShape { + pub fn new(model: usize, network: usize) -> Self { + let features = Features::from_network(model, network); + Self { features, network } + } + + pub fn from_features(features: Features) -> Self { + Self { + features, + network: features.size(), + } + } +} + +#[derive(Clone, Copy, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] +#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] +pub struct Features { + pub(crate) inputs: usize, + pub(crate) outputs: usize, +} + +impl Features { + /// Create a new, unchecked [Features] instance. + /// + pub fn new(inputs: usize, outputs: usize) -> Self { + debug_assert_ne!(inputs, 0); + debug_assert_ne!(outputs, 0); + + Self { inputs, outputs } + } + /// Attempts to build a new [Features] instance from the given dimension ([`D`](Dimension)) + pub fn from_dimension(dim: D) -> Result + where + D: Dimension, + { + _from_dim(dim) + } + /// Builds a new instance from the given shape ([`Sh`](ShapeBuilder)); + /// Unlike [Features::from_dimension], this method requires the dimension (`D`) to + /// additionally implement the [RemoveAxis] trait + pub fn from_shape(shape: Sh) -> Self + where + D: RemoveAxis, + Sh: ShapeBuilder, + { + let dim = shape.into_shape().raw_dim().clone(); + _from_dim(dim).unwrap() + } + /// Creates a new instance given the model size (`inputs`, `d_model`) and total number of nodes within the network (`size`, `network`, `d_network`) + pub fn from_network(model: usize, network: usize) -> Self { + let outputs = network / model; + Self::new(model, outputs) + } + + pub const fn as_array(&self) -> [usize; 2] { + [self.outputs(), self.inputs()] + } + /// Creates a new two-tuple instance from the given dimensions; + pub const fn as_tuple(&self) -> (usize, usize) { + (self.outputs(), self.inputs()) + } + pub fn check_dim(&self, dim: D) -> bool + where + D: Dimension, + { + if dim.ndim() == 1 { + self.inputs() == dim[0] + } else if dim.ndim() >= 2 { + self.outputs() == dim[0] && self.inputs() == dim[1] + } else { + false + } + } + /// Forwards the [into_pattern](ndarray::Dimension::into_pattern) method from the [Dimension] trait + #[inline] + pub fn into_pattern(self) -> (usize, usize) { + self.into_dimension().into_pattern() + } + /// An aliased function that returns the number of input features + pub const fn d_model(&self) -> usize { + self.inputs() + } + /// Returns the number of input features + pub const fn inputs(&self) -> usize { + self.inputs + } + /// Checks to see if the features speak to a so-called `unit`; + /// i.e. see if the number of output features is equal to 1. + pub fn is_unit(&self) -> bool { + self.outputs() == 1 + } + /// Returns the number of output features + pub const fn outputs(&self) -> usize { + self.outputs + } + /// Computes the total number of nodes in the network + pub fn size(&self) -> usize { + self.inputs() * self.outputs() + } + #[doc(hidden)] + pub fn uniform_scale(&self) -> f64 { + (self.inputs as f64).recip().sqrt() + } +} + +impl IntoDimension for Features { + type Dim = Ix2; + + fn into_dimension(self) -> Self::Dim { + (self.outputs, self.inputs).into_dimension() + } +} + +impl From for Features { + fn from(dim: Ix1) -> Self { + Self::new(1, dim[0]) + } +} + +impl From for Features { + fn from(dim: Ix2) -> Self { + Self::new(dim[1], dim[0]) + } +} + +impl From for Ix2 { + fn from(features: Features) -> Self { + features.into_dimension() + } +} + +impl PartialEq for Features +where + [usize; 2]: PartialEq, +{ + fn eq(&self, other: &U) -> bool { + self.as_array() == *other + } +} diff --git a/core/tests/fft.rs b/core/tests/fft.rs index adf29566..61fff93b 100644 --- a/core/tests/fft.rs +++ b/core/tests/fft.rs @@ -8,34 +8,10 @@ use approx::assert_abs_diff_eq; use concision::ops::fft::*; use lazy_static::lazy_static; use num::complex::{Complex, ComplexFloat}; +use num::traits::Float; const EPSILON: f64 = 1e-6; -fn fft_permutation(length: usize) -> Vec { - let mut result = Vec::new(); - result.reserve_exact(length); - for i in 0..length { - result.push(i); - } - let mut reverse = 0_usize; - let mut position = 1_usize; - while position < length { - let mut bit = length >> 1; - while bit & reverse != 0 { - reverse ^= bit; - bit >>= 1; - } - reverse ^= bit; - // This is equivalent to adding 1 to a reversed number - if position < reverse { - // Only swap each element once - result.swap(position, reverse); - } - position += 1; - } - result -} - lazy_static! { static ref EXPECTED_RFFT: Vec> = vec![ Complex { re: 28.0, im: 0.0 }, @@ -55,11 +31,33 @@ lazy_static! { ]; } +fn handle(data: Vec) -> Vec +where + S: ComplexFloat, + T: Copy + Float, +{ + let tmp = { + let mut inner = data + .iter() + .cloned() + .filter(|i| i.im() > T::zero()) + .map(|i| i.conj()) + .collect::>(); + inner.sort_by(|a, b| a.im().partial_cmp(&b.im()).unwrap()); + inner + }; + let mut out = data.clone(); + out.sort_by(|a, b| a.re().partial_cmp(&b.re()).unwrap()); + out.sort_by(|a, b| a.im().partial_cmp(&b.im()).unwrap()); + out.extend(tmp); + out +} + #[test] fn test_plan() { let samples = 16; - let plan = FftPlan::new(samples); + let plan = FftPlan::new(samples).build(); assert_eq!(plan.plan(), fft_permutation(16).as_slice()); } @@ -67,39 +65,26 @@ fn test_plan() { #[ignore = "Needs to be fixed"] fn test_rfft() { let polynomial = (0..8).map(|i| i as f64).collect::>(); - let plan = FftPlan::new(polynomial.len()); - println!("Function Values: {:?}", &polynomial); - println!("Plan: {:?}", &plan); + let plan = FftPlan::new(polynomial.len()).build(); + println!("Function Values: {:?}\nPlan: {:?}", &polynomial, &plan); let fft = rfft(&polynomial, &plan); - let mut tmp = fft - .iter() - .cloned() - .filter(|i| i.im() > 0.0) - .map(|i| i.conj()) - .collect::>(); - tmp.sort_by(|a, b| a.im().partial_cmp(&b.im()).unwrap()); - println!("FFT: {:?}", &tmp); - let mut res = fft.clone(); - res.sort_by(|a, b| a.re().partial_cmp(&b.re()).unwrap()); - res.sort_by(|a, b| a.im().partial_cmp(&b.im()).unwrap()); - println!("R: {:?}", &res); - res.extend(tmp); + let res = handle(fft.clone()); assert!(fft.len() == EXPECTED_RFFT.len()); for (x, y) in fft.iter().zip(EXPECTED_RFFT.iter()) { assert_abs_diff_eq!(x.re(), y.re()); assert_abs_diff_eq!(x.im(), y.im()); } - // let plan = FftPlan::new(fft.len()); - let ifft = dbg!(irfft(&res, &plan)); - for (x, y) in ifft.iter().zip(polynomial.iter()) { - assert_abs_diff_eq!(*x, *y, epsilon = EPSILON); - } + let plan = FftPlan::new(fft.len()).build(); + let _ifft = dbg!(irfft(&res, &plan)); + // for (x, y) in ifft.iter().zip(polynomial.iter()) { + // assert_abs_diff_eq!(*x, *y, epsilon = EPSILON); + // } } #[test] fn small_polynomial_returns_self() { let polynomial = vec![1.0f64, 1.0, 0.0, 2.5]; - let permutation = FftPlan::new(polynomial.len()); + let permutation = FftPlan::new(polynomial.len()).build(); let fft = fft(&polynomial, &permutation); let ifft = ifft(&fft, &permutation) .into_iter() @@ -114,10 +99,10 @@ fn small_polynomial_returns_self() { fn square_small_polynomial() { let mut polynomial = vec![1.0f64, 1.0, 0.0, 2.0]; polynomial.append(&mut vec![0.0; 4]); - let permutation = FftPlan::new(polynomial.len()); - let mut fft = fft(&polynomial, &permutation); + let plan = FftPlan::new(polynomial.len()).build(); + let mut fft = fft(&polynomial, &plan); fft.iter_mut().for_each(|num| *num *= *num); - let ifft = ifft(&fft, &permutation) + let ifft = ifft(&fft, &plan) .into_iter() .map(|i| i.re()) .collect::>(); @@ -135,7 +120,7 @@ fn square_big_polynomial() { let n = 1 << 17; // ~100_000 let mut polynomial = vec![1.0f64; n]; polynomial.append(&mut vec![0.0f64; n]); - let permutation = FftPlan::new(polynomial.len()); + let permutation = FftPlan::new(polynomial.len()).build(); let mut fft = fft(&polynomial, &permutation); fft.iter_mut().for_each(|num| *num *= *num); let ifft = irfft(&fft, &permutation) diff --git a/core/tests/init.rs b/core/tests/init.rs new file mode 100644 index 00000000..39d29e5d --- /dev/null +++ b/core/tests/init.rs @@ -0,0 +1,45 @@ +/* + Appellation: random + Contrib: FL03 +*/ +extern crate concision_core as cnc; + +use cnc::init::distr::LecunNormal; +use cnc::init::InitializeExt; +use ndarray::prelude::*; + +#[test] +fn test_init_ext() { + let shape = [3, 3]; + let seed = 0u64; + let a = Array2::::stdnorm(shape); + let b = Array2::::stdnorm_from_seed(shape, seed); + + assert_eq!(a.shape(), shape); + assert_eq!(a.shape(), b.shape()); +} + +#[test] +fn test_lecun_normal() { + let n = 3; + let shape = (3, 3); + + let distr = LecunNormal::new(n); + + let bnd = 2f64 * distr.std_dev::(); + + let arr = Array2::::lecun_normal(shape, n); + + assert!(arr.iter().all(|&x| x >= -bnd && x <= bnd)); + + assert_eq!(arr.dim(), shape); +} + +#[test] +fn test_truncnorm() { + let (mean, std) = (0f64, 2f64); + let bnd = 2f64 * std; + let shape = (3, 3); + let arr = Array::truncnorm(shape, mean, std).unwrap(); + assert!(arr.iter().all(|&x| x >= -bnd && x <= bnd)); +} diff --git a/core/tests/nn.rs b/core/tests/nn.rs new file mode 100644 index 00000000..55b51198 --- /dev/null +++ b/core/tests/nn.rs @@ -0,0 +1,18 @@ +#![allow(unused_imports)] +extern crate concision_core as concision; + +use concision::nn::DropoutLayer; +use concision::Forward; +use ndarray::prelude::*; + +#[test] +#[cfg(feature = "rand")] +fn test_dropout() { + let shape = (512, 2048); + let arr = Array2::::ones(shape); + let dropout = DropoutLayer::new(0.5); + let out = dropout.forward(&arr); + + assert!(arr.iter().all(|&x| x == 1.0)); + assert!(out.iter().any(|&x| x == 0.0)); +} diff --git a/core/tests/traits.rs b/core/tests/traits.rs index 1778fefd..b1038f94 100644 --- a/core/tests/traits.rs +++ b/core/tests/traits.rs @@ -4,20 +4,32 @@ */ extern crate concision_core as cnc; -use cnc::traits::{Affine, AsComplex, Matpow}; -use ndarray::prelude::{array, Array2}; +use cnc::linarr; +use ndarray::prelude::*; use num::Complex; #[test] fn test_affine() { + use cnc::traits::Affine; let x = array![[0.0, 1.0], [2.0, 3.0]]; let y = x.affine(4.0, -2.0); assert_eq!(y, array![[-2.0, 2.0], [6.0, 10.0]]); } +#[test] +fn test_masked_fill() { + use cnc::traits::MaskFill; + let shape = (2, 2); + let mask = array![[true, false], [false, true]]; + let arr = linarr::(shape).unwrap(); + let a = arr.masked_fill(&mask, 0.0); + assert_eq!(a, array![[0.0, 1.0], [2.0, 0.0]]); +} + #[test] fn test_as_complex() { + use cnc::traits::AsComplex; let x = 1.0; let y = x.as_re(); assert_eq!(y, Complex::new(1.0, 0.0)); @@ -25,8 +37,19 @@ fn test_as_complex() { #[test] fn test_matrix_power() { + use cnc::traits::Matpow; let x = array![[1.0, 2.0], [3.0, 4.0]]; assert_eq!(x.pow(0), Array2::::eye(2)); assert_eq!(x.pow(1), x); assert_eq!(x.pow(2), x.dot(&x)); } + +#[test] +fn test_unsqueeze() { + use cnc::traits::Unsqueeze; + let arr = array![1, 2, 3, 4]; + let a = arr.clone().unsqueeze(0); + assert_eq!(a.dim(), (1, 4)); + let b = arr.unsqueeze(1); + assert_eq!(b.dim(), (4, 1)); +} diff --git a/data/Cargo.toml b/data/Cargo.toml index 742c4424..e6a1c57f 100644 --- a/data/Cargo.toml +++ b/data/Cargo.toml @@ -80,22 +80,37 @@ crate-type = ["lib"] doctest = false test = true +[[test]] +name = "params" +required-features = ["std"] + [build-dependencies] [dependencies] -approx = { optional = true, version = "0.5" } itertools.workspace = true ndarray.workspace = true num.workspace = true -serde = { default-features = false, features = ["derive"], optional = true, version = "1" } smart-default.workspace = true strum.workspace = true -tracing = { optional = true, version = "0.1" } + +[dependencies.approx] +optional = true +version = "0.5" [dependencies.concision-core] default-features = false path = "../core" -version = "0.1.13" +version = "0.1.14" + +[dependencies.serde] +default-features = false +features = ["derive"] +optional = true +version = "1" + +[dependencies.tracing] +optional = true +version = "0.1" [dev-dependencies] diff --git a/data/src/kernel/mod.rs b/data/src/kernel/mod.rs new file mode 100644 index 00000000..e69de29b diff --git a/data/src/lib.rs b/data/src/lib.rs index 13450090..5f1d6ead 100644 --- a/data/src/lib.rs +++ b/data/src/lib.rs @@ -5,25 +5,28 @@ //! # Data //! //! This library works to provide a comprehensive set of utilities for working with datasets. - #![cfg_attr(not(feature = "std"), no_std)] #[cfg(feature = "alloc")] extern crate alloc; extern crate concision_core as concision; +extern crate ndarray as nd; -pub use self::{dataset::Dataset, traits::prelude::*, utils::*}; - -pub(crate) mod utils; +pub use self::dataset::Dataset; +pub use self::traits::prelude::*; pub mod dataset; +pub mod params; +#[doc(hidden)] +pub mod preproc; pub mod tensor; pub mod traits; +pub mod types; pub mod prelude { - pub use crate::utils::*; - - pub use crate::dataset::*; - pub use crate::traits::prelude::*; + pub use super::dataset::*; + pub use super::params::prelude::*; + pub use super::traits::prelude::*; + pub use super::types::prelude::*; } diff --git a/data/src/params/impls/impl_rand.rs b/data/src/params/impls/impl_rand.rs new file mode 100644 index 00000000..d5f066cc --- /dev/null +++ b/data/src/params/impls/impl_rand.rs @@ -0,0 +1,27 @@ +/* + Appellation: impl_rand + Contrib: FL03 +*/ +use crate::params::Parameter; +use concision::init::rand_distr::uniform::SampleUniform; +use concision::init::rand_distr::{Distribution, StandardNormal}; +use concision::InitializeExt; +use ndarray::{Array, Dimension}; +use num::Float; + +impl Parameter +where + D: Dimension, + T: Float, + StandardNormal: Distribution, +{ + pub fn init_uniform(mut self, dk: T) -> Self + where + T: SampleUniform, + ::Sampler: Clone, + { + let dim = self.value.dim(); + self.value = Array::uniform(dim, dk); + self + } +} diff --git a/core/src/params/kinds.rs b/data/src/params/kinds.rs similarity index 86% rename from core/src/params/kinds.rs rename to data/src/params/kinds.rs index 70a71885..b9b92225 100644 --- a/core/src/params/kinds.rs +++ b/data/src/params/kinds.rs @@ -4,19 +4,6 @@ */ use strum::{AsRefStr, EnumCount, EnumIs, EnumIter, EnumString, VariantNames}; -pub trait ParamType: ToString { - fn kind(&self) -> String; -} - -impl ParamType for T -where - T: ToString, -{ - fn kind(&self) -> String { - self.to_string() - } -} - #[derive( AsRefStr, Clone, diff --git a/core/src/params/mod.rs b/data/src/params/mod.rs similarity index 91% rename from core/src/params/mod.rs rename to data/src/params/mod.rs index 5ec083bf..8b35b030 100644 --- a/core/src/params/mod.rs +++ b/data/src/params/mod.rs @@ -37,9 +37,9 @@ pub(crate) mod prelude { #[cfg(test)] mod tests { use super::*; - use crate::linarr; - use ndarray::linalg::Dot; - use ndarray::prelude::{Ix1, Ix2}; + use concision::linarr; + use nd::linalg::Dot; + use nd::prelude::*; #[test] fn test_parameter() { diff --git a/core/src/params/parameter.rs b/data/src/params/parameter.rs similarity index 100% rename from core/src/params/parameter.rs rename to data/src/params/parameter.rs diff --git a/core/src/params/store.rs b/data/src/params/store.rs similarity index 93% rename from core/src/params/store.rs rename to data/src/params/store.rs index 6428c985..cf451e37 100644 --- a/core/src/params/store.rs +++ b/data/src/params/store.rs @@ -2,11 +2,16 @@ Appellation: store Contrib: FL03 */ +#![cfg(any(feature = "alloc", feature = "std"))] use super::{ParamKind, Parameter}; -use crate::prelude::Map; use ndarray::prelude::{Dimension, Ix2}; use num::Float; +#[cfg(all(feature = "alloc", no_std))] +use alloc::collections::BTreeMap as Map; +#[cfg(feature = "std")] +use std::collections::HashMap as Map; + #[derive(Clone, Debug, Default, Eq, PartialEq)] #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] pub struct ParamStore diff --git a/data/src/preproc/mod.rs b/data/src/preproc/mod.rs new file mode 100644 index 00000000..dee29e42 --- /dev/null +++ b/data/src/preproc/mod.rs @@ -0,0 +1,7 @@ +/* + Appellation: preproc + Contrib: FL03 +*/ +//! # Preprocessing +//! +//! This module works to provide a complete set of preprocessing utilities for datasets. diff --git a/data/src/tensor/mod.rs b/data/src/tensor/mod.rs index 17945f2f..7d31345f 100644 --- a/data/src/tensor/mod.rs +++ b/data/src/tensor/mod.rs @@ -2,6 +2,6 @@ Appellation: tensor Contrib: FL03 */ -pub use self::ndtensor::NdTensor; +pub use self::ndtensor::NdContainer; pub mod ndtensor; diff --git a/data/src/tensor/ndtensor/traits.rs b/data/src/tensor/ndtensor/traits.rs index c55c3afe..b125c25b 100644 --- a/data/src/tensor/ndtensor/traits.rs +++ b/data/src/tensor/ndtensor/traits.rs @@ -11,7 +11,7 @@ pub trait TensorData { fn as_mut_slice(&mut self) -> &mut [Self::Elem]; } -pub trait NdTensor { +pub trait NdContainer { const RANK: Option = None; type Data: TensorData; diff --git a/data/src/traits/build.rs b/data/src/traits/build.rs new file mode 100644 index 00000000..7944014b --- /dev/null +++ b/data/src/traits/build.rs @@ -0,0 +1,140 @@ +/* + Appellation: ndarray + Contrib: FL03 +*/ +use crate::traits::Dimensional; +use nd::{ArrayBase, DataOwned, Dimension, RawData, ShapeBuilder}; +use num::{One, Zero}; + +/// [NdBuilder] describes common creation routines for [ArrayBase] +pub trait NdBuilder +where + D: Dimension, +{ + type Data: RawData; + + /// Create a new array with the given shape whose elements are set to the default value of the element type. + fn default(shape: Sh) -> Self + where + A: Default, + Sh: ShapeBuilder, + Self::Data: DataOwned; + + fn fill(shape: Sh, elem: A) -> Self + where + A: Clone, + Sh: ShapeBuilder, + Self::Data: DataOwned; + + fn ones(shape: Sh) -> Self + where + A: Clone + One, + Sh: ShapeBuilder, + Self::Data: DataOwned; + + fn zeros(shape: Sh) -> Self + where + A: Clone + Zero, + Sh: ShapeBuilder, + Self::Data: DataOwned; +} + +pub trait NdBuilderExt: NdBuilder + Sized +where + D: Dimension, +{ + fn dim(&self) -> D::Pattern; + + fn default_like(&self) -> Self + where + A: Default, + Sh: ShapeBuilder, + Self::Data: DataOwned, + { + Self::default(self.dim()) + } + + fn fill_like(&self, elem: A) -> Self + where + A: Clone, + Sh: ShapeBuilder, + Self::Data: DataOwned, + { + Self::fill(self.dim(), elem) + } + + fn ones_like(&self) -> Self + where + A: Clone + One, + Sh: ShapeBuilder, + Self::Data: DataOwned, + { + Self::ones(self.dim()) + } + + fn zeros_like(&self) -> Self + where + A: Clone + Zero, + Sh: ShapeBuilder, + Self::Data: DataOwned, + { + Self::zeros(self.dim()) + } +} + +/* + ************* Implementations ************* +*/ +impl NdBuilder for ArrayBase +where + D: Dimension, + S: RawData, +{ + type Data = S; + + fn default(shape: Sh) -> Self + where + A: Default, + Sh: ShapeBuilder, + Self::Data: DataOwned, + { + ArrayBase::default(shape) + } + + fn fill(shape: Sh, elem: A) -> Self + where + A: Clone, + S: DataOwned, + Sh: ShapeBuilder, + { + ArrayBase::from_elem(shape, elem) + } + + fn ones(shape: Sh) -> Self + where + A: Clone + One, + Sh: ShapeBuilder, + Self::Data: DataOwned, + { + ArrayBase::ones(shape) + } + + fn zeros(shape: Sh) -> Self + where + A: Clone + Zero, + Sh: ShapeBuilder, + Self::Data: DataOwned, + { + ArrayBase::zeros(shape) + } +} + +impl NdBuilderExt for U +where + U: Dimensional + NdBuilder, + D: Dimension, +{ + fn dim(&self) -> D::Pattern { + self.dim() + } +} diff --git a/data/src/traits/data/container.rs b/data/src/traits/data/container.rs new file mode 100644 index 00000000..0d9e0044 --- /dev/null +++ b/data/src/traits/data/container.rs @@ -0,0 +1,25 @@ +/* + Appellation: container + Contrib: FL03 +*/ +use crate::traits::{ContainerRepr, Dimensional}; + +pub trait Container { + type Data: ContainerRepr; +} + +/// This trait describes the basic operations for any n-dimensional container. +pub trait NdContainer: Dimensional { + type Data: ContainerRepr; + + fn as_slice(&self) -> &[A]; + + fn as_mut_slice(&mut self) -> &mut [A]; +} + +/* + ************* Implementations ************* +*/ +impl Container for Vec { + type Data = Vec; +} diff --git a/data/src/traits/data/repr.rs b/data/src/traits/data/repr.rs new file mode 100644 index 00000000..3c583d95 --- /dev/null +++ b/data/src/traits/data/repr.rs @@ -0,0 +1,15 @@ +/* + Appellation: data + Contrib: FL03 +*/ + +pub trait ContainerRepr { + type Elem; +} + +/* + ************* Implementations ************* +*/ +impl ContainerRepr for Vec { + type Elem = T; +} diff --git a/data/src/traits/ext/ndarray.rs b/data/src/traits/ext/ndarray.rs new file mode 100644 index 00000000..6d3e6ed8 --- /dev/null +++ b/data/src/traits/ext/ndarray.rs @@ -0,0 +1,45 @@ +/* + Appellation: ndarray + Contrib: FL03 +*/ +use nd::iter::{Iter, IterMut}; +use nd::{Dimension, RawData}; + +pub trait NdArray +where + D: Dimension, +{ + type Data: RawData; + + fn as_slice(&self) -> &[A]; + + fn as_mut_slice(&mut self) -> &mut [A]; + + fn iter(&self) -> Iter<'_, A, D>; + + fn iter_mut(&mut self) -> IterMut<'_, A, D>; + + fn map(&self, f: F) -> Self + where + F: FnMut(&A) -> A; + + fn mapv(&mut self, f: F) + where + A: Clone, + F: FnMut(A) -> A; +} + +pub trait NdIter +where + D: Dimension, +{ + type Data: RawData; + + fn iter(&self) -> Iter<'_, A, D>; + + fn iter_mut(&mut self) -> IterMut<'_, A, D>; +} + +/* + ************* Implementations ************* +*/ diff --git a/data/src/traits/ext/ndtensor.rs b/data/src/traits/ext/ndtensor.rs new file mode 100644 index 00000000..0edbd756 --- /dev/null +++ b/data/src/traits/ext/ndtensor.rs @@ -0,0 +1,52 @@ +/* + Appellation: ndtensor + Contrib: FL03 +*/ +use nd::{ArrayBase, Data, Dimension, RawData}; +use num::complex::ComplexFloat; +use num::traits::Float; + +pub trait Scalar { + type R: Float; +} + +pub trait NdTensor +where + A: ComplexFloat, + D: Dimension, +{ + type Data: RawData; + type Output; + + fn conj(&self) -> Self::Output; + + fn cos(&self) -> Self::Output; + + fn cosh(&self) -> Self::Output; +} + +/* + ************* Implementations ************* +*/ +impl NdTensor for ArrayBase +where + A: ComplexFloat, + D: Dimension, + S: Data, + Self: Clone, +{ + type Data = S; + type Output = nd::Array; + + fn conj(&self) -> Self::Output { + self.mapv(|x| x.conj()) + } + + fn cos(&self) -> Self::Output { + self.mapv(|x| x.cos()) + } + + fn cosh(&self) -> Self::Output { + self.mapv(|x| x.cosh()) + } +} diff --git a/data/src/traits/ext/ndview.rs b/data/src/traits/ext/ndview.rs new file mode 100644 index 00000000..56b88c3f --- /dev/null +++ b/data/src/traits/ext/ndview.rs @@ -0,0 +1,157 @@ +/* + Appellation: ndview + Contrib: FL03 +*/ +/* + Appellation: ndarray + Contrib: FL03 +*/ +use nd::prelude::*; +use nd::{Data, DataMut, DataOwned, OwnedRepr, RawData}; + +pub trait AsOwned +where + D: Dimension, + S: RawData, +{ + type Output; + + fn into_owned(self) -> Self::Output + where + A: Clone, + S: Data; + + fn to_owned(&self) -> Self::Output + where + A: Clone, + S: Data; +} + +pub trait AsShared +where + D: Dimension, + S: RawData, +{ + type Output; + + fn into_shared(self) -> Self::Output + where + S: DataOwned, + S::Elem: Clone; + + fn to_shared(&self) -> Self::Output + where + S: DataOwned, + S::Elem: Clone; +} + +pub trait NdView, D = Ix2>: AsOwned + AsShared +where + D: Dimension, + S: RawData, +{ + fn view(&self) -> ArrayView<'_, A, D> + where + A: Clone, + S: Data; + + fn view_mut(&mut self) -> ArrayViewMut<'_, A, D> + where + A: Clone, + S: DataMut; +} + +pub trait View +where + D: Dimension, +{ + type Data: RawData; + type Output; + + fn view(&self) -> Self::Output + where + A: Clone, + Self::Data: Data; +} +pub trait ViewMut: View +where + D: Dimension, +{ + fn view_mut(&mut self) -> ArrayViewMut<'_, A, D> + where + A: Clone, + Self::Data: DataMut; +} + +/* + ************* Implementations ************* +*/ +impl AsOwned for ArrayBase +where + D: Dimension, + S: RawData, +{ + type Output = Array; + + fn into_owned(self) -> Self::Output + where + A: Clone, + S: Data, + { + self.into_owned() + } + + fn to_owned(&self) -> Self::Output + where + A: Clone, + S: Data, + { + self.to_owned() + } +} + +impl AsShared for ArrayBase +where + D: Dimension, + S: RawData, +{ + type Output = ArcArray; + + fn into_shared(self) -> Self::Output + where + A: Clone, + S: DataOwned, + { + self.into_shared() + } + + fn to_shared(&self) -> Self::Output + where + A: Clone, + S: DataOwned, + { + self.to_shared() + } +} + +impl NdView for ArrayBase +where + D: Dimension, + S: RawData, +{ + fn view(&self) -> ArrayView<'_, A, D> + where + A: Clone, + S: Data, + { + self.view() + } + + fn view_mut(&mut self) -> ArrayViewMut<'_, A, D> + where + A: Clone, + S: DataMut, + { + self.view_mut() + } +} diff --git a/data/src/traits/mod.rs b/data/src/traits/mod.rs index 83d994c3..14b24d38 100644 --- a/data/src/traits/mod.rs +++ b/data/src/traits/mod.rs @@ -2,10 +2,43 @@ Appellation: traits Contrib: FL03 */ -pub use self::prelude::*; +pub use self::{data::*, ext::*, records::*, shape::*}; + +pub mod build; pub mod records; +pub mod shape; + +#[doc(hidden)] +pub mod data { + pub use self::{container::*, repr::*}; + + pub(crate) mod container; + pub(crate) mod repr; + + pub(crate) mod prelude { + pub use super::container::*; + pub use super::repr::*; + } +} + +pub mod ext { + pub use self::{ndarray::*, ndtensor::*, ndview::*}; + + pub(crate) mod ndarray; + pub(crate) mod ndtensor; + pub(crate) mod ndview; + + pub(crate) mod prelude { + pub use super::ndarray::*; + pub use super::ndtensor::*; + pub use super::ndview::*; + } +} pub(crate) mod prelude { + pub use super::data::prelude::*; + pub use super::ext::prelude::*; pub use super::records::*; + pub use super::shape::*; } diff --git a/data/src/traits/shape.rs b/data/src/traits/shape.rs new file mode 100644 index 00000000..a8127e46 --- /dev/null +++ b/data/src/traits/shape.rs @@ -0,0 +1,96 @@ +/* + Appellation: shape + Contrib: FL03 +*/ +use nd::{ArrayBase, Dimension, RawData}; + +pub trait IntoPattern { + type Pattern; + + fn into_pattern(self) -> Self::Pattern; +} + +/// [Dimensional] provides a common interface for containers to access their shape and dimension. +pub trait Dimensional { + const RANK: Option = None; + + type Dim: IntoPattern; + + fn dim(&self) -> ::Pattern { + self.raw_dim().into_pattern() + } + + fn is_scalar(&self) -> bool { + self.rank() == 0 || self.shape().iter().all(|x| *x == 1) + } + + fn rank(&self) -> usize { + Self::RANK.unwrap_or(self.shape().len()) + } + + fn raw_dim(&self) -> Self::Dim; + + fn size(&self) -> usize { + self.shape().iter().product() + } + + fn shape(&self) -> &[usize]; +} + +/* + ******** implementations ******** +*/ +impl IntoPattern for D +where + D: Dimension, +{ + type Pattern = D::Pattern; + + fn into_pattern(self) -> Self::Pattern { + Dimension::into_pattern(self) + } +} + +// impl Dimensional for D +// where +// D: Dimension + IntoPattern, +// { +// type Dim = D; + +// fn dim(&self) -> D::Pattern { +// self.clone().into_pattern() +// } + +// fn raw_dim(&self) -> D { +// self.clone() +// } + +// fn shape(&self) -> &[usize] { +// D::slice(self) +// } +// } + +impl Dimensional for ArrayBase +where + D: Dimension, + S: RawData, +{ + const RANK: Option = D::NDIM; + type Dim = D; + + fn dim(&self) -> D::Pattern { + ArrayBase::dim(self) + } + + fn raw_dim(&self) -> D { + ArrayBase::raw_dim(self) + } + + fn shape(&self) -> &[usize] { + ArrayBase::shape(self) + } + + fn size(&self) -> usize { + ArrayBase::len(self) + } +} diff --git a/data/src/types/kernel.rs b/data/src/types/kernel.rs new file mode 100644 index 00000000..248ad95f --- /dev/null +++ b/data/src/types/kernel.rs @@ -0,0 +1,6 @@ +/* + Appellation: kernel + Contrib: FL03 +*/ + +pub struct Kernel; diff --git a/data/src/types/mod.rs b/data/src/types/mod.rs new file mode 100644 index 00000000..b8ca6da5 --- /dev/null +++ b/data/src/types/mod.rs @@ -0,0 +1,11 @@ +/* + Appellation: types + Contrib: FL03 +*/ +pub use self::kernel::Kernel; + +pub mod kernel; + +pub(crate) mod prelude { + pub use super::kernel::Kernel; +} diff --git a/data/src/utils.rs b/data/src/utils.rs deleted file mode 100644 index c916d4f5..00000000 --- a/data/src/utils.rs +++ /dev/null @@ -1,23 +0,0 @@ -/* - Appellation: utils - Contrib: FL03 -*/ -use ndarray::linalg::Dot; -use ndarray::prelude::Array2; -use num::traits::Num; - -/// Raise a matrix to a power -pub fn powmat(a: &Array2, n: usize) -> Array2 -where - T: Clone + Num + 'static, - Array2: Dot, Output = Array2>, -{ - if !a.is_square() { - panic!("Matrix must be square"); - } - let mut res = Array2::::eye(a.nrows()); - for _ in 0..n { - res = res.dot(a); - } - res -} diff --git a/core/tests/params.rs b/data/tests/params.rs similarity index 83% rename from core/tests/params.rs rename to data/tests/params.rs index 70a927d6..134ff343 100644 --- a/core/tests/params.rs +++ b/data/tests/params.rs @@ -2,11 +2,13 @@ Appellation: params Contrib: FL03 */ -extern crate concision_core as cnc; +extern crate concision_core as concision; +extern crate concision_data as data; -use cnc::prelude::{linarr, ParamKind, Parameter}; +use concision::linarr; +use data::params::{ParamKind, Parameter}; use ndarray::linalg::Dot; -use ndarray::*; +use ndarray::prelude::*; #[test] fn test_parameter() { diff --git a/derive/Cargo.toml b/derive/Cargo.toml index f885219f..ed6f5917 100644 --- a/derive/Cargo.toml +++ b/derive/Cargo.toml @@ -12,11 +12,20 @@ repository.workspace = true version.workspace = true [features] -default = [] +default = [ + "std" +] + +alloc = [] + +std = [] + +wasi = [] + +wasm = [] [lib] bench = false -doctest = false proc-macro = true test = false diff --git a/derive/build.rs b/derive/build.rs new file mode 100644 index 00000000..940a4ce4 --- /dev/null +++ b/derive/build.rs @@ -0,0 +1,8 @@ +/* + Appellation: build + Contrib: FL03 +*/ + +fn main() { + println!("cargo::rustc-check-cfg=cfg(no_std)"); +} diff --git a/derive/src/lib.rs b/derive/src/lib.rs index 3a2cb732..1303b1fb 100644 --- a/derive/src/lib.rs +++ b/derive/src/lib.rs @@ -1,5 +1,10 @@ /* - Appellation: concision-Derive + Appellation: concision-derive Contrib: FL03 */ -//! # Concision Derive +#![cfg_attr(not(feature = "std"), no_std)] +#![crate_name = "concision_derive"] +#![crate_type = "proc-macro"] + +#[cfg(feature = "alloc")] +extern crate alloc; diff --git a/macros/Cargo.toml b/macros/Cargo.toml index 092e3d2d..2a63e229 100644 --- a/macros/Cargo.toml +++ b/macros/Cargo.toml @@ -12,17 +12,24 @@ repository.workspace = true version.workspace = true [features] -default = [] +default = [ + "std" +] +alloc = [] + +std = [] + +wasi = [] + +wasm = [] [lib] bench = false -crate-type = ["rlib"] -doctest = false +proc-macro = true test = false [dependencies] -ndarray = { features = [], version = "0.15" } proc-macro2 = "1" quote = "1" syn = { features = ["full"], version = "2" } diff --git a/macros/build.rs b/macros/build.rs new file mode 100644 index 00000000..940a4ce4 --- /dev/null +++ b/macros/build.rs @@ -0,0 +1,8 @@ +/* + Appellation: build + Contrib: FL03 +*/ + +fn main() { + println!("cargo::rustc-check-cfg=cfg(no_std)"); +} diff --git a/macros/src/lib.rs b/macros/src/lib.rs index 59dd418c..c46e60d8 100644 --- a/macros/src/lib.rs +++ b/macros/src/lib.rs @@ -2,24 +2,9 @@ Appellation: concision-macros Contrib: FL03 */ -//! # Concision Macros +#![cfg_attr(not(feature = "std"), no_std)] +#![crate_name = "concision_macros"] +#![crate_type = "proc-macro"] -#[macro_export] -macro_rules! linspace { - ( $x:expr ) => { - { - let dim = $x.into_dimension(); - let n = $dim.as_array_view().product(); - ndarray::Array::linspace(T::one(), T::from(n).unwrap(), n).into_shape(dim).unwrap() - } - }; - ( $( $x:expr ),* ) => { - { - let mut res = Vec::new(); - $( - res.push(linarr!($x)); - )* - res - } - }; -} +#[cfg(feature = "alloc")] +extern crate alloc; diff --git a/models/gnn/Cargo.toml b/models/gnn/Cargo.toml index aceb0c23..6cb2b8be 100644 --- a/models/gnn/Cargo.toml +++ b/models/gnn/Cargo.toml @@ -24,6 +24,8 @@ full = [ "serde", ] +# ********* [FF] Dependencies ********* + alloc = [ "concision-core/alloc", ] @@ -41,11 +43,9 @@ blas = [ rand = [ "concision-core/rand", - "dep:ndarray-rand", "num/rand" ] - serde = [ "dep:serde", "serde-ext", @@ -57,18 +57,22 @@ serde-ext = [ "num/serde" ] + + +tracing = [ + "dep:tracing", +] + +# ********* [FF] Environments ********* + std = [ "concision-core/std", "ndarray/std", "num/std", - "serde/std", + "serde?/std", "strum/std", ] -tracing = [ - "dep:tracing", -] - wasm = [ "concision-core/wasm", ] @@ -86,20 +90,29 @@ test = true [build-dependencies] [dependencies] -approx = { optional = true, version = "0.5"} ndarray.workspace = true -ndarray-rand = { optional = true, version = "0.14" } -ndarray-stats.workspace = true num.workspace = true -serde = { default-features = false, features = ["derive"], optional = true, version = "1" } smart-default.workspace = true strum.workspace = true -tracing = { optional = true, version = "0.1" } + +[dependencies.approx] +optional = true +version = "0.5" [dependencies.concision-core] default-features = false path = "../../core" -version = "0.1.13" +version = "0.1.14" + +[dependencies.serde] +default-features = false +features = ["derive"] +optional = true +version = "1" + +[dependencies.tracing] +optional = true +version = "0.1" [dev-dependencies] lazy_static.workspace = true diff --git a/models/gnn/src/lib.rs b/models/gnn/src/lib.rs index df93e46c..546bba6b 100644 --- a/models/gnn/src/lib.rs +++ b/models/gnn/src/lib.rs @@ -9,12 +9,10 @@ #![cfg_attr(not(feature = "std"), no_std)] #![crate_name = "concision_gnn"] -#[cfg(no_std)] +#[cfg(feature = "alloc")] extern crate alloc; + extern crate concision_core as concision; extern crate ndarray as nd; -#[cfg(feature = "rand")] -extern crate ndarray_rand as ndrand; -extern crate ndarray_stats as stats; pub mod prelude {} diff --git a/models/kan/Cargo.toml b/models/kan/Cargo.toml index e6fd3a07..087950e8 100644 --- a/models/kan/Cargo.toml +++ b/models/kan/Cargo.toml @@ -24,6 +24,8 @@ full = [ "serde", ] +# ********* [FF] Dependencies ********* + alloc = [ "concision-core/alloc", ] @@ -44,30 +46,30 @@ rand = [ "num/rand" ] - serde = [ - "dep:serde", - "serde-ext", -] - -serde-ext = [ + "serde-1", "concision-core/serde", "ndarray/serde-1", "num/serde" ] +serde-1 = [ + "dep:serde", +] + +tracing = [ + "dep:tracing", +] + +# ********* [FF] Environments ********* std = [ "concision-core/std", "ndarray/std", "num/std", - "serde/std", + "serde?/std", "strum/std", ] -tracing = [ - "dep:tracing", -] - wasm = [ "concision-core/wasm", ] @@ -85,18 +87,29 @@ test = true [build-dependencies] [dependencies] -approx = { optional = true, version = "0.5"} ndarray.workspace = true num.workspace = true -serde = { default-features = false, features = ["derive"], optional = true, version = "1" } smart-default.workspace = true strum.workspace = true -tracing = { optional = true, version = "0.1" } + +[dependencies.approx] +optional = true +version = "0.5" [dependencies.concision-core] default-features = false path = "../../core" -version = "0.1.13" +version = "0.1.14" + +[dependencies.serde] +default-features = false +features = ["derive"] +optional = true +version = "1" + +[dependencies.tracing] +optional = true +version = "0.1" [dev-dependencies] lazy_static.workspace = true diff --git a/models/kan/src/lib.rs b/models/kan/src/lib.rs index ed4652bd..cd6bc790 100644 --- a/models/kan/src/lib.rs +++ b/models/kan/src/lib.rs @@ -4,11 +4,16 @@ */ //! # Kolmogorov-Arnold Networks (KAN) //! +//! Kolmogorov-Arnold Networks (KAN) are a novel class of neural networks based on the Kolmogorov-Arnold Representation Theorem. +//! These models have already demonstrated that they are viable alternatives to traditional multi-layer perceptrons (MLPs) and convolutional neural networks (CNNs). +//! +//! ### Resources +//! +//! - [Kolmogorov-Arnold Representation](https://arxiv.org/abs/2404.19756) //! - #![cfg_attr(not(feature = "std"), no_std)] -#[cfg(no_std)] +#[cfg(feature = "alloc")] extern crate alloc; extern crate concision_core as concision; extern crate ndarray as nd; diff --git a/models/linear/Cargo.toml b/models/linear/Cargo.toml index 38ebb9a6..ab7770d1 100644 --- a/models/linear/Cargo.toml +++ b/models/linear/Cargo.toml @@ -87,18 +87,21 @@ doctest = true test = true [[test]] -name = "model" -required-features = ["rand"] +name = "linear" +required-features = ["std"] + +[[test]] +name = "norm" +required-features = ["approx", "std"] [[test]] name = "params" -required-features = ["rand"] +required-features = ["std"] [build-dependencies] [dependencies] ndarray.workspace = true -ndarray-stats.workspace = true num.workspace = true smart-default.workspace = true strum.workspace = true @@ -107,6 +110,11 @@ strum.workspace = true optional = true version = "0.5" +[dependencies.concision-core] +default-features = false +path = "../../core" +version = "0.1.14" + [dependencies.serde] default-features = false features = ["derive"] @@ -117,12 +125,6 @@ version = "1" optional = true version = "0.1" - -[dependencies.concision-core] -default-features = false -path = "../../core" -version = "0.1.13" - [dev-dependencies] lazy_static.workspace = true diff --git a/models/linear/src/impls/impl_rand.rs b/models/linear/src/impls/impl_rand.rs index 5f91451e..28bb0126 100644 --- a/models/linear/src/impls/impl_rand.rs +++ b/models/linear/src/impls/impl_rand.rs @@ -4,71 +4,190 @@ */ #![cfg(feature = "rand")] -use crate::params::{ParamMode, ParamsBase}; +use crate::params::{LinearParams, ParamMode, ParamsBase}; use crate::{bias_dim, Linear}; -use concision::prelude::GenerateRandom; -use concision::rand::rand_distr::{uniform, Distribution, StandardNormal}; +use concision::init::rand::Rng; +use concision::init::rand_distr::{uniform::SampleUniform, Distribution, StandardNormal}; +use concision::{Initialize, InitializeExt}; use nd::*; use num::Float; -impl Linear +impl Linear where - A: Float + uniform::SampleUniform, + A: Clone + Float, D: RemoveAxis, K: ParamMode, + S: DataOwned, StandardNormal: Distribution, { - pub fn uniform(self) -> Self { - let biased = self.is_biased(); - Self { - params: self.params.init_uniform(biased), - ..self + pub fn uniform(self) -> Linear> + where + A: SampleUniform, + ::Sampler: Clone, + { + Linear { + config: self.config, + params: self.params.uniform(), } } } -impl ParamsBase, D, K> +impl ParamsBase where - A: Float + uniform::SampleUniform, + A: Clone + Float + SampleUniform, D: RemoveAxis, K: ParamMode, + S: RawData, StandardNormal: Distribution, + ::Sampler: Clone, { + /// Computes the reciprocal of the input features. pub(crate) fn dk(&self) -> A { - A::from(self.in_features()).unwrap().recip().sqrt() + A::from(self.in_features()).unwrap().recip() + } + /// Computes the square root of the reciprical of the input features. + pub(crate) fn dk_sqrt(&self) -> A { + self.dk().sqrt() } - pub fn init_uniform(mut self, biased: bool) -> Self { - if biased { - self = self.init_bias(); + pub fn uniform(self) -> LinearParams + where + S: DataOwned, + { + let dk = self.dk_sqrt(); + self.uniform_between(-dk, dk) + } + + pub fn uniform_between(self, low: A, high: A) -> LinearParams + where + S: DataOwned, + { + let weight = Array::uniform_between(self.raw_dim(), low, high); + let bias = if self.is_biased() && !self.bias.is_some() { + let b_dim = bias_dim(self.raw_dim()); + Some(Array::uniform_between(b_dim, low, high)) + } else if !self.is_biased() && self.bias.is_some() { + None + } else { + self.bias + .as_ref() + .map(|b| Array::uniform_between(b.raw_dim(), low, high)) + }; + LinearParams { + weight, + bias, + _mode: core::marker::PhantomData::, } - self.init_weight() + } +} + +impl Initialize for Linear +where + D: RemoveAxis, + K: ParamMode, + S: DataOwned, + StandardNormal: Distribution, +{ + type Data = OwnedRepr; + fn rand(shape: Sh, distr: Ds) -> Self + where + Sh: ShapeBuilder, + Ds: Clone + Distribution, + { + Self::from_params(ParamsBase::rand(shape, distr)) + } + + fn rand_with(shape: Sh, distr: Ds, rng: &mut R) -> Self + where + R: Rng + ?Sized, + Ds: Clone + Distribution, + Sh: ShapeBuilder, + { + Self::from_params(ParamsBase::rand_with(shape, distr, rng)) } - pub fn init_bias(mut self) -> Self { - let dim = bias_dim(self.raw_dim()); - self.bias = Some(Array::uniform_between(self.dk(), dim)); - self + fn init_rand(self, distr: Ds) -> Self + where + Ds: Clone + Distribution, + Self: Sized, + { + Self::rand(self.dim(), distr) } - pub fn init_weight(mut self) -> Self { - self.weights = Array::uniform_between(self.dk(), self.raw_dim()); - self + fn init_rand_with(self, distr: Ds, rng: &mut R) -> Self + where + R: Rng + ?Sized, + Ds: Clone + Distribution, + { + Self::rand_with(self.dim(), distr, rng) + } +} + +impl Initialize for ParamsBase +where + D: RemoveAxis, + K: ParamMode, + S: DataOwned, + StandardNormal: Distribution, +{ + type Data = S; + fn rand(shape: Sh, distr: Dstr) -> Self + where + Sh: ShapeBuilder, + Dstr: Clone + Distribution, + { + let dim = shape.into_shape().raw_dim().clone(); + let bias = if K::BIASED { + Some(ArrayBase::rand(bias_dim(dim.clone()), distr.clone())) + } else { + None + }; + Self { + weight: ArrayBase::rand(dim, distr), + bias, + _mode: core::marker::PhantomData::, + } } - pub fn uniform(self) -> Self { - let dk = self.dk(); - let bias = if self.is_biased() { - let dim = bias_dim(self.raw_dim()); - Some(Array::uniform_between(dk, dim)) + fn rand_with(shape: Sh, distr: Ds, rng: &mut R) -> Self + where + R: Rng + ?Sized, + S: DataOwned, + Ds: Clone + Distribution, + Sh: ShapeBuilder, + { + let dim = shape.into_shape().raw_dim().clone(); + let bias = if K::BIASED { + Some(ArrayBase::rand_with( + bias_dim(dim.clone()), + distr.clone(), + rng, + )) } else { None }; - let weights = Array::uniform_between(dk, self.raw_dim()); Self { + weight: ArrayBase::rand_with(dim, distr, rng), bias, - weights, - _mode: self._mode, + _mode: core::marker::PhantomData::, } } + + fn init_rand(self, distr: Ds) -> Self + where + S: DataOwned, + Ds: Clone + Distribution, + Self: Sized, + { + Self::rand(self.dim(), distr) + } + + fn init_rand_with(self, distr: Ds, rng: &mut R) -> Self + where + R: Rng + ?Sized, + S: DataOwned, + Ds: Clone + Distribution, + { + Self::rand_with(self.dim(), distr, rng) + } } diff --git a/models/linear/src/impls/model/impl_linear.rs b/models/linear/src/impls/model/impl_linear.rs index a728351b..49ee85ba 100644 --- a/models/linear/src/impls/model/impl_linear.rs +++ b/models/linear/src/impls/model/impl_linear.rs @@ -2,50 +2,52 @@ Appellation: impl_linear Contrib: FL03 */ -use crate::{Config, Linear, LinearParams, ParamMode}; +use crate::{Config, Linear, ParamMode, ParamsBase}; use core::borrow::{Borrow, BorrowMut}; -use nd::RemoveAxis; +use nd::{DataOwned, Ix2, RawData, RemoveAxis}; -impl Linear +impl Linear where K: ParamMode, + S: RawData, { pub fn from_features(inputs: usize, outputs: usize) -> Self where A: Clone + Default, + S: DataOwned, { let config = Config::std(inputs, outputs); - let params = LinearParams::default(config.dim()); + let params = ParamsBase::new(config.dim()); Self { config, params } } } -impl Borrow> for Linear +impl Borrow> for Linear where D: RemoveAxis, - K: ParamMode, + S: RawData, { - fn borrow(&self) -> &Config { + fn borrow(&self) -> &Config { &self.config } } -impl Borrow> for Linear +impl Borrow> for Linear where D: RemoveAxis, - K: ParamMode, + S: RawData, { - fn borrow(&self) -> &LinearParams { + fn borrow(&self) -> &ParamsBase { &self.params } } -impl BorrowMut> for Linear +impl BorrowMut> for Linear where D: RemoveAxis, - K: ParamMode, + S: RawData, { - fn borrow_mut(&mut self) -> &mut LinearParams { + fn borrow_mut(&mut self) -> &mut ParamsBase { &mut self.params } } diff --git a/models/linear/src/impls/model/impl_model.rs b/models/linear/src/impls/model/impl_model.rs index d107fef5..475c66bd 100644 --- a/models/linear/src/impls/model/impl_model.rs +++ b/models/linear/src/impls/model/impl_model.rs @@ -2,17 +2,16 @@ Appellation: impl_model Contrib: FL03 */ -use crate::{Config, Linear, LinearParams, ParamMode}; +use crate::{Config, Linear, LinearParams}; use concision::prelude::{Module, Predict, PredictError}; use nd::RemoveAxis; -impl Module for Linear +impl Module for Linear where D: RemoveAxis, - K: ParamMode, { - type Config = Config; - type Params = LinearParams; + type Config = Config; + type Params = LinearParams; fn config(&self) -> &Self::Config { &self.config @@ -27,11 +26,10 @@ where } } -impl Predict for Linear +impl Predict for Linear where D: RemoveAxis, - K: ParamMode, - LinearParams: Predict, + LinearParams: Predict, { type Output = V; diff --git a/models/linear/src/impls/params/impl_from.rs b/models/linear/src/impls/params/impl_from.rs index 91c2b1b4..108d7bd3 100644 --- a/models/linear/src/impls/params/impl_from.rs +++ b/models/linear/src/impls/params/impl_from.rs @@ -2,8 +2,7 @@ Appellation: impl_from Contrib: FL03 */ -use crate::params::{Biased, NodeBase, Pair, ParamMode, ParamsBase, Unbiased}; -use crate::Features; +use crate::{Biased, Features, NodeBase, Pair, ParamsBase, Unbiased}; #[cfg(all(feature = "alloc", no_std))] use alloc::vec; use core::marker::PhantomData; @@ -24,10 +23,9 @@ where fn into_iter(self) -> Self::IntoIter { let axis = Axis(0); - let bias = self.bias().unwrap(); self.weights() .axis_iter(axis) - .zip(bias.axis_iter(axis)) + .zip(self.bias().axis_iter(axis)) .map(|(w, b)| (w.to_owned(), b.to_owned())) .collect::>() .into_iter() @@ -63,7 +61,7 @@ where let mut iter = nodes.iter(); let node = iter.next().unwrap(); let shape = Features::new(node.0.len(), nodes.len()); - let mut params = ParamsBase::default(shape); + let mut params = ParamsBase::new(shape); params.set_node(0, node.clone()); for (i, node) in iter.into_iter().enumerate() { params.set_node(i + 1, node.clone()); @@ -78,7 +76,7 @@ macro_rules! impl_from { }; (@impl $b:ty) => { - + }; } @@ -92,7 +90,7 @@ where let bias = ArrayBase::from_elem((), bias); Self { bias: Some(bias), - weights, + weight: weights, _mode: PhantomData, } } @@ -100,42 +98,40 @@ where impl From<(Array1, Option)> for ParamsBase, Ix1, K> where A: Clone, - K: ParamMode, { fn from((weights, bias): (Array1, Option)) -> Self { Self { bias: bias.map(|b| ArrayBase::from_elem((), b)), - weights, + weight: weights, _mode: PhantomData, } } } impl From> for ParamsBase - where - D: RemoveAxis, - K: ParamMode, - S: RawData, - { - fn from((weights, bias): NodeBase) -> Self { - Self { - bias, - weights, - _mode: PhantomData::, - } +where + D: RemoveAxis, + S: RawData, +{ + fn from((weights, bias): NodeBase) -> Self { + Self { + bias, + weight: weights, + _mode: PhantomData::, } } +} - impl From, ArrayBase>> for ParamsBase - where - D: RemoveAxis, - S: RawData, - { - fn from((weights, bias): Pair, ArrayBase>) -> Self { - Self { - bias: Some(bias), - weights, - _mode: PhantomData::, - } +impl From, ArrayBase>> for ParamsBase +where + D: RemoveAxis, + S: RawData, +{ + fn from((weights, bias): Pair, ArrayBase>) -> Self { + Self { + bias: Some(bias), + weight: weights, + _mode: PhantomData::, } } +} diff --git a/models/linear/src/impls/params/impl_params.rs b/models/linear/src/impls/params/impl_params.rs index 1f29c8d3..17ae9863 100644 --- a/models/linear/src/impls/params/impl_params.rs +++ b/models/linear/src/impls/params/impl_params.rs @@ -2,7 +2,6 @@ Appellation: params Contrib: FL03 */ -use crate::params::mode::*; use crate::params::ParamsBase; use concision::prelude::{Predict, PredictError}; use core::ops::Add; @@ -13,7 +12,6 @@ use num::complex::ComplexFloat; impl ParamsBase where D: RemoveAxis, - K: ParamMode, S: RawData, { pub fn activate(&mut self, args: &X, f: F) -> Y @@ -26,27 +24,6 @@ where } } -impl<'a, A, B, T, S, D, K> Predict for &'a ParamsBase -where - A: Dot, Output = B>, - B: Add<&'a ArrayBase, Output = B>, - D: RemoveAxis, - K: ParamMode, - S: Data, - T: NdFloat, -{ - type Output = B; - - fn predict(&self, input: &A) -> Result { - let wt = self.weights().t().to_owned(); - let mut res = input.dot(&wt); - if let Some(bias) = self.bias() { - res = res + bias; - } - Ok(res) - } -} - impl Clone for ParamsBase where A: Clone, @@ -55,7 +32,7 @@ where { fn clone(&self) -> Self { Self { - weights: self.weights.clone(), + weight: self.weight.clone(), bias: self.bias.clone(), _mode: self._mode, } @@ -78,84 +55,69 @@ where S: Data, { fn eq(&self, other: &Self) -> bool { - self.weights == other.weights && self.bias == other.bias + self.weights() == other.weight && self.bias == other.bias } } -impl PartialEq<(ArrayBase, Option>)> for ParamsBase +impl PartialEq<(ArrayBase, Option>)> + for ParamsBase where A: PartialEq, D: RemoveAxis, S: Data, { fn eq(&self, (weights, bias): &(ArrayBase, Option>)) -> bool { - self.weights == weights && self.bias == *bias + self.weights() == weights && self.bias.as_ref() == bias.as_ref() } } -impl PartialEq<(ArrayBase, ArrayBase)> for ParamsBase +impl PartialEq<(ArrayBase, ArrayBase)> for ParamsBase where A: PartialEq, D: RemoveAxis, S: Data, { fn eq(&self, (weights, bias): &(ArrayBase, ArrayBase)) -> bool { - let mut cmp = self.weights == weights; - if let Some(b) = &self.bias { - cmp &= b == bias; - } - cmp + self.weights() == weights && self.bias.as_ref() == Some(bias) } } -macro_rules! impl_predict { - ($( $($lt:lifetime)? $name:ident),* $(,)?) => { - $(impl_predict!(@impl $($lt)? $name);)* - }; - (@impl $name:ident) => { - impl Predict for $name - where - A: Dot, Output = B>, - B: for<'a> Add<&'a ArrayBase, Output = B>, - D: RemoveAxis, - K: ParamMode, - S: Data, - T: ComplexFloat, - { - type Output = B; +impl Predict for ParamsBase +where + A: Dot, Output = B>, + B: for<'a> Add<&'a ArrayBase, Output = B>, + D: RemoveAxis, + S: Data, + T: ComplexFloat, +{ + type Output = B; - fn predict(&self, input: &A) -> Result { - let wt = self.weights().t().to_owned(); - let mut res = input.dot(&wt); - if let Some(bias) = self.bias() { - res = res + bias; - } - Ok(res) - } + fn predict(&self, input: &A) -> Result { + let wt = self.weights().t().to_owned(); + let mut res = input.dot(&wt); + if let Some(bias) = self.bias.as_ref() { + res = res + bias; } - }; - (@impl $lt:lifetime $name:ident) => { - impl<'a, A, B, T, S, D, K> Predict for $name - where - A: Dot, Output = B>, - B: for<'a> Add<&'a ArrayBase, Output = B>, - D: RemoveAxis, - K: ParamMode, - S: Data, - T: ComplexFloat, - { - type Output = B; + Ok(res) + } +} + +impl<'a, A, B, T, S, D, K> Predict for &'a ParamsBase +where + A: Dot, Output = B>, + B: Add<&'a ArrayBase, Output = B>, + D: RemoveAxis, + S: Data, + T: ComplexFloat, +{ + type Output = B; - fn predict(&self, input: &A) -> Result { - let wt = self.weights().t().to_owned(); - let mut res = input.dot(&wt); - if let Some(bias) = self.bias() { - res = res + bias; - } - Ok(res) - } + fn predict(&self, input: &A) -> Result { + let wt = self.weights().t().to_owned(); + let mut res = input.dot(&wt); + if let Some(bias) = self.bias.as_ref() { + res = res + bias; } - }; + Ok(res) + } } - -impl_predict!(ParamsBase); diff --git a/models/linear/src/impls/params/impl_serde.rs b/models/linear/src/impls/params/impl_serde.rs index 9d0b77b5..407c0c69 100644 --- a/models/linear/src/impls/params/impl_serde.rs +++ b/models/linear/src/impls/params/impl_serde.rs @@ -4,7 +4,7 @@ */ #![cfg(feature = "serde")] -use crate::params::{Entry, ParamMode, ParamsBase}; +use crate::params::{Parameter, ParamsBase}; use core::marker::PhantomData; use nd::*; use serde::{Deserialize, Deserializer, Serialize, Serializer}; @@ -23,7 +23,7 @@ where let (bias, weights) = Deserialize::deserialize(deserializer)?; Ok(Self { bias, - weights, + weight: weights, _mode: PhantomData, }) } @@ -33,7 +33,6 @@ impl Serialize for ParamsBase where A: Serialize, D: RemoveAxis + Serialize, - K: ParamMode, S: Data, ::Smaller: Dimension + Serialize, { @@ -41,11 +40,11 @@ where where Ser: Serializer, { - (self.bias(), self.weights()).serialize(serializer) + (self.bias.as_ref(), self.weights()).serialize(serializer) } } -impl Serialize for Entry +impl Serialize for Parameter where A: Serialize, S: Data, diff --git a/models/linear/src/lib.rs b/models/linear/src/lib.rs index 88c58908..6f457ab7 100644 --- a/models/linear/src/lib.rs +++ b/models/linear/src/lib.rs @@ -4,8 +4,9 @@ */ //! # Linear Models //! -//! This library implements the framework for building linear models. -//! +//! This library works to provide the necessary tools for creating and training linear models. +//! The primary focus is on the [Linear] model, which is a simple linear model that can be used +//! for regression or classification tasks. #![cfg_attr(not(feature = "std"), no_std)] #[cfg(feature = "alloc")] @@ -13,15 +14,17 @@ extern crate alloc; extern crate concision_core as concision; extern crate ndarray as nd; -extern crate ndarray_stats as stats; +// extern crate ndarray_stats as ndstats; pub use self::model::{Config, Features, Layout, Linear}; -pub use self::params::{mode::*, LinearParams}; +pub use self::norm::LayerNorm; +pub use self::params::{mode::*, ParamsBase}; #[allow(unused_imports)] -pub use self::{traits::prelude::*, utils::*}; +pub use self::{primitives::*, traits::*, utils::*}; #[macro_use] pub(crate) mod macros; +pub(crate) mod primitives; #[macro_use] pub(crate) mod seal; pub(crate) mod utils; @@ -33,6 +36,7 @@ pub mod dense; #[doc(hidden)] pub mod mlp; pub mod model; +pub mod norm; pub mod params; pub mod traits; @@ -53,6 +57,7 @@ mod impls { pub mod prelude { pub use crate::model::prelude::*; + pub use crate::norm::prelude::*; pub use crate::params::prelude::*; - pub use crate::traits::prelude::*; + pub use crate::traits::*; } diff --git a/models/linear/src/macros.rs b/models/linear/src/macros.rs index 977fa5a3..b5672fb9 100644 --- a/models/linear/src/macros.rs +++ b/models/linear/src/macros.rs @@ -3,46 +3,7 @@ Contrib: FL03 */ -#[allow(unused_macros)] -macro_rules! params { - {$($k:ident: $v:expr),* $(,)?} => { - params!(@new $($k: $v),*); - }; - (@new bias: $b:expr, weights: $w:expr, mode: $mode:ty) => { - $crate::params::ParamsBase { - bias: $b, - weights: $w, - _mode: core::marker::PhantomData::<$mode>, - } - }; - (@new bias: $b:expr, weights: $w:expr) => { - params!(@new bias: $b, weights: $w, mode: $crate::params::mode::Biased); - }; - (@new bias: $b:expr, weights: $w:expr) => { - params!(@new bias: Some($b), weights: $w, mode: $crate::params::mode::Biased); - }; - (@new weights: $w:expr) => { - params!(@new bias: None, weights: $w, mode: $crate::params::mode::Unbiased); - }; -} - -macro_rules! impl_param_builder { - ($call:ident where $($rest:tt)*) => { - impl_param_builder!(@impl $call where $($rest)*); - }; - (@impl $call:ident where $($rest:tt)*) => { - pub fn $call(shape: Sh) -> Self - where - Sh: ndarray::ShapeBuilder, - $($rest)* - { - let shape = shape.into_shape(); - let dim = shape.raw_dim().clone(); - ParamsBase { - bias: build_bias(K::BIASED, dim.clone(), |dim| ndarray::ArrayBase::$call(dim)), - weights: ndarray::ArrayBase::$call(dim), - _mode: core::marker::PhantomData, - } - } - }; -} +#[macro_use] +mod model; +#[macro_use] +mod params; diff --git a/models/linear/src/macros/model.rs b/models/linear/src/macros/model.rs new file mode 100644 index 00000000..e0cba303 --- /dev/null +++ b/models/linear/src/macros/model.rs @@ -0,0 +1,22 @@ +/* + Appellation: model + Contrib: FL03 +*/ + +macro_rules! mbuilder { + ($method:ident$(.$call:ident)? where $($rest:tt)*) => { + mbuilder!(@impl $method$(.$call)? where $($rest)*); + }; + (@impl $method:ident where $($rest:tt)*) => { + mbuilder!(@impl $method.$method where $($rest)*); + }; + (@impl $method:ident.$call:ident where $($rest:tt)*) => { + pub fn $method(shape: Sh) -> Self + where + Sh: ndarray::ShapeBuilder, + $($rest)* + { + Linear::from_params($crate::params::ParamsBase::$call(shape)) + } + }; +} diff --git a/models/linear/src/macros/params.rs b/models/linear/src/macros/params.rs new file mode 100644 index 00000000..7da00db5 --- /dev/null +++ b/models/linear/src/macros/params.rs @@ -0,0 +1,92 @@ +/* + Appellation: params + Contrib: FL03 +*/ + +macro_rules! pbuilder { + ($method:ident$(.$call:ident)? where $($rest:tt)*) => { + pbuilder!(@impl $method$(.$call)? where $($rest)*); + }; + (@impl $method:ident where $($rest:tt)*) => { + pbuilder!(@impl $method.$method where $($rest)*); + }; + (@impl $method:ident.$call:ident where $($rest:tt)*) => { + pub fn $method(shape: Sh) -> Self + where + K: $crate::params::mode::ParamMode, + Sh: ndarray::ShapeBuilder, + $($rest)* + { + let dim = shape.into_shape().raw_dim().clone(); + ParamsBase { + bias: build_bias(K::BIASED, dim.clone(), |dim| ndarray::ArrayBase::$call(dim)), + weight: ndarray::ArrayBase::$call(dim), + _mode: ::core::marker::PhantomData::, + } + } + }; +} + +macro_rules! wnbview { + ($method:ident::$($rest:tt)*) => { + wnbview!(@impl $method.$method::$($rest)*); + }; + ($method:ident.$call:ident::$($rest:tt)*) => { + wnbview!(@impl $method.$call::$($rest)*); + }; + (@impl $method:ident.$call:ident::<$view:ident>(self) where $($rest:tt)*) => { + pub fn $method(self) -> $crate::params::ParamsBase<$view, D, K> + where + $($rest)* + { + wnbview!(@apply $call(self)) + } + }; + (@impl $method:ident.$call:ident::<$view:ident>(mut self) where $($rest:tt)*) => { + pub fn $method(mut self) -> $crate::params::ParamsBase<$view, D, K> + where + $($rest)* + { + wnbview!(@apply $call(self).as_mut()) + } + }; + (@impl $method:ident.$call:ident::<$view:ident>(&self) where $($rest:tt)*) => { + pub fn $method(&self) -> $crate::params::ParamsBase<$view, D, K> + where + $($rest)* + { + wnbview!(@apply $call(self).as_ref()) + } + }; + (@impl $method:ident.$call:ident::<$view:ident>(&mut self) where $($rest:tt)*) => { + pub fn $method(&mut self) -> $crate::params::ParamsBase<$view, D, K> + where + $($rest)* + { + wnbview!(@apply $call(self).as_mut()) + } + }; + (@impl $method:ident.$call:ident::<'a, $view:ident>(&self) where $($rest:tt)*) => { + pub fn $method(&self) -> $crate::params::ParamsBase<$view<&'_ A>, D, K> + where + $($rest)* + { + wnbview!(@apply $call(&self).as_ref()) + } + }; + (@impl $method:ident.$call:ident::<'a, $view:ident>(&mut self) where $($rest:tt)*) => { + pub fn $method(&mut self) -> $crate::params::ParamsBase<$view<&'_ mut A>, D, K> + where + $($rest)* + { + wnbview!(@apply $call(self).as_mut()) + } + }; + (@apply $call:ident($self:expr)$(.$as:ident())?) => { + $crate::params::ParamsBase { + bias: $self.bias$(.$as())?.map(|arr| arr.$call()), + weight: $self.weight.$call(), + _mode: $self._mode, + } + }; +} diff --git a/models/linear/src/mlp/mod.rs b/models/linear/src/mlp/mod.rs index 218ee165..c040a93a 100644 --- a/models/linear/src/mlp/mod.rs +++ b/models/linear/src/mlp/mod.rs @@ -18,6 +18,10 @@ pub trait MultiLayerPerceptron { type Output; } -pub trait Neuron { - type Rho; +pub trait Neuron {} + +pub trait Rho { + type Output; + + fn activate(&self, args: T) -> Self::Output; } diff --git a/models/linear/src/mlp/model.rs b/models/linear/src/mlp/model.rs new file mode 100644 index 00000000..4128cb88 --- /dev/null +++ b/models/linear/src/mlp/model.rs @@ -0,0 +1,10 @@ +/* + Appellation: model + Contrib: FL03 +*/ + +pub struct Mlp { + input: I, + hidden: H, + output: O, +} \ No newline at end of file diff --git a/models/linear/src/model/config.rs b/models/linear/src/model/config.rs index 2cb8d9db..ad6f9347 100644 --- a/models/linear/src/model/config.rs +++ b/models/linear/src/model/config.rs @@ -3,47 +3,62 @@ Contrib: FL03 */ use super::layout::{Features, Layout}; -use crate::params::mode::*; +use crate::params::{Biased, Unbiased}; use core::marker::PhantomData; -use nd::{Dimension, IntoDimension, Ix2, RemoveAxis}; +use nd::prelude::*; +use nd::{IntoDimension, RemoveAxis, ShapeError}; #[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] -pub struct Config -where - D: Dimension, -{ +pub struct Config { pub layout: Layout, pub name: String, - _biased: PhantomData, + _biased: PhantomData, } -impl Config +impl Config where D: Dimension, - K: ParamMode, { pub fn new() -> Self { Self { layout: Layout::default(), name: String::new(), - _biased: PhantomData, + _biased: PhantomData::, } } - pub fn into_biased(self) -> Config { + pub fn from_dim(dim: D) -> Result + where + D: Dimension, + { + let layout = Layout::from_dim(dim)?; + let res = Self::new().with_layout(layout); + Ok(res) + } + + pub fn from_shape(shape: Sh) -> Self + where + D: RemoveAxis, + Sh: ShapeBuilder, + { + let layout = Layout::from_shape(shape); + Self::new().with_layout(layout) + } + + pub fn into_biased(self) -> Config { Config { - _biased: PhantomData, layout: self.layout, name: self.name, + _biased: PhantomData::, } } - pub fn into_unbiased(self) -> Config { + pub fn into_unbiased(self) -> Config { Config { - _biased: PhantomData, layout: self.layout, name: self.name, + _biased: PhantomData::, } } @@ -54,7 +69,7 @@ where } } - pub fn with_layout(self, layout: Layout) -> Config + pub fn with_layout(self, layout: Layout) -> Config where E: Dimension, { @@ -65,15 +80,20 @@ where } } - pub fn dim(&self) -> D { - self.layout.dim().clone() - } - - pub fn into_pattern(self) -> D::Pattern { - self.dim().into_pattern() + pub fn with_shape(self, shape: Sh) -> Config + where + E: RemoveAxis, + Sh: ShapeBuilder, + { + Config { + layout: self.layout.with_shape(shape), + name: self.name, + _biased: self._biased, + } } - pub fn into_dimensionality(self, dim: E) -> Result, nd::ShapeError> + /// This function attempts to convert the [layout](Layout) of the [Config] into a new [dimension](ndarray::Dimension) + pub fn into_dimensionality(self, dim: E) -> Result, nd::ShapeError> where E: Dimension, { @@ -84,7 +104,8 @@ where }; Ok(tmp) } - + /// Determine whether the [Config] is [Biased]; + /// Returns true by comparing the [TypeId](core::any::TypeId) of `K` against the [TypeId](core::any::TypeId) of the [Biased] type pub fn is_biased(&self) -> bool where K: 'static, @@ -93,68 +114,65 @@ where TypeId::of::() == TypeId::of::() } - + /// Returns an instance to the [Features] of the [Layout] pub fn features(&self) -> Features { - self.layout.features() + self.layout().features() } - - pub fn layout(&self) -> &Layout { + /// Returns an owned reference to the [Layout] + pub const fn layout(&self) -> &Layout { &self.layout } - + /// Returns an immutable reference to the `name` of the model. pub fn name(&self) -> &str { &self.name } + /// Returns a cloned reference to the [dimension](ndarray::Dimension) of the [layout](Layout) + pub fn dim(&self) -> D { + self.layout().dim() + } + + pub fn into_pattern(self) -> D::Pattern { + self.dim().into_pattern() + } + pub fn ndim(&self) -> usize { - self.layout.ndim() + self.layout().ndim() } } -impl Config { +impl Config { pub fn std(inputs: usize, outputs: usize) -> Self { Self { layout: Layout::new((outputs, inputs).into_dimension()), name: String::new(), - _biased: PhantomData, + _biased: PhantomData::, } } } -impl Config +impl Config where D: Dimension, { + /// The default constructor method for building [Biased] configurations. pub fn biased() -> Self { Self::new() } - - pub fn from_dim_biased(dim: D) -> Self - where - D: RemoveAxis, - { - let layout = Layout::from_dim(dim).unwrap(); - Self::new().with_layout(layout) - } } -impl Config +impl Config where D: Dimension, { pub fn unbiased() -> Self { Self::new() } - - pub fn from_dim(dim: D) -> Config - where - D: RemoveAxis, - { - Config::::new().with_layout(Layout::from_dim(dim).unwrap()) - } } -impl Default for Config +impl concision::Config for Config where D: Dimension {} + +impl Default for Config where D: Dimension, { diff --git a/models/linear/src/model/layer.rs b/models/linear/src/model/layer.rs new file mode 100644 index 00000000..9bb00f52 --- /dev/null +++ b/models/linear/src/model/layer.rs @@ -0,0 +1,193 @@ +/* + Appellation: model + Contrib: FL03 +*/ +use super::{Config, Layout}; +use crate::{Biased, LinearParams, ParamMode, ParamsBase, Unbiased}; +use concision::prelude::{Predict, Result}; +use nd::prelude::*; +use nd::{DataOwned, OwnedRepr, RawData, RemoveAxis}; + +/// An implementation of a linear model. +/// +/// In an effort to streamline the api, the [Linear] model relies upon a [ParamMode] type ([Biased] or [Unbiased](crate::params::mode::Unbiased)) +/// which enables the model to automatically determine whether or not to include a bias term. Doing so allows the model to inherit several methods +/// familar to the underlying [ndarray](https://docs.rs/ndarray) crate. +pub struct Linear> +where + D: Dimension, + S: RawData, +{ + pub(crate) config: Config, + pub(crate) params: ParamsBase, +} + +impl Linear> +where + K: ParamMode, +{ + pub fn std(inputs: usize, outputs: usize) -> Self + where + A: Default, + { + let config = Config::::new().with_shape((inputs, outputs)); + let params = ParamsBase::new(config.features()); + Linear { config, params } + } +} + +impl Linear +where + D: RemoveAxis, + K: ParamMode, + S: RawData, +{ + mbuilder!(new where A: Default, S: DataOwned); + mbuilder!(ones where A: Clone + num::One, S: DataOwned); + mbuilder!(zeros where A: Clone + num::Zero, S: DataOwned); + + pub fn from_config(config: Config) -> Self + where + A: Clone + Default, + K: ParamMode, + S: DataOwned, + { + let params = ParamsBase::new(config.dim()); + Self { config, params } + } + + pub fn from_layout(layout: Layout) -> Self + where + A: Clone + Default, + K: ParamMode, + S: DataOwned, + { + let config = Config::::new().with_layout(layout); + let params = ParamsBase::new(config.dim()); + Self { config, params } + } + + pub fn from_params(params: ParamsBase) -> Self { + let config = Config::::new().with_shape(params.raw_dim()); + Self { config, params } + } + + /// Applies an activcation function onto the prediction of the model. + pub fn activate(&self, args: &X, func: F) -> Result + where + F: Fn(Y) -> Y, + Self: Predict, + { + Ok(func(self.predict(args)?)) + } + + pub const fn config(&self) -> &Config { + &self.config + } + + pub fn weights(&self) -> &ArrayBase { + self.params.weights() + } + + pub fn weights_mut(&mut self) -> &mut ArrayBase { + self.params.weights_mut() + } + + pub const fn params(&self) -> &ParamsBase { + &self.params + } + + pub fn params_mut(&mut self) -> &mut ParamsBase { + &mut self.params + } + + pub fn into_biased(self) -> Linear + where + A: Default, + K: 'static, + S: DataOwned, + { + Linear { + config: self.config.into_biased(), + params: self.params.into_biased(), + } + } + + pub fn into_unbiased(self) -> Linear + where + A: Default, + K: 'static, + S: DataOwned, + { + Linear { + config: self.config.into_unbiased(), + params: self.params.into_unbiased(), + } + } + + pub fn is_biased(&self) -> bool + where + K: 'static, + { + self.config().is_biased() + } + + pub fn with_params(self, params: LinearParams) -> Linear + where + E: RemoveAxis, + { + let config = self.config.into_dimensionality(params.raw_dim()).unwrap(); + Linear { config, params } + } + + pub fn with_name(self, name: impl ToString) -> Self { + Self { + config: self.config.with_name(name), + ..self + } + } + + concision::dimensional!(params()); +} + +impl Linear +where + D: RemoveAxis, + S: RawData, +{ + pub fn biased(shape: Sh) -> Self + where + A: Default, + S: DataOwned, + Sh: ShapeBuilder, + { + let config = Config::::new().with_shape(shape); + let params = ParamsBase::biased(config.dim()); + Linear { config, params } + } + + pub fn bias(&self) -> &ArrayBase { + self.params().bias() + } + + pub fn bias_mut(&mut self) -> &mut ArrayBase { + self.params_mut().bias_mut() + } +} + +impl Linear +where + D: RemoveAxis, + S: RawData, +{ + pub fn unbiased(shape: Sh) -> Self + where + A: Default, + S: DataOwned, + Sh: ShapeBuilder, + { + let config = Config::::new().with_shape(shape); + let params = ParamsBase::unbiased(config.dim()); + Linear { config, params } + } +} diff --git a/models/linear/src/model/layout/layout.rs b/models/linear/src/model/layout/layout.rs index d4ce877a..0cf5df2f 100644 --- a/models/linear/src/model/layout/layout.rs +++ b/models/linear/src/model/layout/layout.rs @@ -9,10 +9,7 @@ use nd::{Dimension, RemoveAxis, ShapeBuilder, ShapeError}; #[derive(Clone, Copy, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] -pub struct Layout -where - D: Dimension, -{ +pub struct Layout { pub(crate) dim: D, pub(crate) features: Features, } @@ -44,6 +41,17 @@ where Self { dim, features } } + pub fn with_shape(self, shape: Sh) -> Layout + where + E: RemoveAxis, + Sh: ShapeBuilder, + { + let shape = shape.into_shape(); + let dim = shape.raw_dim().clone(); + let features = Features::from_shape(dim.clone()); + Layout { dim, features } + } + pub fn as_slice(&self) -> &[usize] { self.dim.slice() } @@ -52,8 +60,8 @@ where self.dim.slice_mut() } - pub fn dim(&self) -> &D { - &self.dim + pub fn dim(&self) -> D { + self.dim.clone() } pub fn features(&self) -> Features { diff --git a/models/linear/src/model/linear.rs b/models/linear/src/model/linear.rs deleted file mode 100644 index 09a10e58..00000000 --- a/models/linear/src/model/linear.rs +++ /dev/null @@ -1,111 +0,0 @@ -/* - Appellation: model - Contrib: FL03 -*/ -use super::{Config, Layout}; -use crate::{Biased, LinearParams, ParamMode}; -use concision::prelude::{Predict, Result}; -use nd::{Array, Dimension, Ix2, RemoveAxis}; - -/// Linear model -pub struct Linear -where - D: Dimension, -{ - pub(crate) config: Config, - pub(crate) params: LinearParams, -} - -impl Linear -where - D: RemoveAxis, - K: ParamMode, -{ - pub fn from_config(config: Config) -> Self - where - A: Clone + Default, - K: 'static, - { - let params = LinearParams::default(config.dim()); - Self { config, params } - } - - pub fn from_layout(layout: Layout) -> Self - where - A: Clone + Default, - { - let config = Config::::new().with_layout(layout); - let params = LinearParams::default(config.dim()); - Self { config, params } - } - - pub fn with_params(self, params: LinearParams) -> Linear - where - E: RemoveAxis, - { - let config = self.config.into_dimensionality(params.raw_dim()).unwrap(); - Linear { config, params } - } - - pub fn activate(&self, args: &X, func: F) -> Result - where - F: for<'a> Fn(&'a Y) -> Y, - Self: Predict, - { - Ok(func(&self.predict(args)?)) - } - - pub const fn config(&self) -> &Config { - &self.config - } - - pub fn weights(&self) -> &Array { - self.params.weights() - } - - pub fn weights_mut(&mut self) -> &mut Array { - self.params.weights_mut() - } - - pub const fn params(&self) -> &LinearParams { - &self.params - } - - pub fn params_mut(&mut self) -> &mut LinearParams { - &mut self.params - } - - pub fn into_biased(self) -> Linear - where - A: Default, - { - Linear { - config: self.config.into_biased(), - params: self.params.into_biased(), - } - } - - pub fn is_biased(&self) -> bool { - K::BIASED || self.config().is_biased() - } - - pub fn with_name(self, name: impl ToString) -> Self { - Self { - config: self.config.with_name(name), - ..self - } - } -} - -impl Linear -where - D: RemoveAxis, -{ - pub fn bias(&self) -> &Array { - self.params.bias().unwrap() - } - - pub fn bias_mut(&mut self) -> &mut Array { - self.params.bias_mut().unwrap() - } -} diff --git a/models/linear/src/model/mod.rs b/models/linear/src/model/mod.rs index f511cb77..66fdce93 100644 --- a/models/linear/src/model/mod.rs +++ b/models/linear/src/model/mod.rs @@ -3,9 +3,9 @@ Contrib: FL03 */ pub use self::layout::prelude::*; -pub use self::{config::Config, linear::Linear}; +pub use self::{config::Config, layer::Linear}; -mod linear; +mod layer; pub mod config; @@ -22,5 +22,5 @@ pub mod layout { } pub(crate) mod prelude { - pub use super::linear::Linear; + pub use super::layer::Linear; } diff --git a/models/linear/src/norm/batch/mod.rs b/models/linear/src/norm/batch/mod.rs new file mode 100644 index 00000000..35c46ebd --- /dev/null +++ b/models/linear/src/norm/batch/mod.rs @@ -0,0 +1,14 @@ +/* + Appellation: batch + Contrib: FL03 +*/ +//! # Batch Normalization +//! +//! +pub use self::model::*; + +mod model; + +pub(crate) mod prelude { + pub use super::BatchNorm; +} diff --git a/models/linear/src/norm/batch/model.rs b/models/linear/src/norm/batch/model.rs new file mode 100644 index 00000000..ab33e536 --- /dev/null +++ b/models/linear/src/norm/batch/model.rs @@ -0,0 +1,6 @@ +/* + Appellation: model + Contrib: FL03 +*/ + +pub struct BatchNorm; diff --git a/models/linear/src/norm/layer/config.rs b/models/linear/src/norm/layer/config.rs new file mode 100644 index 00000000..ef9e7469 --- /dev/null +++ b/models/linear/src/norm/layer/config.rs @@ -0,0 +1,116 @@ +/* + Appellation: config + Contrib: FL03 +*/ +use super::EPSILON; +use nd::prelude::{Axis, Dimension, Ix2}; + +pub struct Config { + pub axis: Option, + pub dim: D, + pub eps: f64, +} + +impl Config +where + D: Dimension, +{ + pub fn new() -> ConfigBuilder { + ConfigBuilder::new() + } + + pub fn axis(&self) -> Option<&Axis> { + self.axis.as_ref() + } + + pub fn axis_mut(&mut self) -> &mut Option { + &mut self.axis + } + + pub fn eps(&self) -> f64 { + self.eps + } + + pub fn eps_mut(&mut self) -> &mut f64 { + &mut self.eps + } + + pub fn dim(&self) -> D::Pattern { + self.raw_dim().into_pattern() + } + + pub fn dim_mut(&mut self) -> &mut D { + &mut self.dim + } + + pub fn ndim(&self) -> usize { + self.dim.ndim() + } + + pub fn raw_dim(&self) -> D { + self.dim.clone() + } + + pub fn shape(&self) -> &[usize] { + self.dim.slice() + } + + pub fn shape_mut(&mut self) -> &mut [usize] { + self.dim.slice_mut() + } +} + +impl Default for Config +where + D: Default, +{ + fn default() -> Self { + Self { + axis: None, + dim: D::default(), + eps: EPSILON, + } + } +} + +pub struct ConfigBuilder { + axis: Option, + dim: D, + eps: f64, +} + +impl ConfigBuilder +where + D: Dimension, +{ + pub fn new() -> Self { + Self { + axis: None, + dim: D::default(), + eps: 1e-5, + } + } + + pub fn axis(mut self, axis: Axis) -> Self { + self.axis = Some(axis); + self + } + + pub fn dim(mut self, dim: D) -> Self { + self.dim = dim; + self + } + + pub fn eps(mut self, eps: f64) -> Self { + self.eps = eps; + self + } + + pub fn build(self) -> Config { + Config { + axis: self.axis, + dim: self.dim, + eps: self.eps, + } + } +} diff --git a/models/linear/src/norm/layer/mod.rs b/models/linear/src/norm/layer/mod.rs new file mode 100644 index 00000000..28254dc1 --- /dev/null +++ b/models/linear/src/norm/layer/mod.rs @@ -0,0 +1,54 @@ +/* + Appellation: layer + Contrib: FL03 +*/ +//! # Layer Normalization +//! +//! This module provides the necessary tools for creating and training layer normalization layers. +pub(crate) use self::utils::*; +pub use self::{config::*, model::*}; + +pub(crate) mod config; +pub(crate) mod model; + +pub const EPSILON: f64 = 1e-5; + +pub(crate) mod prelude { + pub use super::config::Config as LayerNormConfig; + pub use super::model::LayerNorm; +} + +pub(crate) mod utils { + use nd::prelude::*; + use nd::{Data, RemoveAxis}; + use num::traits::{Float, FromPrimitive}; + + pub(crate) fn layer_norm(x: &ArrayBase, eps: f64) -> Array + where + A: Float + FromPrimitive, + D: Dimension, + S: Data, + { + let mean = x.mean().unwrap(); + let denom = { + let eps = A::from(eps).unwrap(); + let var = x.var(A::zero()); + (var + eps).sqrt() + }; + x.mapv(|xi| (xi - mean) / denom) + } + + pub(crate) fn layer_norm_axis(x: &ArrayBase, axis: Axis, eps: f64) -> Array + where + A: Float + FromPrimitive, + D: RemoveAxis, + S: Data, + { + let eps = A::from(eps).unwrap(); + let mean = x.mean_axis(axis).unwrap(); + let var = x.var_axis(axis, A::zero()); + let inv_std = var.mapv(|v| (v + eps).recip().sqrt()); + let x_norm = (x - &mean) * &inv_std; + x_norm + } +} diff --git a/models/linear/src/norm/layer/model.rs b/models/linear/src/norm/layer/model.rs new file mode 100644 index 00000000..1cca2419 --- /dev/null +++ b/models/linear/src/norm/layer/model.rs @@ -0,0 +1,176 @@ +/* + Appellation: layer + Contrib: FL03 +*/ +use super::Config; +use crate::{Biased, LinearParams, ParamMode, Unbiased}; +use concision::Forward; +use nd::prelude::*; +use nd::{Data, RemoveAxis}; +use num::traits::{Float, FromPrimitive, One, Zero}; + +// #62 +/// +/// Layer Normalization directly estimates the normalization statistics from the summed inputs +/// to the neurons within a _hidden_ layer, eliminating the need to introduce any additional dependencies. +/// +/// [LayerNorm] follows the [Layer Normalization](https://arxiv.org/abs/1607.06450) paper. +/// +/// ### Resources +pub struct LayerNorm +where + D: Dimension, +{ + config: Config, + params: LinearParams, +} + +macro_rules! impl_norm_builder { + ($method:ident$(.$call:ident)? where $($rest:tt)*) => { + impl_norm_builder!(@impl $method$(.$call)? where $($rest)*); + }; + (@impl $method:ident where $($rest:tt)*) => { + impl_norm_builder!(@impl $method.$method where $($rest)*); + }; + (@impl $method:ident.$call:ident where $($rest:tt)*) => { + pub fn $method(shape: Sh) -> Self + where + Sh: ShapeBuilder, + $($rest)* + { + Self::from_params(LinearParams::::$call(shape)) + } + }; +} + +impl LayerNorm +where + D: RemoveAxis, + K: ParamMode, +{ + pub fn from_config(config: Config) -> Self + where + A: Default, + { + let params = LinearParams::::new(config.dim()); + Self { config, params } + } + + pub fn from_elem(shape: Sh, elem: A) -> Self + where + A: Clone, + Sh: ShapeBuilder, + { + let dim = shape.into_shape().raw_dim().clone(); + let config = Config::new().dim(dim.clone()).build(); + let params = LinearParams::::from_elem(dim, elem); + Self { config, params } + } + + pub fn from_params(params: LinearParams) -> Self { + let config = Config::new().dim(params.raw_dim()).build(); + Self { config, params } + } + + impl_norm_builder!(new where A: Default); + impl_norm_builder!(ones where A: Clone + One); + impl_norm_builder!(zeros where A: Clone + Zero); + + pub const fn config(&self) -> &Config { + &self.config + } + + pub fn is_biased(&self) -> bool { + self.params().is_biased() + } + /// Returns an immutable reference to the layer's parameters. + pub const fn params(&self) -> &LinearParams { + &self.params + } + /// Returns a mutable reference to the layer's parameters. + pub fn params_mut(&mut self) -> &mut LinearParams { + &mut self.params + } + + pub fn dim(&self) -> D::Pattern { + self.config().dim() + } + + pub fn eps(&self) -> f64 { + self.config().eps() + } + + pub fn ndim(&self) -> usize { + self.config().ndim() + } + + pub fn raw_dim(&self) -> D { + self.config().raw_dim() + } + + pub fn shape(&self) -> &[usize] { + self.config().shape() + } +} + +impl Default for LayerNorm +where + A: Default, + D: RemoveAxis, +{ + fn default() -> Self { + Self { + config: Config::default(), + params: Default::default(), + } + } +} + +impl Default for LayerNorm +where + A: Default, + D: RemoveAxis, +{ + fn default() -> Self { + Self { + config: Config::default(), + params: Default::default(), + } + } +} + +impl Forward> for LayerNorm +where + A: Float + FromPrimitive, + D: RemoveAxis, + S: Data, +{ + type Output = Array; + + fn forward(&self, x: &ArrayBase) -> Self::Output { + let norm = if let Some(axis) = self.config().axis() { + super::layer_norm_axis(x, *axis, self.eps()) + } else { + super::layer_norm(x, self.eps()) + }; + norm * self.params().weights() + self.params().bias() + } +} + +impl Forward> for LayerNorm +where + A: Float + FromPrimitive, + D: RemoveAxis, + S: Data, +{ + type Output = Array; + + fn forward(&self, x: &ArrayBase) -> Self::Output { + let norm = if let Some(axis) = self.config().axis() { + super::layer_norm_axis(x, *axis, self.eps()) + } else { + super::layer_norm(x, self.eps()) + }; + norm * self.params().weights() + } +} diff --git a/models/linear/src/norm/mod.rs b/models/linear/src/norm/mod.rs new file mode 100644 index 00000000..5fa9972d --- /dev/null +++ b/models/linear/src/norm/mod.rs @@ -0,0 +1,16 @@ +/* + Appellation: norm + Contrib: FL03 +*/ +//! # Normalization +//! +//! +pub use self::layer::LayerNorm; + +pub mod batch; +pub mod layer; + +pub(crate) mod prelude { + pub use super::batch::prelude::*; + pub use super::layer::prelude::*; +} diff --git a/models/linear/src/params/entry.rs b/models/linear/src/params/item.rs similarity index 88% rename from models/linear/src/params/entry.rs rename to models/linear/src/params/item.rs index 2a193361..9ddbf28b 100644 --- a/models/linear/src/params/entry.rs +++ b/models/linear/src/params/item.rs @@ -34,7 +34,7 @@ use strum::{AsRefStr, EnumDiscriminants, EnumIs, VariantNames}; ), strum(serialize_all = "lowercase") )] -pub enum Entry +pub enum Parameter where S: RawData, D: RemoveAxis, @@ -43,16 +43,16 @@ where Weight(ArrayBase), } -impl Entry +impl Parameter where D: RemoveAxis, S: RawData, { - pub fn bias(data: ArrayBase) -> Self { + pub fn from_bias(data: ArrayBase) -> Self { Self::Bias(data) } - pub fn weight(data: ArrayBase) -> Self { + pub fn from_weight(data: ArrayBase) -> Self { Self::Weight(data) } } diff --git a/models/linear/src/params/mod.rs b/models/linear/src/params/mod.rs index b5d72583..571bdecd 100644 --- a/models/linear/src/params/mod.rs +++ b/models/linear/src/params/mod.rs @@ -3,53 +3,18 @@ Contrib: FL03 */ #[doc(inline)] -pub use self::entry::{Entry, Param}; -pub use self::mode::{Biased, ParamMode, Unbiased}; -pub use self::params::ParamsBase; +pub use self::item::{Param, Parameter}; +pub use self::mode::*; +pub use self::store::*; -mod params; +mod store; -pub mod entry; +pub mod item; pub mod mode; -use nd::{ArrayBase, Ix0, Ix1}; - -pub(crate) type Pair = (A, B); -pub(crate) type MaybePair = Pair>; -pub(crate) type NodeBase = MaybePair, ArrayBase>; -pub(crate) type Node = NodeBase, D, E>; - -macro_rules! params_ty { - ($($name:ident<$repr:ident>),* $(,)?) => { - $(params_ty!(@impl $name<$repr>);)* - }; - (@impl $name:ident<$repr:ident>) => { - pub type $name = $crate::params::ParamsBase, D, K>; - }; -} - -params_ty!(LinearParams, LinearParamsShared,); +#[doc(inline)] +pub use crate::primitives::params::*; pub(crate) mod prelude { - pub use super::{LinearParams, LinearParamsShared}; -} - -#[cfg(test)] -mod tests { - use super::*; - use core::str::FromStr; - - #[test] - fn test_param_kind() { - for i in [(Param::Bias, "bias"), (Param::Weight, "weight")].iter() { - let kind = Param::from_str(i.1).unwrap(); - assert_eq!(i.0, kind); - } - } - - #[test] - fn test_ones() { - let a = LinearParams::::ones((1, 300)); - assert!(a.is_biased()); - } + pub use super::mode::*; } diff --git a/models/linear/src/params/mode.rs b/models/linear/src/params/mode.rs index 53e16726..ff153868 100644 --- a/models/linear/src/params/mode.rs +++ b/models/linear/src/params/mode.rs @@ -2,21 +2,23 @@ Appellation: mode Contrib: FL03 */ +use concision::Toggle; +use core::option::Option; -pub trait State { - type Mode: ParamMode; -} - -pub trait ParamMode: 'static { +pub trait ParamMode: Toggle { const BIASED: bool = false; fn is_biased(&self) -> bool { - Self::BIASED + core::any::TypeId::of::() == core::any::TypeId::of::() } private!(); } +/* + ************* Implementations ************* +*/ + impl ParamMode for Option where T: 'static, @@ -30,15 +32,17 @@ where seal!(); } -macro_rules! param_mode { +macro_rules! mode { {$($T:ident: $opt:expr),* $(,)?} => { - $(param_mode!(@impl $T: $opt);)* + $(mode!(@impl $T: $opt);)* }; (@impl $T:ident: $opt:expr) => { #[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize,))] pub enum $T {} + impl Toggle for $T {} + impl ParamMode for $T { const BIASED: bool = $opt; @@ -52,7 +56,7 @@ macro_rules! param_mode { } -param_mode! { +mode! { Biased: true, Unbiased: false, } diff --git a/models/linear/src/params/params.rs b/models/linear/src/params/params.rs deleted file mode 100644 index 2769f01e..00000000 --- a/models/linear/src/params/params.rs +++ /dev/null @@ -1,204 +0,0 @@ -/* - Appellation: params - Contrib: FL03 -*/ -use super::mode::*; -use super::Node; -use crate::{build_bias, Features}; -use core::marker::PhantomData; -use nd::*; -use num::{One, Zero}; - -/// -pub struct ParamsBase, D = Ix2, K = Unbiased> -where - D: Dimension, - S: RawData, -{ - pub(crate) bias: Option>, - pub(crate) weights: ArrayBase, - pub(crate) _mode: PhantomData, -} - -impl ParamsBase -where - D: RemoveAxis, - K: ParamMode, - S: RawData, -{ - impl_param_builder!(default where A: Default, S: DataOwned); - impl_param_builder!(ones where A: Clone + One, S: DataOwned); - impl_param_builder!(zeros where A: Clone + Zero, S: DataOwned); - - pub fn new(shape: Sh) -> Self - where - A: Default, - S: DataOwned, - Sh: ShapeBuilder, - { - Self { - bias: None, - weights: ArrayBase::default(shape), - _mode: PhantomData, - } - } - - pub fn build(shape: Sh, builder: F) -> Self - where - F: Fn(Sh) -> ArrayBase, - Sh: ShapeBuilder, - { - let _weights = builder(shape); - unimplemented!() - } - - pub fn into_biased(self) -> ParamsBase - where - A: Default, - S: DataOwned, - { - let sm = crate::bias_dim(self.raw_dim()); - ParamsBase { - bias: Some(ArrayBase::default(sm)), - weights: self.weights, - _mode: PhantomData, - } - } - - pub fn into_unbiased(self) -> ParamsBase { - ParamsBase { - bias: None, - weights: self.weights, - _mode: PhantomData, - } - } - - pub fn bias(&self) -> Option<&ArrayBase> { - self.bias.as_ref() - } - - pub fn bias_mut(&mut self) -> Option<&mut ArrayBase> { - self.bias.as_mut() - } - - pub const fn weights(&self) -> &ArrayBase { - &self.weights - } - - pub fn weights_mut(&mut self) -> &mut ArrayBase { - &mut self.weights - } - - pub fn features(&self) -> Features { - Features::from_shape(self.shape()) - } - - pub fn in_features(&self) -> usize { - self.features().dmodel() - } - - pub fn is_biased(&self) -> bool { - self.bias().is_some() - } - - pub fn ndim(&self) -> usize { - self.weights().ndim() - } - - pub fn out_features(&self) -> usize { - if self.ndim() == 1 { - return 1; - } - self.shape()[1] - } - /// Returns the raw dimension of the weights. - pub fn raw_dim(&self) -> D { - self.weights().raw_dim() - } - /// Returns the shape of the weights. - pub fn shape(&self) -> &[usize] { - self.weights().shape() - } -} - -impl ParamsBase -where - D: RemoveAxis, - S: RawData, -{ - pub fn biased(shape: Sh) -> Self - where - A: Default, - S: DataOwned, - Sh: ShapeBuilder, - { - let dim = shape.into_shape().raw_dim().clone(); - Self { - bias: build_bias(true, dim.clone(), ArrayBase::default), - weights: ArrayBase::default(dim), - _mode: PhantomData, - } - } -} - -impl ParamsBase -where - D: Dimension, - S: RawData, -{ - pub fn unbiased() -> Self - where - A: Default, - S: DataOwned, - { - Self { - bias: None, - weights: Default::default(), - _mode: PhantomData, - } - } -} -impl ParamsBase -where - K: ParamMode, - S: RawData, -{ - pub fn set_node(&mut self, idx: usize, node: Node) - where - A: Clone + Default, - S: DataMut + DataOwned, - { - let (weight, bias) = node; - if let Some(bias) = bias { - if !self.is_biased() { - let mut tmp = ArrayBase::default(self.out_features()); - tmp.index_axis_mut(Axis(0), idx).assign(&bias); - self.bias = Some(tmp); - } - self.bias - .as_mut() - .unwrap() - .index_axis_mut(Axis(0), idx) - .assign(&bias); - } - - self.weights_mut() - .index_axis_mut(Axis(0), idx) - .assign(&weight); - } -} - -impl Default for ParamsBase -where - A: Default, - D: Dimension, - S: DataOwned, -{ - fn default() -> Self { - Self { - bias: None, - weights: Default::default(), - _mode: PhantomData, - } - } -} diff --git a/models/linear/src/params/store.rs b/models/linear/src/params/store.rs new file mode 100644 index 00000000..b8afc752 --- /dev/null +++ b/models/linear/src/params/store.rs @@ -0,0 +1,240 @@ +/* + Appellation: params + Contrib: FL03 +*/ +use crate::{build_bias, Biased, Features, Node, ParamMode, Unbiased}; +use concision::dimensional; +use core::marker::PhantomData; +use nd::*; +use num::{One, Zero}; + +/// The [ParamsBase] struct is a generic store for linear parameters. The store mimics +/// the underlying [ArrayBase](ndarray::ArrayBase), enabling developers to specify +/// the data repr and dimension. Additionally, the store is parameterized to +/// accept a `K` type, used to designate the store as either [Biased](crate::Biased) or [Unbiased](crate::Unbiased). +pub struct ParamsBase, D = Ix2, K = Biased> +where + D: Dimension, + S: RawData, +{ + pub(crate) bias: Option>, + pub(crate) weight: ArrayBase, + pub(crate) _mode: PhantomData, +} + +impl ParamsBase +where + D: RemoveAxis, + S: RawData, +{ + pub fn from_elem(shape: Sh, elem: A) -> Self + where + A: Clone, + K: ParamMode, + S: DataOwned, + Sh: ShapeBuilder, + { + let dim = shape.into_shape().raw_dim().clone(); + let bias = if K::BIASED { + Some(ArrayBase::from_elem( + crate::bias_dim(dim.clone()), + elem.clone(), + )) + } else { + None + }; + Self { + bias, + weight: ArrayBase::from_elem(dim, elem), + _mode: PhantomData::, + } + } + + pub fn into_biased(self) -> ParamsBase + where + A: Default, + K: 'static, + S: DataOwned, + { + if self.is_biased() { + return ParamsBase { + bias: self.bias, + weight: self.weight, + _mode: PhantomData::, + }; + } + let sm = crate::bias_dim(self.raw_dim()); + ParamsBase { + bias: Some(ArrayBase::default(sm)), + weight: self.weight, + _mode: PhantomData::, + } + } + + pub fn into_unbiased(self) -> ParamsBase { + ParamsBase { + bias: None, + weight: self.weight, + _mode: PhantomData::, + } + } + + pub const fn weights(&self) -> &ArrayBase { + &self.weight + } + + pub fn weights_mut(&mut self) -> &mut ArrayBase { + &mut self.weight + } + + pub fn features(&self) -> Features { + Features::from_shape(self.shape()) + } + + pub fn in_features(&self) -> usize { + self.features().dmodel() + } + + pub fn out_features(&self) -> usize { + if self.ndim() == 1 { + return 1; + } + self.shape()[1] + } + /// Returns true if the parameter store is biased; + /// Compares the [TypeId](core::any::TypeId) of the store with the [Biased](crate::Biased) type. + pub fn is_biased(&self) -> bool + where + K: 'static, + { + crate::is_biased::() + } + + pbuilder!(new.default where A: Default, S: DataOwned); + + pbuilder!(ones where A: Clone + One, S: DataOwned); + + pbuilder!(zeros where A: Clone + Zero, S: DataOwned); + + dimensional!(weights()); + + wnbview!(into_owned::(self) where A: Clone, S: Data); + + wnbview!(into_shared::(self) where A: Clone, S: DataOwned); + + wnbview!(to_owned::(&self) where A: Clone, S: Data); + + wnbview!(to_shared::(&self) where A: Clone, S: DataOwned); + + wnbview!(view::<'a, ViewRepr>(&self) where A: Clone, S: Data); + + wnbview!(view_mut::<'a, ViewRepr>(&mut self) where A: Clone, S: DataMut); +} + +impl ParamsBase +where + D: RemoveAxis, + S: RawData, +{ + /// Create a new biased parameter store from the given shape. + pub fn biased(shape: Sh) -> Self + where + A: Default, + S: DataOwned, + Sh: ShapeBuilder, + { + let dim = shape.into_shape().raw_dim().clone(); + Self { + bias: build_bias(true, dim.clone(), ArrayBase::default), + weight: ArrayBase::default(dim), + _mode: PhantomData::, + } + } + /// Return an unwraped, immutable reference to the bias array. + pub fn bias(&self) -> &ArrayBase { + self.bias.as_ref().unwrap() + } + /// Return an unwraped, mutable reference to the bias array. + pub fn bias_mut(&mut self) -> &mut ArrayBase { + self.bias.as_mut().unwrap() + } +} + +impl ParamsBase +where + D: Dimension, + S: RawData, +{ + /// Create a new unbiased parameter store from the given shape. + pub fn unbiased(shape: Sh) -> Self + where + A: Default, + S: DataOwned, + Sh: ShapeBuilder, + { + Self { + bias: None, + weight: ArrayBase::default(shape), + _mode: PhantomData::, + } + } +} +impl ParamsBase +where + K: 'static, + S: RawData, +{ + pub fn set_node(&mut self, idx: usize, node: Node) + where + A: Clone + Default, + S: DataMut + DataOwned, + { + let (weight, bias) = node; + if let Some(bias) = bias { + if !self.is_biased() { + let mut tmp = ArrayBase::default(self.out_features()); + tmp.index_axis_mut(Axis(0), idx).assign(&bias); + self.bias = Some(tmp); + } + self.bias + .as_mut() + .unwrap() + .index_axis_mut(Axis(0), idx) + .assign(&bias); + } + + self.weights_mut() + .index_axis_mut(Axis(0), idx) + .assign(&weight); + } +} + +impl Default for ParamsBase +where + A: Default, + D: Dimension, + S: DataOwned, +{ + fn default() -> Self { + Self { + bias: Some(Default::default()), + weight: Default::default(), + _mode: PhantomData::, + } + } +} + +impl Default for ParamsBase +where + A: Default, + D: Dimension, + S: DataOwned, +{ + fn default() -> Self { + Self { + bias: None, + weight: Default::default(), + _mode: PhantomData::, + } + } +} diff --git a/models/linear/src/primitives.rs b/models/linear/src/primitives.rs new file mode 100644 index 00000000..6cda38fd --- /dev/null +++ b/models/linear/src/primitives.rs @@ -0,0 +1,28 @@ +/* + Appellation: primitives + Contrib: FL03 +*/ +pub use self::params::*; +use nd::{ArrayBase, Ix0, Ix1}; + +pub(crate) type Pair = (A, B); + +pub(crate) type MaybePair = Pair>; + +pub(crate) type NodeBase = MaybePair, ArrayBase>; + +pub(crate) type Node = NodeBase, D, E>; + +pub(crate) mod params { + + macro_rules! params_ty { + ($($name:ident<$repr:ident>),* $(,)?) => { + $(params_ty!(@impl $name<$repr>);)* + }; + (@impl $name:ident<$repr:ident>) => { + pub type $name = $crate::params::ParamsBase, D, K>; + }; + } + + params_ty!(LinearParams, LinearParamsShared,); +} diff --git a/models/linear/src/traits.rs b/models/linear/src/traits.rs new file mode 100644 index 00000000..5d609a02 --- /dev/null +++ b/models/linear/src/traits.rs @@ -0,0 +1,18 @@ +/* + Appellation: traits + Contrib: FL03 +*/ +use crate::Biased; + +pub trait IsBiased { + fn is_biased(&self) -> bool; +} + +impl IsBiased for T +where + T: 'static, +{ + fn is_biased(&self) -> bool { + core::any::TypeId::of::() == core::any::TypeId::of::() + } +} diff --git a/models/linear/src/traits/mod.rs b/models/linear/src/traits/mod.rs deleted file mode 100644 index 30db944d..00000000 --- a/models/linear/src/traits/mod.rs +++ /dev/null @@ -1,11 +0,0 @@ -/* - Appellation: traits - Contrib: FL03 -*/ -pub use self::wnb::*; - -pub mod wnb; - -pub(crate) mod prelude { - pub use super::wnb::*; -} diff --git a/models/linear/src/traits/wnb.rs b/models/linear/src/traits/wnb.rs deleted file mode 100644 index 03e511cc..00000000 --- a/models/linear/src/traits/wnb.rs +++ /dev/null @@ -1,57 +0,0 @@ -/* - Appellation: wnb - Contrib: FL03 -*/ -use nd::*; - -pub trait WnB -where - D: Dimension, - S: RawData, -{ - fn bias(&self) -> Option<&ArrayBase>; - - fn bias_mut(&mut self) -> Option<&mut ArrayBase>; - - fn weight(&self) -> &ArrayBase; - - fn weight_mut(&mut self) -> &mut ArrayBase; -} - -pub trait Dimensional { - type Pattern; - - fn dim(&self) -> Self::Pattern; - - fn raw_dim(&self) -> D; - - fn shape(&self) -> &[usize]; -} - -impl Dimensional for ArrayBase -where - D: Dimension, - S: RawData, -{ - type Pattern = D::Pattern; - - fn shape(&self) -> &[usize] { - ArrayBase::shape(self) - } - - fn dim(&self) -> Self::Pattern { - ArrayBase::dim(self) - } - - fn raw_dim(&self) -> D { - ArrayBase::raw_dim(self) - } -} - -pub trait IsBiased { - fn is_biased(&self) -> bool; -} - -/* - ********* Implementations ********* -*/ diff --git a/models/linear/src/utils.rs b/models/linear/src/utils.rs index abd8fb77..ca6142dc 100644 --- a/models/linear/src/utils.rs +++ b/models/linear/src/utils.rs @@ -2,7 +2,8 @@ Appellation: utils Contrib: FL03 */ -use nd::*; +use crate::params::Biased; +use nd::{ArrayBase, Axis, Dimension, RawData, RemoveAxis}; /// A utilitarian funciton for building bias tensors. pub(crate) fn build_bias(biased: bool, dim: D, builder: F) -> Option> @@ -30,3 +31,10 @@ where dim.remove_axis(Axis(1)) } } + +/// A utilitarian function for checking if a type is [Biased]; returns false otherwise. +/// Compares the [TypeId](core::any::TypeId) of `K` to the [TypeId](core::any::TypeId) of [Biased]. +pub fn is_biased() -> bool { + use core::any::TypeId; + TypeId::of::() == TypeId::of::() +} diff --git a/models/linear/tests/model.rs b/models/linear/tests/linear.rs similarity index 69% rename from models/linear/tests/model.rs rename to models/linear/tests/linear.rs index ca168e60..a941510d 100644 --- a/models/linear/tests/model.rs +++ b/models/linear/tests/linear.rs @@ -23,32 +23,35 @@ lazy_static! { #[test] fn test_config() { let dim = FEATURES.clone().into_dimension(); - let config = Config::from_dim_biased(dim); + let config = Config::::from_shape(dim); assert!(config.is_biased()); - let config = Config::from_dim(dim); + let config = Config::::from_shape(dim); assert!(!config.is_biased()); } #[test] -fn test_linear() { - let (samples, (outputs, inputs)) = SHAPE; +fn test_model_toggle() { + let (_samples, (outputs, inputs)) = SHAPE; - let model: Linear = Linear::from_features(inputs, outputs).uniform(); + let model = Linear::::from_features(inputs, outputs); + assert!(model.is_biased()); - let data = linarr::((samples, inputs)).unwrap(); - let y = model.activate(&data, Sigmoid::sigmoid).unwrap(); + let model = Linear::::from_features(inputs, outputs); + assert!(!model.is_biased()); - assert_eq!(y.shape(), &[samples, outputs]); + let model = Linear::::from_features(inputs, outputs).into_unbiased(); + assert!(!model.is_biased()); } #[test] -fn test_bias_ty() { - use linear::{Biased, Unbiased}; - let (_samples, (outputs, inputs)) = SHAPE; +#[cfg(feature = "rand")] +fn test_linear() { + let (samples, (outputs, inputs)) = SHAPE; - let model: Linear = Linear::from_features(inputs, outputs).uniform(); - assert!(model.is_biased()); + let model = Linear::::from_features(inputs, outputs).uniform(); - let model: Linear = Linear::from_features(inputs, outputs).uniform(); - assert!(!model.is_biased()); + let data = linarr::((samples, inputs)).unwrap(); + let y = model.activate(&data, Sigmoid::sigmoid).unwrap(); + + assert_eq!(y.shape(), &[samples, outputs]); } diff --git a/models/linear/tests/norm.rs b/models/linear/tests/norm.rs new file mode 100644 index 00000000..119b67d4 --- /dev/null +++ b/models/linear/tests/norm.rs @@ -0,0 +1,35 @@ +/* + Appellation: norm + Contrib: FL03 +*/ +extern crate concision_core as concision; +extern crate concision_linear as linear; + +use concision::{linarr, Forward}; +use linear::{Biased, LayerNorm}; + +use approx::assert_abs_diff_eq; +use lazy_static::lazy_static; +use ndarray::prelude::*; + +const SHAPE: (usize, usize) = (3, 3); + +lazy_static! { + static ref NORM: Array2 = array![ + [-0.5492, -0.1619, 0.2254], + [0.6127, 1.0000, 1.3873], + [1.7746, 2.1619, 2.5492], + ]; +} + +#[test] +fn test_layer_norm() { + let shape = SHAPE; + let x = linarr::(shape).unwrap(); + + let ln = LayerNorm::::ones(shape); + let y = ln.forward(&x); + + assert_eq!(y.dim(), shape); + assert_abs_diff_eq!(y, *NORM, epsilon = 1e-4); +} diff --git a/models/linear/tests/params.rs b/models/linear/tests/params.rs index acfaa093..fed65fb7 100644 --- a/models/linear/tests/params.rs +++ b/models/linear/tests/params.rs @@ -2,22 +2,46 @@ Appellation: params Contrib: FL03 */ +#![allow(unused_imports)] extern crate concision_core as concision; extern crate concision_linear as linear; use concision::Predict; -use linear::{Features, LinearParams}; +use core::str::FromStr; +use linear::params::{LinearParams, Param, Unbiased}; +use linear::Features; use ndarray::prelude::*; const SAMPLES: usize = 20; -const INPUTS: usize = 5; -const DMODEL: usize = 3; +const D_MODEL: usize = 5; +const FEATURES: usize = 3; #[test] +fn test_keys() { + for i in [(Param::Bias, "bias"), (Param::Weight, "weight")].iter() { + let kind = Param::from_str(i.1).unwrap(); + assert_eq!(i.0, kind); + } +} + +#[test] +fn test_builders() { + let shape = (D_MODEL, FEATURES); + let params = LinearParams::::ones(shape); + assert!(params.is_biased()); + assert_eq!(params.weights(), &Array2::ones(shape)); + assert_eq!(params.bias(), &Array1::ones(D_MODEL)); + let params = LinearParams::::zeros(shape); + assert!(!params.is_biased()); + assert_eq!(params.weights(), &Array2::zeros(shape)); +} + +#[test] +#[cfg(feature = "rand")] fn test_linear_params() { - let (samples, inputs, outputs) = (SAMPLES, INPUTS, DMODEL); + let (samples, inputs, outputs) = (SAMPLES, D_MODEL, FEATURES); let features = Features::new(outputs, inputs); - let data = Array2::::zeros((samples, inputs)); + let data = Array2::::ones((samples, inputs)); let params = LinearParams::biased(features).uniform(); let y: Array2 = params.predict(&data).unwrap(); assert_eq!(y.dim(), (samples, outputs)); diff --git a/models/transformers/Cargo.toml b/models/transformers/Cargo.toml new file mode 100644 index 00000000..7dc36953 --- /dev/null +++ b/models/transformers/Cargo.toml @@ -0,0 +1,144 @@ +[package] +authors.workspace = true +build = "build.rs" +categories.workspace = true +description.workspace = true +edition.workspace = true +homepage.workspace = true +keywords.workspace = true +license.workspace = true +name = "concision-transformers" +readme.workspace = true +repository.workspace = true +version.workspace = true + +[features] +default = [ + "std", +] + +full = [ + "default", + "approx", + "rand", + "serde", +] + +# ********* [FF] Dependencies ********* + +alloc = [ + "concision-core/alloc", + "concision-linear/alloc", + "serde?/alloc", +] + +approx = [ + "dep:approx", + "concision-core/approx", + "concision-linear/approx", + "ndarray/approx-0_5", +] + +blas = [ + "concision-core/blas", + "concision-linear/blas", + "ndarray/blas", +] + +rand = [ + "concision-core/rand", + "concision-linear/rand", + "num/rand" +] + +serde = [ + "serde-1", + "concision-core/serde", + "concision-linear/serde", + "ndarray/serde-1", + "num/serde" +] + +serde-1 = [ + "dep:serde", +] + +tracing = [ + "dep:tracing", + "concision-core/tracing", + "concision-linear/tracing", +] + +# ********* [FF] Environments ********* +std = [ + "concision-core/std", + "concision-linear/std", + "ndarray/std", + "num/std", + "serde?/std", + "strum/std", +] + +wasm = [ + "concision-core/wasm", + "concision-linear/wasm", +] + +wasi = [ + "concision-core/wasi", + "concision-linear/wasi", +] + +[lib] +bench = false +crate-type = ["lib"] +doctest = true +test = true + +[[test]] +name = "attention" +required-features = ["approx", "rand"] + +[build-dependencies] + +[dependencies] +ndarray.workspace = true +num.workspace = true +paste.workspace = true +smart-default.workspace = true +strum.workspace = true + +[dependencies.approx] +optional = true +version = "0.5" + +[dependencies.concision-core] +default-features = false +path = "../../core" +version = "0.1.14" + +[dependencies.concision-linear] +default-features = false +path = "../linear" +version = "0.1.14" + +[dependencies.serde] +default-features = false +features = ["derive"] +optional = true +version = "1" + +[dependencies.tracing] +optional = true +version = "0.1" + +[dev-dependencies] +lazy_static.workspace = true + +[package.metadata.docs.rs] +all-features = true +rustc-args = ["--cfg", "docsrs"] + +[target.wasm32-unknown-unknown] + +[target.wasm32-wasi] diff --git a/models/transformers/build.rs b/models/transformers/build.rs new file mode 100644 index 00000000..940a4ce4 --- /dev/null +++ b/models/transformers/build.rs @@ -0,0 +1,8 @@ +/* + Appellation: build + Contrib: FL03 +*/ + +fn main() { + println!("cargo::rustc-check-cfg=cfg(no_std)"); +} diff --git a/models/transformers/src/attention/head.rs b/models/transformers/src/attention/head.rs new file mode 100644 index 00000000..e80fdda9 --- /dev/null +++ b/models/transformers/src/attention/head.rs @@ -0,0 +1,174 @@ +/* + Appellation: head + Contrib: FL03 +*/ +use super::{Score, _attention}; +use crate::params::QkvBase; +use concision::getters; +use concision::nn::DropoutLayer; +use nd::linalg::Dot; +use nd::*; +use num::complex::ComplexFloat; + +// #68 +/// [AttentionHead] implements the scaled dot-product attention mechanism formally defined in +/// [Attention is all you need](https://arxiv.org/abs/1706.03762). The structure is designed to +/// be flexible, relying upon the n-dimensional [QkvBase] to store the query, key, and value tensors. +/// More so, the head may be configured with an optional dropout and/or masking layers. +/// +/// ### Dropout +/// +/// The [DropoutLayer] is an optional layer applied after the softmax function is applied to the +/// score. The layer is used to prevent overfitting by randomly setting a fraction of the input +/// units to zero at each update during training time. +/// +/// ### Masking +/// +/// After computing the dot-product of the query and key tensors, an optional mask may be applied to +/// the attention score. The mask is used to prevent the model from attending to certain parts of the +/// input sequence. For example, in the case of a language model, the mask may be used to prevent the +/// model from attending to the padding tokens. +pub struct AttentionHead> +where + D: Dimension, + S: RawData, +{ + #[cfg(feature = "rand")] + pub(crate) dropout: Option, + pub(crate) mask: Option>, + pub(crate) params: QkvBase, +} + +impl AttentionHead +where + S: RawData, +{ + pub fn std(dm: usize, dk: usize) -> Self + where + A: Default, + S: DataOwned, + { + Self::from_params(QkvBase::new((dk, dm))) + } +} + +impl AttentionHead +where + D: Dimension, + S: RawData, +{ + pub fn from_params(params: QkvBase) -> Self { + Self { + #[cfg(feature = "rand")] + dropout: None, + mask: None, + params, + } + } + + pub fn builder(shape: Sh, builder: F) -> Self + where + F: Fn(D) -> ArrayBase, + Sh: ShapeBuilder, + { + Self::from_params(QkvBase::builder(shape, builder)) + } + + pub fn from_elem(shape: Sh, value: A) -> Self + where + Sh: ShapeBuilder, + A: Clone, + S: DataOwned, + { + Self::from_params(QkvBase::from_elem(shape, value)) + } + /// Computes the [Score] using scaled dot-product attention. + pub fn attention(&self) -> Score + where + A: ComplexFloat + ScalarOperand, + S: Data, + ArrayBase: for<'a> Dot, Output = Array>, + Array: Dot, Output = Array>, + { + let (q, k, v) = self.qkv(); + _attention(q, k, v, self.mask(), self.dropout()) + } + /// Returns an immutable reference to the, optional, mask. + pub fn mask(&self) -> Option<&Array> { + self.mask.as_ref() + } + /// Returns an immuable reference to the underlying parameters. + pub const fn params(&self) -> &QkvBase { + &self.params + } + /// Returns a mutable reference to the underlying parameters. + pub fn params_mut(&mut self) -> &mut QkvBase { + &mut self.params + } + /// Returns a three-tuple consisting of immputable references to the query, key, and value matrices respectively. + pub fn qkv(&self) -> (&ArrayBase, &ArrayBase, &ArrayBase) { + self.params().qkv() + } + /// Consumes the head, returning a three-tuple consisting of mutable references to the query, key, and value matrices respectively. + pub fn into_qkv(self) -> (ArrayBase, ArrayBase, ArrayBase) { + self.params.into_qkv() + } + /// Sets the dropout layer for the [AttentionHead] + #[cfg(feature = "rand")] + pub fn set_dropout(&mut self, dropout: Option) { + self.dropout = dropout; + } + /// Sets the mask for the [AttentionHead] + pub fn set_mask(&mut self, mask: Option>) { + self.mask = mask; + } + /// Configure the [AttentionHead] with a [DropoutLayer] + #[cfg(feature = "rand")] + pub fn with_dropout(self, dropout: DropoutLayer) -> Self { + Self { + dropout: Some(dropout), + ..self + } + } + /// Consume and store a mask for the [AttentionHead] + pub fn with_mask(self, mask: Array) -> Self { + Self { + mask: Some(mask), + ..self + } + } + + getters!(params::<[q, k, v]> => ArrayBase); + ndbuilder!(new::default() where A: Default, S: DataOwned); + ndbuilder!(ones() where A: Clone + num::One, S: DataOwned); + ndbuilder!(zeros() where A: Clone + num::Zero, S: DataOwned); +} + +#[cfg(feature = "rand")] +impl AttentionHead +where + D: Dimension, + S: RawData, +{ + /// Returns an immutable reference to the, optional, [dropout](DropoutLayer) layer. + /// With the `rand` feature flag disabled, the dropout layer is + /// unavailable and returns `None`. + pub fn dropout(&self) -> Option<&DropoutLayer> { + self.dropout.as_ref() + } +} + +#[cfg(not(feature = "rand"))] +impl AttentionHead +where + D: Dimension, + S: RawData, +{ + /// Returns an immutable reference to the, optional, [dropout](DropoutLayer) layer. + /// With the `rand` feature flag disabled, the dropout layer is + /// unavailable and returns `None`. + #[cfg(not(feature = "rand"))] + pub fn dropout(&self) -> Option<&DropoutLayer> { + None + } +} diff --git a/models/transformers/src/attention/mod.rs b/models/transformers/src/attention/mod.rs new file mode 100644 index 00000000..a500b5f5 --- /dev/null +++ b/models/transformers/src/attention/mod.rs @@ -0,0 +1,102 @@ +/* + Appellation: attention + Contrib: FL03 +*/ +//! # Attention +//! +//! Attention allows a model to focus on specific parts of the input sequence. +//! Today, these mechanisms are found in several state-of-the-art models, such as +//! the Transformer model, primarily due to its capabilities in natural language +//! processing (NLP) domains +pub(crate) use self::_impl_methods::*; +pub use self::head::AttentionHead; +pub use self::score::Score; +pub use self::utils::*; + +pub(crate) mod head; +pub(crate) mod score; + +// #69: Multi-Head Attention implementation +pub mod multi; + +pub(crate) mod prelude { + pub use super::head::AttentionHead; + pub use super::multi::prelude::*; + pub use super::score::Score; + pub use super::utils::*; +} + +pub trait Attention { + type Output; + + fn attention(&self) -> Self::Output; +} + +pub(crate) mod utils { + use super::Score; + use concision::nn::DropoutLayer; + use nd::linalg::Dot; + use nd::prelude::*; + use num::complex::ComplexFloat; + + /// A functional implementation of the scaled dot-product attention mechanism; + pub fn scaled_dot_product_attention( + q: &ArrayBase, + k: &ArrayBase, + v: &ArrayBase, + mask: Option<&Array>, + dropout: Option<&DropoutLayer>, + ) -> Score + where + A: ComplexFloat + nd::ScalarOperand, + S: nd::Data, + D: Dimension, + ArrayBase: for<'a> Dot, Output = Array>, + Array: Dot, Output = Array>, + { + super::_attention(q, k, v, mask, dropout) + } +} + +mod _impl_methods { + use super::Score; + use concision::prelude::{DropoutLayer, MaskFill, Softmax}; + use nd::linalg::Dot; + use nd::prelude::*; + use num::complex::ComplexFloat; + + pub(crate) fn _attention( + q: &ArrayBase, + k: &ArrayBase, + v: &ArrayBase, + mask: Option<&Array>, + dropout: Option<&DropoutLayer>, + ) -> Score + where + A: ComplexFloat + nd::ScalarOperand, + S: nd::Data, + D: Dimension, + ArrayBase: for<'a> Dot, Output = Array>, + Array: Dot, Output = Array>, + { + use concision::Forward; + let dk = scale::(k.len_of(nd::Axis(1))); + let mut z = q.dot(&k.t()) * dk; + if let Some(mask) = mask { + z = z.masked_fill(mask, A::zero()); + } + z = z.softmax(); + #[cfg(feature = "rand")] + if let Some(dropout) = dropout { + z = dropout.forward(&z); + } + (z.dot(&v), z).into() + } + + pub(crate) fn scale(dk: usize) -> A + where + A: ComplexFloat, + { + A::from(dk).unwrap().sqrt().recip() + } +} diff --git a/models/transformers/src/attention/multi/config.rs b/models/transformers/src/attention/multi/config.rs new file mode 100644 index 00000000..58c510c6 --- /dev/null +++ b/models/transformers/src/attention/multi/config.rs @@ -0,0 +1,49 @@ +/* + Appellation: config + Contrib: FL03 +*/ + +pub(crate) fn dk(d_model: usize, heads: usize) -> usize { + d_model / heads +} + +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +#[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] +pub struct Config { + pub d_model: usize, + pub heads: usize, +} + +impl Config { + pub fn new() -> ConfigBuilder { + ConfigBuilder::new() + } + + pub fn d_model(&self) -> usize { + self.d_model + } + + pub fn dk(&self) -> usize { + dk(self.d_model(), self.heads()) + } + + pub fn heads(&self) -> usize { + self.heads + } +} + +impl Default for Config { + fn default() -> Self { + Self { + d_model: crate::D_MODEL, + heads: crate::HEADS, + } + } +} + +concision::builder! { + ConfigBuilder(Config) { + d_model: usize, + heads: usize, + } +} diff --git a/models/transformers/src/attention/multi/mod.rs b/models/transformers/src/attention/multi/mod.rs new file mode 100644 index 00000000..e101f032 --- /dev/null +++ b/models/transformers/src/attention/multi/mod.rs @@ -0,0 +1,16 @@ +/* + Appellation: multi + Contrib: FL03 +*/ +//! # Multi-Head Attention +//! +//! +pub use self::{config::Config, multi_head::*}; + +pub(crate) mod config; +pub(crate) mod multi_head; + +pub(crate) mod prelude { + pub use super::config::Config as MultiHeadAttentionConfig; + pub use super::multi_head::MultiHeadAttention; +} diff --git a/models/transformers/src/attention/multi/multi_head.rs b/models/transformers/src/attention/multi/multi_head.rs new file mode 100644 index 00000000..36a4051d --- /dev/null +++ b/models/transformers/src/attention/multi/multi_head.rs @@ -0,0 +1,77 @@ +/* + Appellation: multi_head + Contrib: FL03 +*/ +use super::Config; +use crate::AttentionHead; +use linear::{Biased, Linear}; +use nd::prelude::*; +use nd::{DataOwned, OwnedRepr, RawData}; + +pub struct MultiHeadAttention> +where + D: Dimension, + S: RawData, +{ + pub(crate) config: Config, + pub(crate) head: AttentionHead, + pub(crate) linears: Vec>, +} + +impl MultiHeadAttention +where + D: Dimension, + S: RawData, +{ + pub const fn config(&self) -> &Config { + &self.config + } + + pub const fn head(&self) -> &AttentionHead { + &self.head + } + + pub fn head_mut(&mut self) -> &mut AttentionHead { + &mut self.head + } + + pub fn linears(&self) -> &[Linear] { + &self.linears + } +} + +impl MultiHeadAttention +where + S: RawData, +{ + pub fn std(d_model: usize, heads: usize) -> Self + where + A: Clone + Default, + S: DataOwned, + { + let config = Config::new().d_model(d_model).heads(heads).build(); + let linears = (0..4) + .map(|_| Linear::from_features(d_model, d_model)) + .collect(); + Self { + config, + head: AttentionHead::std(d_model, config.dk()), + linears, + } + } +} + +impl Default for MultiHeadAttention +where + A: Default, + D: Dimension, + S: DataOwned, +{ + fn default() -> Self { + Self { + config: Config::default(), + head: AttentionHead::default(), + linears: Vec::new(), + } + } +} diff --git a/models/transformers/src/attention/score.rs b/models/transformers/src/attention/score.rs new file mode 100644 index 00000000..3e1df96e --- /dev/null +++ b/models/transformers/src/attention/score.rs @@ -0,0 +1,94 @@ +/* + Appellation: score + Contrib: FL03 +*/ +use core::fmt; +use nd::{Array, Dimension}; + +/// [Score] is a created as a result of invoking an attention mechanism; +/// +/// - attention: the actual result; returns the dot product of the score with the value tensor +/// - score: the attention score tensor +#[derive(Clone, Eq, Hash, PartialEq)] +pub struct Score +where + D: Dimension, +{ + pub(crate) attention: Array, + pub(crate) score: Array, +} + +impl Score +where + D: Dimension, +{ + pub(crate) fn new(attention: Array, score: Array) -> Self { + Self { attention, score } + } + /// Consumes the instance and returns the attention tensor. + pub fn into_attention(self) -> Array { + self.attention + } + /// Consumes the container and returns the score tensor. + pub fn into_score(self) -> Array { + self.score + } + + /// Retrieve the attention tensor. + pub fn attention(&self) -> &Array { + &self.attention + } + /// Retrieve the score tensor + pub fn score(&self) -> &Array { + &self.score + } +} + +impl Copy for Score +where + A: Copy, + D: Copy + Dimension, + Array: Copy, +{ +} + +impl fmt::Debug for Score +where + A: fmt::Debug, + D: Dimension, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Score") + .field("attention", &self.attention) + .field("score", &self.score) + .finish() + } +} + +impl fmt::Display for Score +where + A: fmt::Display, + D: Dimension, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "({}, {})", self.attention, self.score) + } +} + +impl From<(Array, Array)> for Score +where + D: Dimension, +{ + fn from((attention, score): (Array, Array)) -> Self { + Self::new(attention, score) + } +} + +impl From> for (Array, Array) +where + D: Dimension, +{ + fn from(score: Score) -> Self { + (score.attention, score.score) + } +} diff --git a/models/transformers/src/codec/decoder.rs b/models/transformers/src/codec/decoder.rs new file mode 100644 index 00000000..3d00af1c --- /dev/null +++ b/models/transformers/src/codec/decoder.rs @@ -0,0 +1,31 @@ +/* + Appellation: decoder + Contrib: FL03 +*/ +pub use self::{config::DecoderConfig, layer::DecoderLayer}; + +pub mod config; +pub mod layer; + +#[derive(Default)] +pub struct Decoder { + config: DecoderConfig, + layers: Vec, +} + +impl Decoder { + pub fn new() -> Self { + Self { + config: DecoderConfig::default(), + layers: Vec::new(), + } + } + + pub const fn config(&self) -> &DecoderConfig { + &self.config + } + + pub fn layers(&self) -> &[DecoderLayer] { + &self.layers + } +} diff --git a/models/transformers/src/codec/decoder/config.rs b/models/transformers/src/codec/decoder/config.rs new file mode 100644 index 00000000..8056b7cd --- /dev/null +++ b/models/transformers/src/codec/decoder/config.rs @@ -0,0 +1,14 @@ +/* + Appellation: config + Contrib: FL03 +*/ + +pub struct DecoderConfig { + pub layers: usize, +} + +impl Default for DecoderConfig { + fn default() -> Self { + Self { layers: crate::N } + } +} diff --git a/models/transformers/src/codec/decoder/layer.rs b/models/transformers/src/codec/decoder/layer.rs new file mode 100644 index 00000000..90514acc --- /dev/null +++ b/models/transformers/src/codec/decoder/layer.rs @@ -0,0 +1,13 @@ +/* + Appellation: layer + Contrib: FL03 +*/ + +#[derive(Default)] +pub struct DecoderLayer {} + +impl DecoderLayer { + pub fn new() -> Self { + Self {} + } +} diff --git a/models/transformers/src/codec/encoder.rs b/models/transformers/src/codec/encoder.rs new file mode 100644 index 00000000..e9fdb490 --- /dev/null +++ b/models/transformers/src/codec/encoder.rs @@ -0,0 +1,39 @@ +/* + Appellation: encoder + Contrib: FL03 +*/ +pub use self::{config::EncoderConfig, layer::EncoderLayer}; + +pub mod config; +pub mod layer; + +use linear::norm::LayerNorm; + +#[derive(Default)] +pub struct Encoder { + config: EncoderConfig, + layers: Vec, + norm: LayerNorm, +} + +impl Encoder { + pub fn new() -> Self { + Self { + config: EncoderConfig::default(), + layers: Vec::new(), + norm: LayerNorm::default(), + } + } + + pub const fn config(&self) -> &EncoderConfig { + &self.config + } + + pub fn layers(&self) -> &[EncoderLayer] { + &self.layers + } + + pub fn norm(&self) -> &LayerNorm { + &self.norm + } +} diff --git a/models/transformers/src/codec/encoder/config.rs b/models/transformers/src/codec/encoder/config.rs new file mode 100644 index 00000000..2c5dcf93 --- /dev/null +++ b/models/transformers/src/codec/encoder/config.rs @@ -0,0 +1,14 @@ +/* + Appellation: config + Contrib: FL03 +*/ + +pub struct EncoderConfig { + pub layers: usize, +} + +impl Default for EncoderConfig { + fn default() -> Self { + Self { layers: crate::N } + } +} diff --git a/models/transformers/src/codec/encoder/layer.rs b/models/transformers/src/codec/encoder/layer.rs new file mode 100644 index 00000000..5c00ebcf --- /dev/null +++ b/models/transformers/src/codec/encoder/layer.rs @@ -0,0 +1,26 @@ +/* + Appellation: layer + Contrib: FL03 +*/ +use crate::attention::multi::MultiHeadAttention; + +#[derive(Default)] +pub struct EncoderLayer { + pub(crate) attention: MultiHeadAttention, +} + +impl EncoderLayer { + pub fn new() -> Self { + let attention = MultiHeadAttention::default(); + + Self { attention } + } + /// Returns an immutable reference to the multi-head, self-attention layer. + pub fn attention(&self) -> &MultiHeadAttention { + &self.attention + } + /// Returns a mutable reference to the multi-head, self-attention layer. + pub fn attention_mut(&mut self) -> &mut MultiHeadAttention { + &mut self.attention + } +} diff --git a/models/transformers/src/codec/mod.rs b/models/transformers/src/codec/mod.rs new file mode 100644 index 00000000..52e34740 --- /dev/null +++ b/models/transformers/src/codec/mod.rs @@ -0,0 +1,32 @@ +/* + Appellation: codec + Contrib: FL03 +*/ +//! # Codec +//! +//! The `codec` module implements the [Decoder] and [Encoder] layers of the [Transformer](crate::Transformer) model. +//! Each layer has two sublayers, namely: +//! - multi-head, self-attention layer +//! - fully-connected, piecewise feed-forward network. +//! +pub use self::{decoder::Decoder, encoder::Encoder, model::*}; + +pub(crate) mod model; + +pub mod decoder; +pub mod encoder; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_codec_builder() { + let ctx = Context::new() + .with_src("src".to_string()) + .with_tgt("tgt".to_string()); + let codec = Codec::new().ctx(ctx).build(); + assert_eq!(codec.context().src, "src"); + assert_eq!(codec.context().tgt, "tgt"); + } +} diff --git a/models/transformers/src/codec/model.rs b/models/transformers/src/codec/model.rs new file mode 100644 index 00000000..470938a5 --- /dev/null +++ b/models/transformers/src/codec/model.rs @@ -0,0 +1,62 @@ +/* + Appellation: codec + Contrib: FL03 +*/ +use super::{Decoder, Encoder}; +use concision::{builder, getters}; + +#[derive(Default)] +pub struct Codec { + ctx: Context, + decoder: Decoder, + encoder: Encoder, +} + +impl Codec { + pub fn new() -> CodecBuilder { + CodecBuilder::new() + } + + getters!( + context.ctx, + decoder, + encoder, + ); +} + +builder! { + CodecBuilder(Codec) { + ctx: Context, + decoder: Decoder, + encoder: Encoder, + } +} + +#[derive(Default)] +pub struct Generator { + pub dmodel: usize, + pub vocab: Vec, +} + +#[derive(Default)] +pub struct Context { + pub src: String, // source embedding + pub tgt: String, // target embedding +} + +impl Context { + pub fn new() -> Self { + Self { + src: String::new(), + tgt: String::new(), + } + } + + pub fn with_src(self, src: String) -> Self { + Self { src, ..self } + } + + pub fn with_tgt(self, tgt: String) -> Self { + Self { tgt, ..self } + } +} diff --git a/models/transformers/src/impls/impl_head.rs b/models/transformers/src/impls/impl_head.rs new file mode 100644 index 00000000..4160975d --- /dev/null +++ b/models/transformers/src/impls/impl_head.rs @@ -0,0 +1,92 @@ +/* + Appellation: impl_head + Contrib: FL03 +*/ +use crate::attention::{Attention, AttentionHead, Score}; +use crate::params::QkvBase; +use core::borrow::{Borrow, BorrowMut}; +use nd::linalg::Dot; +use nd::prelude::*; +use nd::{Data, DataOwned, RawData, RawDataClone, ScalarOperand}; +use num::complex::ComplexFloat; + +impl Attention for AttentionHead +where + A: ComplexFloat + ScalarOperand, + D: Dimension, + S: Data, + ArrayBase: for<'a> Dot, Output = Array>, + Array: Dot, Output = Array>, +{ + type Output = Score; + + fn attention(&self) -> Self::Output { + self.attention() + } +} + +impl Borrow> for AttentionHead +where + D: Dimension, + S: RawData, +{ + fn borrow(&self) -> &QkvBase { + self.params() + } +} + +impl BorrowMut> for AttentionHead +where + D: Dimension, + S: RawData, +{ + fn borrow_mut(&mut self) -> &mut QkvBase { + self.params_mut() + } +} + +impl Clone for AttentionHead +where + A: Copy, + D: Dimension, + S: RawDataClone, +{ + fn clone(&self) -> Self { + Self { + #[cfg(feature = "rand")] + dropout: self.dropout.clone(), + mask: self.mask.clone(), + params: self.params.clone(), + } + } +} + +impl Copy for AttentionHead +where + A: Copy, + D: Copy + Dimension, + S: Copy + RawDataClone, + Array: Copy, +{ +} + +impl Default for AttentionHead +where + A: Default, + D: Dimension, + S: DataOwned, +{ + fn default() -> Self { + Self::from_params(QkvBase::default()) + } +} + +impl From> for AttentionHead +where + D: Dimension, + S: RawData, +{ + fn from(params: QkvBase) -> Self { + Self::from_params(params) + } +} diff --git a/models/transformers/src/impls/impl_init.rs b/models/transformers/src/impls/impl_init.rs new file mode 100644 index 00000000..1ed7effd --- /dev/null +++ b/models/transformers/src/impls/impl_init.rs @@ -0,0 +1,64 @@ +/* + Appellation: init + Contrib: FL03 +*/ +#![cfg(feature = "rand")] +use crate::QkvBase; +use concision::Initialize; +use concision::init::rand::Rng; +use concision::init::rand_distr::{Distribution, StandardNormal}; +use concision::init::rand_distr::uniform::SampleUniform; +use nd::{ArrayBase, DataOwned, Dimension, ShapeBuilder}; + +impl Initialize for QkvBase where + D: RemoveAxis, + S: DataOwned, + StandardNormal: Distribution, +{ + type Data = S; + + fn rand(shape: Sh, distr: Dstr) -> Self + where + Sh: ShapeBuilder, + Dstr: Clone + Distribution, + { + let dim = shape.into_shape().raw_dim().clone(); + Self { + q: ArrayBase::rand(dim.clone(), distr.clone()), + k: ArrayBase::rand(dim.clone(), distr.clone()), + v: ArrayBase::rand(dim, distr) + } + } + + fn rand_with(shape: Sh, distr: Ds, rng: &mut R) -> Self + where + R: Rng + ?Sized, + Ds: Clone + Distribution, + Sh: ShapeBuilder, + { + let dim = shape.into_shape().raw_dim().clone(); + Self { + q: ArrayBase::rand_with(dim.clone(), distr.clone(), &mut rng), + k: ArrayBase::rand_with(dim.clone(), distr.clone(), &mut rng), + v: ArrayBase::rand_with(dim, distr, &mut rng) + } + } + + fn init_rand(self, distr: Ds) -> Self + where + Ds: Clone + Distribution, + Self: Sized, + { + Self::rand(self.dim(), distr) + } + + fn init_rand_with(self, distr: Ds, rng: &mut R) -> Self + where + R: Rng + ?Sized, + Ds: Clone + Distribution, + { + Self::rand_with(self.dim(), distr, rng) + } +} + + diff --git a/models/transformers/src/impls/impl_linalg.rs b/models/transformers/src/impls/impl_linalg.rs new file mode 100644 index 00000000..c2ab8812 --- /dev/null +++ b/models/transformers/src/impls/impl_linalg.rs @@ -0,0 +1,50 @@ +/* + Appellation: impl_linalg + Contrib: FL03 +*/ +use crate::params::{Qkv, QkvBase}; +use concision::Matmul; +use nd::linalg::Dot; +use nd::*; + +impl Matmul> for QkvBase +where + A: LinalgScalar, + D: Dimension, + E: Dimension, + F: Dimension, + S: Data, + T: Data, + ArrayBase: Dot, Output = Array>, +{ + type Output = Qkv; + + fn matmul(&self, rhs: &QkvBase) -> Self::Output { + QkvBase { + q: self.q().dot(rhs.q()), + k: self.k().dot(rhs.k()), + v: self.v().dot(rhs.v()), + } + } +} + +impl Matmul> for QkvBase +where + A: LinalgScalar, + D: Dimension, + E: Dimension, + F: Dimension, + S: Data, + T: Data, + ArrayBase: Dot, Output = Array>, +{ + type Output = Qkv; + + fn matmul(&self, rhs: &ArrayBase) -> Self::Output { + QkvBase { + q: self.q().dot(rhs), + k: self.k().dot(rhs), + v: self.v().dot(rhs), + } + } +} diff --git a/models/transformers/src/impls/impl_params.rs b/models/transformers/src/impls/impl_params.rs new file mode 100644 index 00000000..2ea7dec4 --- /dev/null +++ b/models/transformers/src/impls/impl_params.rs @@ -0,0 +1,86 @@ +/* + Appellation: impl_params + Contrib: FL03 +*/ +use crate::params::QkvBase; +use nd::prelude::*; +use nd::{Data, DataOwned, RawDataClone}; + +pub(crate) type ThreeTuple = (A, B, C); + +impl Clone for QkvBase +where + D: Dimension, + S: RawDataClone, +{ + fn clone(&self) -> Self { + Self { + q: self.q.clone(), + k: self.k.clone(), + v: self.v.clone(), + } + } +} + +impl Copy for QkvBase +where + D: Copy + Dimension, + S: Copy + RawDataClone, +{ +} + +impl Default for QkvBase +where + A: Default, + D: Dimension, + S: DataOwned, +{ + fn default() -> Self { + Self { + q: Default::default(), + k: Default::default(), + v: Default::default(), + } + } +} + +impl PartialEq for QkvBase +where + A: PartialEq, + D: Dimension, + S: Data, +{ + fn eq(&self, other: &Self) -> bool { + self.q() == other.q() && self.k() == other.k() && self.v() == other.v() + } +} + +impl PartialEq> for QkvBase +where + A: PartialEq, + B: PartialEq, + D: Dimension, + S: Data, + S2: Data, + D2: Dimension, + ArrayBase: PartialEq>, +{ + fn eq(&self, other: &ArrayBase) -> bool { + self.q() == other && self.k() == other && self.v() == other + } +} + +impl PartialEq>> for QkvBase +where + A: PartialEq, + B: PartialEq, + D: Dimension, + S: Data, + S2: Data, + D2: Dimension, + ArrayBase: PartialEq>, +{ + fn eq(&self, (q, k, v): &ThreeTuple>) -> bool { + self.q() == q && self.k() == k && self.v() == v + } +} diff --git a/models/transformers/src/lib.rs b/models/transformers/src/lib.rs new file mode 100644 index 00000000..89cc41f1 --- /dev/null +++ b/models/transformers/src/lib.rs @@ -0,0 +1,45 @@ +/* + Appellation: concision-transformers + Contrib: FL03 +*/ +//! # Transformers +//! +//! ### Resources +//! +//! - [Attention is All You Need](https://arxiv.org/abs/1706.03762) + +#![cfg_attr(not(feature = "std"), no_std)] + +#[cfg(feature = "alloc")] +extern crate alloc; + +extern crate concision_core as concision; +extern crate concision_linear as linear; +extern crate ndarray as nd; + +pub use self::attention::{scaled_dot_product_attention, AttentionHead}; +pub use self::params::*; +pub use self::primitives::*; +pub use self::transformer::Transformer; + +#[macro_use] +pub(crate) mod macros; +pub(crate) mod primitives; +pub(crate) mod transformer; + +pub mod attention; +pub mod codec; +pub mod model; +pub mod ops; +pub mod params; + +pub(crate) mod impls { + pub mod impl_head; + pub mod impl_linalg; + pub mod impl_params; +} + +pub mod prelude { + pub use super::attention::prelude::*; + pub use super::Transformer; +} diff --git a/models/transformers/src/macros.rs b/models/transformers/src/macros.rs new file mode 100644 index 00000000..e25dafae --- /dev/null +++ b/models/transformers/src/macros.rs @@ -0,0 +1,85 @@ +/* + Appellation: macros + Contrib: FL03 +*/ + +#[macro_use] +mod params; + +macro_rules! ndbuilder { + ($method:ident$(::$call:ident)?() $($where:tt)*) => { + ndbuilder!(@impl $method$(::$call)?() $($where)*); + }; + (@impl $method:ident() $($where:tt)*) => { + ndbuilder!(@impl $method::$method() $($where)*); + }; + (@impl $method:ident::$call:ident() $($where:tt)*) => { + pub fn $method>(shape: Sh) -> Self $($where)* { + Self::builder(shape, ndarray::ArrayBase::$call) + } + }; +} + +#[allow(unused_macros)] +macro_rules! cbuilder { + (@impl derive: [$($D:ident),* $(,)?], $name:ident {$($vis:vis $field:ident: $type:ty),*}) => { + #[derive(Clone, Debug, PartialEq, $($D),*)] + #[cfg_attr(feature = "serde", derive(serde::Deserialize, serde::Serialize))] + pub struct $name { + $($vis $field: $type),* + } + impl $name { + paste::paste! { + pub fn new() -> [<$name Builder>] { + [<$name Builder>]::new() + } + } + + $( + pub fn $field(mut self, $field: $type) -> Self { + self.$field = $field; + self + } + )* + } + }; + (@builder derive: [$($D:ident),* $(,)?], $name:ident {$($field:ident: $type:ty),*}) => { + pub struct $name { + $(pub(crate) $field: $type),* + } + + impl $name { + pub fn new() -> Self { + Self { + $($field: None),* + } + } + + $( + pub fn $field(mut self, $field: $type) -> Self { + self.$field = Some($field); + self + } + )* + + pub fn build(&self) -> Config { + Config { + $($field: self.$field.unwrap_or_else(|| crate::$field),)* + } + } + } + + impl Default for $name { + fn default() -> Self { + Self::new() + } + } + }; +} + +/// This macro helps create a stack of identical sublayers. +/// +#[allow(unused_macros)] +macro_rules! sublayer { + (@impl heads: $heads:expr) => {}; +} diff --git a/models/transformers/src/macros/params.rs b/models/transformers/src/macros/params.rs new file mode 100644 index 00000000..f7e12e32 --- /dev/null +++ b/models/transformers/src/macros/params.rs @@ -0,0 +1,50 @@ +/* + Appellation: params + Contrib: FL03 +*/ + +macro_rules! qkv_view { + ($method:ident$(.$call:ident)?::$($rest:tt)*) => { + qkv_view!(@impl $method$(.$call)?::$($rest)*); + }; + (@impl $method:ident::$($rest:tt)*) => { + qkv_view!(@impl $method.$method::$($rest)*); + }; + (@impl $method:ident.$call:ident::<$view:ident>(self) $($rest:tt)*) => { + pub fn $method(self) -> $crate::params::QkvBase<$view, D> $($rest)* { + qkv_view!(@apply $call(self)) + } + }; + (@impl $method:ident.$call:ident::<$view:ident>(mut self) $($rest:tt)*) => { + pub fn $method(mut self) -> $crate::params::QkvBase<$view, D> $($rest)* { + qkv_view!(@apply $call(self)) + } + }; + (@impl $method:ident.$call:ident::<$view:ident>(&self) $($rest:tt)*) => { + pub fn $method(&self) -> $crate::params::QkvBase<$view, D> $($rest)* { + qkv_view!(@apply $call(self)) + } + }; + (@impl $method:ident.$call:ident::<$view:ident>(&mut self) $($rest:tt)*) => { + pub fn $method(&mut self) -> $crate::params::QkvBase<$view, D> $($rest)* { + qkv_view!(@apply $call(self)) + } + }; + (@impl $method:ident.$call:ident::<'a, $view:ident>(&self) $($rest:tt)*) => { + pub fn $method(&self) -> $crate::params::QkvBase<$view<&'_ A>, D> $($rest)* { + qkv_view!(@apply $call(self)) + } + }; + (@impl $method:ident.$call:ident::<'a, $view:ident>(&mut self) $($rest:tt)*) => { + pub fn $method(&mut self) -> $crate::params::QkvBase<$view<&'_ mut A>, D> $($rest)* { + qkv_view!(@apply $call(self)) + } + }; + (@apply $call:ident($self:expr)) => { + $crate::params::QkvBase { + q: $self.q.$call(), + k: $self.k.$call(), + v: $self.v.$call(), + } + }; +} diff --git a/models/transformers/src/model/mod.rs b/models/transformers/src/model/mod.rs new file mode 100644 index 00000000..ac227da3 --- /dev/null +++ b/models/transformers/src/model/mod.rs @@ -0,0 +1,6 @@ +/* + Appellation: model + Contrib: FL03 +*/ + +pub mod sublayer; diff --git a/models/transformers/src/model/sublayer.rs b/models/transformers/src/model/sublayer.rs new file mode 100644 index 00000000..a1a5fbe7 --- /dev/null +++ b/models/transformers/src/model/sublayer.rs @@ -0,0 +1,74 @@ +/* + Appellation: sublayer + Contrib: FL03 +*/ +#![cfg(feature = "rand")] +use concision::nn::DropoutLayer; +use concision::Forward; +use linear::{Biased, LayerNorm, ParamMode, Unbiased}; +use nd::prelude::*; +use nd::{DataOwned, RemoveAxis, ScalarOperand}; +use num::traits::{Float, FromPrimitive}; + +/// A residual connection followed by a [layer norm](LayerNorm) +/// [Transformer](crate::Transformer) +pub struct Sublayer +where + D: Dimension, +{ + pub(crate) dropout: DropoutLayer, + pub(crate) norm: LayerNorm, +} + +impl Sublayer +where + D: RemoveAxis, +{ + pub fn new(shape: Sh, dropout: f64) -> Self + where + A: Default, + K: ParamMode, + Sh: ShapeBuilder, + { + Self { + dropout: DropoutLayer::new(dropout), + norm: LayerNorm::new(shape), + } + } + + pub fn dropout(&self) -> &DropoutLayer { + &self.dropout + } + + pub fn norm(&self) -> &LayerNorm { + &self.norm + } +} + +impl Forward> for Sublayer +where + A: Float + FromPrimitive + ScalarOperand, + D: RemoveAxis, + S: DataOwned, +{ + type Output = Array; + + fn forward(&self, input: &ArrayBase) -> Self::Output { + let normal = self.norm().forward(input); + input + self.dropout().forward(&normal) + } +} + +impl Forward> for Sublayer +where + A: Float + FromPrimitive + ScalarOperand, + D: RemoveAxis, + S: DataOwned, +{ + type Output = Array; + + fn forward(&self, input: &ArrayBase) -> Self::Output { + let normal = self.norm().forward(input); + input + self.dropout().forward(&normal) + } +} diff --git a/models/transformers/src/ops/merge.rs b/models/transformers/src/ops/merge.rs new file mode 100644 index 00000000..c7e66e3e --- /dev/null +++ b/models/transformers/src/ops/merge.rs @@ -0,0 +1,60 @@ +/* + Appellation: merge + Contrib: FL03 +*/ +use super::_merge_dim; +use concision::NdResult; +use nd::{Array, ArrayBase, Data, Dimension, RemoveAxis}; + +pub trait DimMerge { + type Output; + + fn merge(&self, tgt: usize) -> Self::Output; +} + +// #67: Optimize the Merge trait +pub trait Merge { + type Output; + + fn merge(&self) -> NdResult { + self.merge_along(0) + } + + fn merge_along(&self, axis: usize) -> NdResult; +} + +/* + ************* Implementations ************* +*/ +impl DimMerge for D +where + D: RemoveAxis, + D::Smaller: Dimension, + D::Larger: Dimension, +{ + type Output = D::Smaller; + + fn merge(&self, tgt: usize) -> Self::Output { + _merge_dim(self, tgt) + } +} + +impl Merge for ArrayBase +where + A: Clone, + D: RemoveAxis, + E: Dimension, + S: Data, + ArrayBase: Clone, +{ + type Output = Array; + + fn merge(&self) -> NdResult { + let swap = if self.ndim() >= 3 { self.ndim() - 3 } else { 0 }; + self.merge_along(swap) + } + + fn merge_along(&self, swap: usize) -> NdResult { + super::_merge(self, swap, swap + 1, super::ORDER) + } +} diff --git a/models/transformers/src/ops/mod.rs b/models/transformers/src/ops/mod.rs new file mode 100644 index 00000000..6612af22 --- /dev/null +++ b/models/transformers/src/ops/mod.rs @@ -0,0 +1,100 @@ +/* + Appellation: ops + Contrib: FL03 +*/ +pub use self::prelude::*; + +mod merge; +mod split; + +pub(crate) mod prelude { + pub use super::merge::*; + pub use super::split::*; + pub(crate) use super::utils::*; +} + +pub(crate) const ORDER: nd::Order = nd::Order::RowMajor; + +pub(crate) mod utils { + use concision::NdResult; + use nd::prelude::*; + use nd::{Data, Order, RemoveAxis}; + + pub(crate) fn _merge( + arr: &ArrayBase, + src: usize, + tgt: usize, + order: Order, + ) -> NdResult> + where + A: Clone, + D: RemoveAxis, + S: Data, + D::Smaller: Dimension, + ArrayBase: Clone, + { + let shape = _merge_dim(&arr.raw_dim(), src); + let mut head = arr.clone(); + head.swap_axes(src, tgt); + head.to_shape((shape, order)).map(|x| x.to_owned()) + } + + pub(crate) fn _split( + arr: &ArrayBase, + h: usize, + order: Order, + ) -> NdResult> + where + A: Clone, + D: Dimension, + E: RemoveAxis, + S: Data, + ArrayBase: Clone, + { + let src = if arr.ndim() >= 2 { arr.ndim() - 2 } else { 0 }; + let tgt = src + 1; + let shape: E = _split_dim(&arr.raw_dim(), h); + let mut head = arr.to_shape((shape, order))?.to_owned(); + head.swap_axes(src, tgt); + Ok(head) + } + /// Creates the new dimension after merging two axes. + pub(crate) fn _merge_dim(dim: &D, axis: usize) -> D::Smaller + where + D: RemoveAxis, + D::Smaller: Dimension, + { + // create a new dimension with one less axis; initialized with zeros + let mut dn = ::Smaller::zeros(dim.ndim() - 1); + // create a mutable vector from the slice + let mut shape = dim.slice().to_vec(); + // multiply the last axis by the target + shape[dn.ndim()] *= shape[axis]; + // remove the last dimension + shape.remove(axis); + + dn.slice_mut().copy_from_slice(&shape); + dn + } + + pub(crate) fn _split_dim(dim: &D::Smaller, h: usize) -> D + where + D: RemoveAxis, + D::Smaller: Dimension, + { + let rank = dim.ndim() + 1; + // create a new dimension with one less axis; initialized with zeros + let mut new_dim = D::zeros(rank); + // create a mutable vector from the slice + let mut shape = dim.slice().to_vec(); + // get and remove the last axis + let bx = shape.pop().unwrap() / h; + // extend the shape with the new axes + shape.push(h); + shape.push(bx); + // shape.swap(rank - 2, rank - 3); + // copy the values into the new dimension + new_dim.slice_mut().copy_from_slice(&shape); + new_dim + } +} diff --git a/models/transformers/src/ops/split.rs b/models/transformers/src/ops/split.rs new file mode 100644 index 00000000..d98d861d --- /dev/null +++ b/models/transformers/src/ops/split.rs @@ -0,0 +1,77 @@ +/* + Appellation: split + Contrib: FL03 +*/ +use ndarray::{Array, ArrayBase, Data, Dimension, RemoveAxis, ShapeError}; + +/// Split a dimension into two parts +pub trait DimSplit { + type Output; + + fn split(&self, h: usize) -> Self::Output; +} + +pub trait SplitHead { + type Output; + + fn split(&self, heads: usize) -> Result; +} + +/* + ************* Implementations ************* +*/ + +impl DimSplit for D +where + D: Dimension, + E: RemoveAxis, +{ + type Output = E; + + fn split(&self, h: usize) -> Self::Output { + super::utils::_split_dim(self, h) + } +} + +impl SplitHead for ArrayBase +where + A: Clone, + D: Dimension, + E: RemoveAxis, + S: Data, + ArrayBase: Clone, +{ + type Output = Array; + + fn split(&self, h: usize) -> Result { + super::_split(self, h, super::ORDER) + } +} + +// impl Split for Array2 { +// type Output = Array3; + +// fn split(&self, heads: usize) -> Result { +// let (seq, model) = self.dim(); +// let query = model / heads; +// // reshape the qkv matrix into a 3d array +// let mut res = self.clone().into_shape((seq, heads, query))?; +// // swap the sequence and head axes +// res.swap_axes(0, 1); +// Ok(res) +// } +// } + +// impl Split for Array3 { +// type Output = Array4; + +// fn split(&self, heads: usize) -> Result { +// let (batch, seq, model) = self.dim(); +// let query = model / heads; +// // reshape the qkv matrix into a 3d array +// let mut res = self.clone().into_shape((batch, seq, heads, query))?; +// // swap the sequence and head axes +// res.swap_axes(1, 2); +// Ok(res) +// } +// } diff --git a/models/transformers/src/params/item.rs b/models/transformers/src/params/item.rs new file mode 100644 index 00000000..67528ab1 --- /dev/null +++ b/models/transformers/src/params/item.rs @@ -0,0 +1,79 @@ +/* + Appellation: kinds + Contrib: FL03 +*/ +use nd::{ArrayBase, Dimension, Ix2, OwnedRepr, RawData}; +use strum::{AsRefStr, EnumCount, EnumDiscriminants, EnumIs, VariantNames}; + +#[derive(AsRefStr, EnumCount, EnumDiscriminants, EnumIs, VariantNames)] +#[strum_discriminants( + derive( + AsRefStr, + EnumCount, + EnumIs, + Hash, + Ord, + PartialOrd, + VariantNames, + strum::Display, + strum::EnumString, + ), + name(QKV), + strum(serialize_all = "lowercase") +)] +#[cfg_attr( + feature = "serde", + strum_discriminants( + derive(serde::Deserialize, serde::Serialize), + serde(rename_all = "lowercase", untagged) + ) +)] +#[strum(serialize_all = "lowercase")] +pub enum Entry, D = Ix2> +where + D: Dimension, + S: RawData, +{ + Q(ArrayBase), + K(ArrayBase), + V(ArrayBase), +} + +impl Entry +where + D: Dimension, + S: RawData, +{ + pub fn from_q(q: ArrayBase) -> Self { + Self::Q(q) + } + + pub fn from_k(k: ArrayBase) -> Self { + Self::K(k) + } + + pub fn from_v(v: ArrayBase) -> Self { + Self::V(v) + } + + pub fn q(&self) -> Option<&ArrayBase> { + match self { + Self::Q(q) => Some(q), + _ => None, + } + } + + pub fn k(&self) -> Option<&ArrayBase> { + match self { + Self::K(k) => Some(k), + _ => None, + } + } + + pub fn v(&self) -> Option<&ArrayBase> { + match self { + Self::V(v) => Some(v), + _ => None, + } + } +} diff --git a/models/transformers/src/params/mod.rs b/models/transformers/src/params/mod.rs new file mode 100644 index 00000000..ba79e10f --- /dev/null +++ b/models/transformers/src/params/mod.rs @@ -0,0 +1,37 @@ +/* + Appellation: params + Contrib: FL03 +*/ +pub use self::{item::*, store::QkvBase}; + +mod store; + +pub mod item; + +macro_rules! params_ty { + ($target:ident {$($name:ident: $(&$lt:lifetime)? $repr:ident),* $(,)?}) => { + $(params_ty!(@impl $target: $name<$(&$lt)? $repr>);)* + }; + (@impl $target:ident: $name:ident<$repr:ident>) => { + pub type $name = $target, D>; + }; + (@impl $target:ident: $name:ident<&'a $repr:ident>) => { + pub type $name<'a, A = f64, D = ndarray::Ix2> = $target, D>; + }; +} + +params_ty!( + QkvBase { + Qkv: OwnedRepr, + ArcQkv: OwnedArcRepr, + ViewQkv: &'a ViewRepr, + + } +); + +#[allow(unused_imports)] +pub(crate) mod prelude { + pub use super::item::{Entry, QKV}; + pub use super::store::QkvBase; + pub use super::{ArcQkv, Qkv, ViewQkv}; +} diff --git a/models/transformers/src/params/store.rs b/models/transformers/src/params/store.rs new file mode 100644 index 00000000..f59ee6eb --- /dev/null +++ b/models/transformers/src/params/store.rs @@ -0,0 +1,130 @@ +/* + Appellation: params + Contrib: FL03 +*/ +use crate::attention::{Score, _attention}; +use concision::nn::DropoutLayer; +use concision::{dimensional, getters}; +use nd::linalg::Dot; +use nd::*; +use num::complex::ComplexFloat; +use num::traits::{One, Zero}; + +/// [QkvBase] is a container for the query, key, and value arrays used in the +/// attention mechanism of the transformer model. +pub struct QkvBase, D = Ix2> +where + D: Dimension, + S: RawData, +{ + pub(crate) q: ArrayBase, + pub(crate) k: ArrayBase, + pub(crate) v: ArrayBase, +} + +impl QkvBase +where + D: Dimension, + S: RawData, +{ + pub fn builder(shape: Sh, builder: F) -> Self + where + F: Fn(D) -> ArrayBase, + Sh: ShapeBuilder, + { + let dim = shape.into_shape().raw_dim().clone(); + Self { + q: builder(dim.clone()), + k: builder(dim.clone()), + v: builder(dim), + } + } + + pub fn from_elem(shape: Sh, value: A) -> Self + where + Sh: ShapeBuilder, + A: Clone, + S: DataOwned, + { + let dim = shape.into_shape().raw_dim().clone(); + Self { + q: ArrayBase::from_elem(dim.clone(), value.clone()), + k: ArrayBase::from_elem(dim.clone(), value.clone()), + v: ArrayBase::from_elem(dim, value), + } + } + + pub fn as_qkv(&self) -> (ArrayView, ArrayView, ArrayView) + where + S: Data, + { + (self.q.view(), self.k.view(), self.v.view()) + } + + /// Consumes the store and returns a three-tuple consisting of the query, key, and value arrays respectively. + pub fn into_qkv(self) -> (ArrayBase, ArrayBase, ArrayBase) { + (self.q, self.k, self.v) + } + + pub fn qkv(&self) -> (&ArrayBase, &ArrayBase, &ArrayBase) { + (&self.q, &self.k, &self.v) + } + + ndbuilder!(new::default() where A: Default, S: DataOwned); + ndbuilder!(ones() where A: Clone + One, S: DataOwned); + ndbuilder!(zeros() where A: Clone + Zero, S: DataOwned); + + getters!(q, k, v => ArrayBase); + + dimensional!(q()); + + qkv_view!(into_owned::(self) where A: Clone, S: Data); + qkv_view!(to_owned::(&self) where A: Clone, S: Data); + + qkv_view!(into_shared::(self) where A: Clone, S: DataOwned); + qkv_view!(to_shared::(&self) where A: Clone, S: DataShared); + + qkv_view!(view::<'a, ViewRepr>(&self) where S: Data); + qkv_view!(view_mut::<'a, ViewRepr>(&mut self) where S: DataMut); +} + +#[cfg(not(feature = "rand"))] +impl QkvBase +where + D: Dimension, + S: RawData, + A: Clone, +{ + /// Computes the [Score] using scaled dot-product attention. + pub fn attention(&self, dropout: Option, mask: Option<&Array>) -> Score + where + A: ComplexFloat + ScalarOperand, + S: Data, + ArrayBase: for<'a> Dot, Output = Array>, + Array: Dot, Output = Array>, + { + let (q, k, v) = self.qkv(); + _attention(q, k, v, mask, None) + } +} + +#[cfg(feature = "rand")] +impl QkvBase +where + D: Dimension, + S: RawData, + A: Clone, +{ + /// Computes the [Score] using scaled dot-product attention. + pub fn attention(&self, dropout: Option, mask: Option<&Array>) -> Score + where + A: ComplexFloat + ScalarOperand, + S: Data, + ArrayBase: for<'a> Dot, Output = Array>, + Array: Dot, Output = Array>, + { + let dropout = dropout.map(DropoutLayer::new); + let (q, k, v) = self.qkv(); + _attention(q, k, v, mask, dropout.as_ref()) + } +} diff --git a/models/transformers/src/primitives.rs b/models/transformers/src/primitives.rs new file mode 100644 index 00000000..3b30e7aa --- /dev/null +++ b/models/transformers/src/primitives.rs @@ -0,0 +1,22 @@ +/* + Appellation: primitives + Contrib: FL03 +*/ +pub use self::consts::*; + +pub mod consts { + /// The default dimension of the model; i.e. the number of inputs + pub const D_MODEL: usize = 512; + /// The default size of the network; i.e. the number of neurons in the network + pub const D_NETWORK: usize = 2048; + /// The default dimension of the key and query vectors + pub const DK: usize = D_MODEL / HEADS; + /// The default number of attention heads + pub const HEADS: usize = 8; + /// The default number of layers used for the encoder / decoder. + pub const N: usize = 6; +} + +pub fn outputs_from_ratio(model: usize, network: usize) -> usize { + network / model +} diff --git a/models/transformers/src/transformer.rs b/models/transformers/src/transformer.rs new file mode 100644 index 00000000..f4f032ee --- /dev/null +++ b/models/transformers/src/transformer.rs @@ -0,0 +1,6 @@ +/* + Appellation: transformer + Contrib: FL03 +*/ + +pub struct Transformer; diff --git a/models/transformers/tests/attention.rs b/models/transformers/tests/attention.rs new file mode 100644 index 00000000..6bc023af --- /dev/null +++ b/models/transformers/tests/attention.rs @@ -0,0 +1,22 @@ +/* + Appellation: attention + Contrib: FL03 +*/ +extern crate concision_core as concision; +extern crate concision_transformers as transformers; + +use approx::AbsDiffEq; +use transformers::AttentionHead; + +use ndarray::prelude::*; + +#[test] +fn attention_head() { + let shape = (3, 3); + + let head = AttentionHead::::ones(shape); + assert_eq!(head.q(), &Array::ones(shape)); + let exp = Array2::from_elem(shape, 1f64 / 3f64); + let score = head.attention(); + assert!(score.attention().abs_diff_eq(&exp, 1e-6)); +} diff --git a/models/transformers/tests/default.rs b/models/transformers/tests/default.rs new file mode 100644 index 00000000..233a07af --- /dev/null +++ b/models/transformers/tests/default.rs @@ -0,0 +1,17 @@ +/* + Appellation: default + Contrib: FL03 +*/ + +fn add(a: A, b: B) -> C +where + A: core::ops::Add, +{ + a + b +} + +#[test] +fn compiles() { + assert_eq!(add(10, 10), 20); + assert_ne!(add(1, 1), 3); +} diff --git a/models/transformers/tests/ops.rs b/models/transformers/tests/ops.rs new file mode 100644 index 00000000..687b50db --- /dev/null +++ b/models/transformers/tests/ops.rs @@ -0,0 +1,129 @@ +/* + Appellation: ops + Contrib: FL03 +*/ +extern crate concision_core as concision; +extern crate concision_transformers as transformers; +extern crate ndarray as nd; + +use concision::linarr; +use nd::prelude::*; +use transformers::ops::*; + +pub const HEADS: usize = 2; +pub const ORDER: nd::Order = nd::Order::RowMajor; + +#[test] +fn test_merge() { + let shape = (3, 4, 5); + let dout = (4, 15); + let arr = linarr::(shape.clone()).unwrap(); + let a = arr.clone().merge().unwrap(); + + assert_eq!(a.dim(), dout); + assert_eq!(a, utils::merge3(&arr).unwrap()); +} + +#[test] +fn test_merge_batch() { + let shape = (2, 3, 4, 5); + let dout = (2, 4, 15); + let arr = linarr::(shape).unwrap(); + let a = arr.merge().unwrap(); + + assert_eq!(a.dim(), dout); + assert_eq!(a, utils::merge4(&arr).unwrap()); +} + +#[test] +fn test_split() { + let heads = 2; + let shape = (4, 6); + let arr = linarr::(shape).unwrap(); + let a = arr.split(heads).unwrap(); + + assert_eq!(a.dim(), (heads, 4, 3)); + assert_eq!(a, utils::split_heads(&arr, heads).unwrap()); +} + +#[test] +fn test_split_batch() { + let heads = 2; + let shape = (3, 4, 6); + let arr = linarr::(shape).unwrap(); + let a = arr.split(heads).unwrap(); + + assert_eq!(a.dim(), (3, heads, 4, 3)); + assert_eq!(a, utils::split_batch(&arr, heads).unwrap()); +} + +#[test] +fn reshape_ops() { + let shape = (2, 4, 6); + let data = linarr::(shape).unwrap(); + + let a = data.split(HEADS).unwrap(); + assert_eq!(a.dim(), (2, HEADS, 4, 3)); + let b = a.merge().unwrap(); + assert_eq!(b.dim(), shape); + // verify that doing the ops consecutively is the identity + assert_eq!(b, data); +} + +#[allow(dead_code)] +pub(crate) mod utils { + use concision::NdResult; + use ndarray::*; + + pub fn merge3(heads: &Array3) -> NdResult> + where + T: Clone, + { + let (n, seq, query) = heads.dim(); + let shape = (seq, n * query); + let mut tmp = heads.clone(); + // swap the head and sequence axes + tmp.swap_axes(0, 1); + // reshape the qkv matrix into a 2d array + tmp.to_shape((shape, super::ORDER)).map(|x| x.to_owned()) + } + + pub fn merge4(heads: &Array4) -> NdResult> + where + T: Clone, + { + let (batch, n, seq, query) = heads.dim(); + let shape = (batch, seq, n * query); + let mut tmp = heads.clone(); + // swap the head and sequence axes + tmp.swap_axes(1, 2); + // reshape the qkv matrix into a 2d array + tmp.to_shape((shape, super::ORDER)).map(|x| x.to_owned()) + } + + pub fn split_heads(param: &Array2, h: usize) -> NdResult> + where + T: Clone, + { + let dim = param.shape().last().unwrap() / h; + // reshape the qkv matrix into a 3d array + let mut res = param.clone().into_shape((param.shape()[0], h, dim))?; + // swap the sequence and head axes + res.swap_axes(0, 1); + Ok(res) + } + + pub fn split_batch(param: &Array3, h: usize) -> NdResult> + where + T: Clone, + { + let dim = param.shape().last().unwrap() / h; + // reshape the qkv matrix into a 3d array + let mut res = param + .clone() + .into_shape((param.shape()[0], param.shape()[1], h, dim))?; + // swap the sequence and head axes + res.swap_axes(1, 2); + Ok(res) + } +} diff --git a/models/transformers/tests/params.rs b/models/transformers/tests/params.rs new file mode 100644 index 00000000..18656be8 --- /dev/null +++ b/models/transformers/tests/params.rs @@ -0,0 +1,35 @@ +/* + Appellation: params + Contrib: FL03 +*/ +extern crate concision_core as concision; +extern crate concision_transformers as transformers; + +use concision::{linarr, Matmul}; +use transformers::Qkv; + +use ndarray::prelude::*; + +#[test] +fn test_qkv() { + let shape = (2048, 10); + let params = Qkv::::new(shape); + assert_eq!(params.q(), &Array::default(shape)); +} + +#[test] +fn test_qkv_matmul() { + let shape = (2048, 10); + // generate some sample data + let data = linarr(shape).unwrap(); + // initialize the parameters + let params = Qkv::::ones(shape); + // calculate the expected result + let exp = Array2::::ones(shape).dot(&data.t()); + // calculate the result + let res = params.matmul(&data.t()); + // compare the results + assert_eq!(res.q(), &exp); + assert_eq!(res.k(), &exp); + assert_eq!(res.v(), &exp); +}