diff --git a/docs/classifiers/logistic-regression.md b/docs/classifiers/logistic-regression.md index cf0d75a7d..3bc6587fc 100644 --- a/docs/classifiers/logistic-regression.md +++ b/docs/classifiers/logistic-regression.md @@ -15,15 +15,15 @@ A linear classifier that uses the logistic (*sigmoid*) function to estimate the | 3 | l2Penalty | 1e-4 | float | The amount of L2 regularization applied to the weights of the output layer. | | 4 | epochs | 1000 | int | The maximum number of training epochs. i.e. the number of times to iterate over the entire training set before terminating. | | 5 | minChange | 1e-4 | float | The minimum change in the training loss necessary to continue training. | -| 6 | costFn | CrossEntropy | ClassificationLoss | The function that computes the loss associated with an erroneous activation during training. | +| 6 | costFn | BinaryCrossEntropy | ClassificationLoss | The function that computes the loss associated with an erroneous activation during training. | ## Example ```php use Rubix\ML\Classifiers\LogisticRegression; use Rubix\ML\NeuralNet\Optimizers\Adam; -use Rubix\ML\NeuralNet\CostFunctions\CrossEntropy; +use Rubix\ML\NeuralNet\CostFunctions\BinaryCrossEntropy; -$estimator = new LogisticRegression(64, new Adam(0.001), 1e-4, 100, 1e-4, new CrossEntropy()); +$estimator = new LogisticRegression(64, new Adam(0.001), 1e-4, 100, 1e-4, new BinaryCrossEntropy()); ``` ## Additional Methods diff --git a/docs/classifiers/multilayer-perceptron.md b/docs/classifiers/multilayer-perceptron.md index cd6062821..68ef41a3d 100644 --- a/docs/classifiers/multilayer-perceptron.md +++ b/docs/classifiers/multilayer-perceptron.md @@ -21,7 +21,7 @@ A multiclass feed-forward neural network classifier with user-defined hidden lay | 6 | evalInterval | 3 | int | The number of epochs to train before evaluating the model using the holdout set. | | 7 | window | 5 | int | The number of epochs without improvement in the validation score to wait before considering an early stop. | | 8 | holdOut | 0.1 | float | The proportion of training samples to use for internal validation. Set to 0 to disable. | -| 9 | costFn | CrossEntropy | ClassificationLoss | The function that computes the loss associated with an erroneous activation during training. | +| 9 | costFn | MulticlassCrossEntropy | ClassificationLoss | The function that computes the loss associated with an erroneous activation during training. | | 10 | metric | FBeta | Metric | The validation metric used to score the generalization performance of the model during training. | ## Example @@ -33,7 +33,7 @@ use Rubix\ML\NeuralNet\Layers\Activation; use Rubix\ML\NeuralNet\Layers\PReLU; use Rubix\ML\NeuralNet\ActivationFunctions\LeakyReLU; use Rubix\ML\NeuralNet\Optimizers\Adam; -use Rubix\ML\NeuralNet\CostFunctions\CrossEntropy; +use Rubix\ML\NeuralNet\CostFunctions\MulticlassCrossEntropy; use Rubix\ML\CrossValidation\Metrics\MCC; $estimator = new MultilayerPerceptron([ @@ -45,7 +45,7 @@ $estimator = new MultilayerPerceptron([ new Dropout(0.3), new Dense(50), new PReLU(), -], 128, new Adam(0.001), 1000, 1e-3, 10, 3, 0.1, new CrossEntropy(), new MCC()); +], 128, new Adam(0.001), 1000, 1e-3, 10, 3, 0.1, new MulticlassCrossEntropy(), new MCC()); ``` ## Additional Methods diff --git a/docs/classifiers/softmax-classifier.md b/docs/classifiers/softmax-classifier.md index 01614dcd5..f92dc7827 100644 --- a/docs/classifiers/softmax-classifier.md +++ b/docs/classifiers/softmax-classifier.md @@ -15,15 +15,15 @@ A multiclass generalization of [Logistic Regression](logistic-regression.md) usi | 3 | alpha | 1e-4 | float | The amount of L2 regularization applied to the weights of the output layer. | | 4 | epochs | 1000 | int | The maximum number of training epochs. i.e. the number of times to iterate over the entire training set before terminating. | | 5 | minChange | 1e-4 | float | The minimum change in the training loss necessary to continue training. | -| 6 | costFn | CrossEntropy | ClassificationLoss | The function that computes the loss associated with an erroneous activation during training. | +| 6 | costFn | BinaryCrossEntropy | ClassificationLoss | The function that computes the loss associated with an erroneous activation during training. | ## Example ```php use Rubix\ML\Classifiers\SoftmaxClassifier; use Rubix\ML\NeuralNet\Optimizers\Momentum; -use Rubix\ML\NeuralNet\CostFunctions\CrossEntropy; +use Rubix\ML\NeuralNet\CostFunctions\MulticlassCrossEntropy; -$estimator = new SoftmaxClassifier(256, new Momentum(0.001), 1e-4, 300, 1e-4, new CrossEntropy()); +$estimator = new SoftmaxClassifier(256, new Momentum(0.001), 1e-4, 300, 1e-4, new BinaryCrossEntropy()); ``` ## Additional Methods diff --git a/docs/neural-network/cost-functions/binary-cross-entropy.md b/docs/neural-network/cost-functions/binary-cross-entropy.md new file mode 100644 index 000000000..998926826 --- /dev/null +++ b/docs/neural-network/cost-functions/binary-cross-entropy.md @@ -0,0 +1,18 @@ +[source] + +# Binary Cross Entropy +Binary Cross Entropy (or *log loss*) measures the performance of a binary classification model whose output is a probability value between 0 and 1. Cross-entropy loss increases as the predicted probability diverges from the actual label. So predicting a probability of .012 when the actual observation label is 1 would be bad and result in a high loss value. A perfect score would have a log loss of 0. + +$$ +Binary\ Cross\ Entropy = -\frac{1}{N}\sum_{i=1}^N[y_i\log(p_i) + (1-y_i)\log(1-p_i)] +$$ + +## Parameters +This cost function does not have any parameters. + +## Example +```php +use Rubix\ML\NeuralNet\CostFunctions\BinaryCrossEntropy; + +$costFunction = new BinaryCrossEntropy(); +``` diff --git a/docs/neural-network/cost-functions/cross-entropy.md b/docs/neural-network/cost-functions/cross-entropy.md deleted file mode 100644 index 2a3e96811..000000000 --- a/docs/neural-network/cost-functions/cross-entropy.md +++ /dev/null @@ -1,18 +0,0 @@ -[source] - -# Cross Entropy -Cross Entropy (or *log loss*) measures the performance of a classification model whose output is a joint probability distribution over the possible classes. Entropy increases as the predicted probability distribution diverges from the actual distribution. - -$$ -Cross Entropy = -\sum_{c=1}^My_{o,c}\log(p_{o,c}) -$$ - -## Parameters -This cost function does not have any parameters. - -## Example -```php -use Rubix\ML\NeuralNet\CostFunctions\CrossEntropy\CrossEntropy; - -$costFunction = new CrossEntropy(); -``` diff --git a/docs/neural-network/cost-functions/multiclass-cross-entropy.md b/docs/neural-network/cost-functions/multiclass-cross-entropy.md new file mode 100644 index 000000000..9b63ba5e2 --- /dev/null +++ b/docs/neural-network/cost-functions/multiclass-cross-entropy.md @@ -0,0 +1,18 @@ +[source] + +# Multiclass Cross Entropy +Multiclass Cross Entropy measures the performance of a multiclass classification model whose output is a probability distribution over the possible classes. Cross-entropy loss increases as the predicted probability distribution diverges from the actual distribution. + +$$ +Multiclass\ Cross\ Entropy = -\frac{1}{N}\sum_{i=1}^N\sum_{c=1}^C y_{i,c}\log(p_{i,c}) +$$ + +## Parameters +This cost function does not have any parameters. + +## Example +```php +use Rubix\ML\NeuralNet\CostFunctions\MulticlassCrossEntropy; + +$costFunction = new MulticlassCrossEntropy(); +``` diff --git a/mkdocs.yml b/mkdocs.yml index 13b89bf03..3bd377756 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -176,7 +176,8 @@ nav: - SiLU: neural-network/activation-functions/silu.md - Thresholded ReLU: neural-network/activation-functions/thresholded-relu.md - Cost Functions: - - Cross Entropy: neural-network/cost-functions/cross-entropy.md + - Binary Cross Entropy: neural-network/cost-functions/binary-cross-entropy.md + - Multiclass Cross Entropy: neural-network/cost-functions/multiclass-cross-entropy.md - Huber Loss: neural-network/cost-functions/huber-loss.md - Least Squares: neural-network/cost-functions/least-squares.md - Relative Entropy: neural-network/cost-functions/relative-entropy.md diff --git a/src/Classifiers/LogisticRegression.php b/src/Classifiers/LogisticRegression.php index c31af2825..57d1d5764 100644 --- a/src/Classifiers/LogisticRegression.php +++ b/src/Classifiers/LogisticRegression.php @@ -28,7 +28,7 @@ use Rubix\ML\Specifications\DatasetIsLabeled; use Rubix\ML\Specifications\DatasetIsNotEmpty; use Rubix\ML\Specifications\SpecificationChain; -use Rubix\ML\NeuralNet\CostFunctions\CrossEntropy; +use Rubix\ML\NeuralNet\CostFunctions\BinaryCrossEntropy; use Rubix\ML\Specifications\DatasetHasDimensionality; use Rubix\ML\NeuralNet\CostFunctions\ClassificationLoss; use Rubix\ML\Specifications\LabelsAreCompatibleWithLearner; @@ -163,7 +163,7 @@ public function __construct( $this->l2Penalty = $l2Penalty; $this->epochs = $epochs; $this->minChange = $minChange; - $this->costFn = $costFn ?? new CrossEntropy(); + $this->costFn = $costFn ?? new BinaryCrossEntropy(); } /** diff --git a/src/Classifiers/MultilayerPerceptron.php b/src/Classifiers/MultilayerPerceptron.php index d50b44f9f..2911b351a 100644 --- a/src/Classifiers/MultilayerPerceptron.php +++ b/src/Classifiers/MultilayerPerceptron.php @@ -30,9 +30,10 @@ use Rubix\ML\Specifications\DatasetIsLabeled; use Rubix\ML\Specifications\DatasetIsNotEmpty; use Rubix\ML\Specifications\SpecificationChain; -use Rubix\ML\NeuralNet\CostFunctions\CrossEntropy; +use Rubix\ML\NeuralNet\CostFunctions\MulticlassCrossEntropy; use Rubix\ML\Specifications\DatasetHasDimensionality; use Rubix\ML\NeuralNet\CostFunctions\ClassificationLoss; +use Rubix\ML\NeuralNet\CostFunctions\BinaryCrossEntropy; use Rubix\ML\Specifications\LabelsAreCompatibleWithLearner; use Rubix\ML\Specifications\EstimatorIsCompatibleWithMetric; use Rubix\ML\Specifications\SamplesAreCompatibleWithEstimator; @@ -243,6 +244,10 @@ public function __construct( . " between 0 and 0.5, $holdOut given."); } + if ($costFn and $costFn instanceof BinaryCrossEntropy) { + throw new InvalidArgumentException('Not compatible with binary cross entropy.'); + } + if ($metric) { EstimatorIsCompatibleWithMetric::with($this, $metric)->check(); } @@ -255,7 +260,7 @@ public function __construct( $this->evalInterval = $evalInterval; $this->window = $window; $this->holdOut = $holdOut; - $this->costFn = $costFn ?? new CrossEntropy(); + $this->costFn = $costFn ?? new MulticlassCrossEntropy(); $this->metric = $metric ?? new FBeta(); } diff --git a/src/Classifiers/SoftmaxClassifier.php b/src/Classifiers/SoftmaxClassifier.php index 13fb1dc3e..c6da32bfa 100644 --- a/src/Classifiers/SoftmaxClassifier.php +++ b/src/Classifiers/SoftmaxClassifier.php @@ -25,7 +25,8 @@ use Rubix\ML\Specifications\DatasetIsLabeled; use Rubix\ML\Specifications\DatasetIsNotEmpty; use Rubix\ML\Specifications\SpecificationChain; -use Rubix\ML\NeuralNet\CostFunctions\CrossEntropy; +use Rubix\ML\NeuralNet\CostFunctions\MulticlassCrossEntropy; +use Rubix\ML\NeuralNet\CostFunctions\BinaryCrossEntropy; use Rubix\ML\Specifications\DatasetHasDimensionality; use Rubix\ML\NeuralNet\CostFunctions\ClassificationLoss; use Rubix\ML\Specifications\LabelsAreCompatibleWithLearner; @@ -154,12 +155,16 @@ public function __construct( . " greater than 0, $minChange given."); } + if ($costFn and $costFn instanceof BinaryCrossEntropy) { + throw new InvalidArgumentException('Not compatible with binary cross entropy.'); + } + $this->batchSize = $batchSize; $this->optimizer = $optimizer ?? new Adam(); $this->l2Penalty = $l2Penalty; $this->epochs = $epochs; $this->minChange = $minChange; - $this->costFn = $costFn ?? new CrossEntropy(); + $this->costFn = $costFn ?? new MulticlassCrossEntropy(); } /** diff --git a/src/NeuralNet/CostFunctions/CrossEntropy.php b/src/NeuralNet/CostFunctions/BinaryCrossEntropy.php similarity index 67% rename from src/NeuralNet/CostFunctions/CrossEntropy.php rename to src/NeuralNet/CostFunctions/BinaryCrossEntropy.php index 435879c2c..451667b07 100644 --- a/src/NeuralNet/CostFunctions/CrossEntropy.php +++ b/src/NeuralNet/CostFunctions/BinaryCrossEntropy.php @@ -13,21 +13,21 @@ use const Rubix\ML\EPSILON; /** - * Cross Entropy + * Binary Cross Entropy * - * Cross Entropy, or log loss, measures the performance of a classification model - * whose output is a probability value between 0 and 1. Cross-entropy loss - * increases as the predicted probability diverges from the actual label. So - * predicting a probability of .012 when the actual observation label is 1 would - * be bad and result in a high loss value. A perfect score would have a log loss - * of 0. + * Binary Cross Entropy, or log loss, measures the performance of a binary + * classification model whose output is a probability value between 0 and 1. + * Cross-entropy loss increases as the predicted probability diverges from the + * actual label. So predicting a probability of .012 when the actual observation + * label is 1 would be bad and result in a high loss value. A perfect score + * would have a log loss of 0. * * @category Machine Learning * @package Rubix/ML * @author Andrew DalPino * @author Samuel Akopyan */ -class CrossEntropy implements ClassificationLoss +class BinaryCrossEntropy implements ClassificationLoss { use AssertsShapes; @@ -42,7 +42,7 @@ public function __construct() /** * Compute the loss score. * - * L(y, ŷ) = -Σ(y * log(ŷ)) / n + * L(y, ŷ) = -Σ(y * log(ŷ) + (1 - y) * log(1 - ŷ)) / n * * @param NDArray $output The output of the network * @param NDArray $target The target values @@ -52,12 +52,17 @@ public function compute(NDArray $output, NDArray $target) : float { $this->assertSameShape($output, $target); - // Clip values to avoid log(0) - $output = NumPower::clip($output, EPSILON, 1.0); + $output = NumPower::clip($output, EPSILON, 1.0 - EPSILON); + $target = NumPower::clip($target, EPSILON, 1.0 - EPSILON); $logOutput = NumPower::log($output); + $logOneMinusOutput = NumPower::log(NumPower::subtract(1.0, $output)); + $oneMinusTarget = NumPower::subtract(1.0, $target); + $product = NumPower::multiply($target, $logOutput); - $negated = NumPower::multiply($product, -1.0); + $product2 = NumPower::multiply($oneMinusTarget, $logOneMinusOutput); + $sum = NumPower::add($product, $product2); + $negated = NumPower::multiply($sum, -1.0); return NumPower::mean($negated); } @@ -75,13 +80,10 @@ public function differentiate(NDArray $output, NDArray $target) : NDArray { $this->assertSameShape($output, $target); - // Numerator = ŷ - y (calculate before clipping to preserve zeros) $numerator = NumPower::subtract($output, $target); - // Clip values to avoid division by zero $output = NumPower::clip($output, EPSILON, 1.0 - EPSILON); - // Denominator = ŷ * (1 - ŷ) $oneMinusOutput = NumPower::subtract(1.0, $output); $denominator = NumPower::multiply($output, $oneMinusOutput); $denominator = NumPower::clip($denominator, EPSILON, 1.0); @@ -96,6 +98,6 @@ public function differentiate(NDArray $output, NDArray $target) : NDArray */ public function __toString() : string { - return 'Cross Entropy'; + return 'Binary Cross Entropy'; } } diff --git a/src/NeuralNet/CostFunctions/MulticlassCrossEntropy.php b/src/NeuralNet/CostFunctions/MulticlassCrossEntropy.php new file mode 100644 index 000000000..64ecb0c09 --- /dev/null +++ b/src/NeuralNet/CostFunctions/MulticlassCrossEntropy.php @@ -0,0 +1,91 @@ + + */ +class MulticlassCrossEntropy implements ClassificationLoss +{ + use AssertsShapes; + + public function __construct() + { + SpecificationChain::with([ + new ExtensionIsLoaded('RubixNumPower'), + new ExtensionMinimumVersion('RubixNumPower', '0.7.0'), + ])->check(); + } + + /** + * Compute the loss score. + * + * L(y, ŷ) = -Σ(y * log(ŷ)) / n + * + * @param NDArray $output The output of the network + * @param NDArray $target The target values + * @return float + */ + public function compute(NDArray $output, NDArray $target) : float + { + $this->assertSameShape($output, $target); + + $output = NumPower::clip($output, EPSILON, 1.0); + + $logOutput = NumPower::log($output); + $product = NumPower::multiply($target, $logOutput); + $negated = NumPower::multiply($product, -1.0); + + return NumPower::mean($negated); + } + + /** + * Calculate the gradient of the cost function with respect to the output. + * + * ∂L/∂ŷ = -y / ŷ + * + * @param NDArray $output The output of the network + * @param NDArray $target The target values + * @return NDArray + */ + public function differentiate(NDArray $output, NDArray $target) : NDArray + { + $this->assertSameShape($output, $target); + + $output = NumPower::clip($output, EPSILON, 1.0); + + $negated = NumPower::multiply($target, -1.0); + + return NumPower::divide($negated, $output); + } + + /** + * Return the string representation of the object. + * + * @return string + */ + public function __toString() : string + { + return 'Multiclass Cross Entropy'; + } +} diff --git a/src/NeuralNet/Layers/Binary.php b/src/NeuralNet/Layers/Binary.php index d641fbff4..feba26f99 100644 --- a/src/NeuralNet/Layers/Binary.php +++ b/src/NeuralNet/Layers/Binary.php @@ -9,7 +9,7 @@ use Rubix\ML\Specifications\ExtensionMinimumVersion; use Rubix\ML\Specifications\SpecificationChain; use Rubix\ML\NeuralNet\Optimizers\Optimizer; -use Rubix\ML\NeuralNet\CostFunctions\CrossEntropy; +use Rubix\ML\NeuralNet\CostFunctions\BinaryCrossEntropy; use Rubix\ML\NeuralNet\ActivationFunctions\Sigmoid; use Rubix\ML\NeuralNet\CostFunctions\ClassificationLoss; use Rubix\ML\Exceptions\InvalidArgumentException; @@ -93,7 +93,7 @@ public function __construct(array $classes, ?ClassificationLoss $costFn = null) ]; $this->classes = $classes; - $this->costFn = $costFn ?? new CrossEntropy(); + $this->costFn = $costFn ?? new BinaryCrossEntropy(); $this->sigmoid = new Sigmoid(); } @@ -197,9 +197,9 @@ public function gradient(NDArray $input, NDArray $output, NDArray $expected) : N { $n = $output->shape()[1]; - if ($this->costFn instanceof CrossEntropy) { - // Optimization specific to (sigmoid +) binary cross entropy: - // the loss derivative cancels with the sigmoid derivative, so dZ = (output - expected). + // Optimization specific to sigmoid + binary cross entropy. + // The loss derivative cancels with the sigmoid derivative, so dZ = (output - expected). + if ($this->costFn instanceof BinaryCrossEntropy) { return NumPower::divide( NumPower::subtract($output, $expected), $n diff --git a/src/NeuralNet/Layers/Multiclass.php b/src/NeuralNet/Layers/Multiclass.php index 7ccf159ba..972ffe781 100644 --- a/src/NeuralNet/Layers/Multiclass.php +++ b/src/NeuralNet/Layers/Multiclass.php @@ -9,7 +9,7 @@ use Rubix\ML\Specifications\ExtensionMinimumVersion; use Rubix\ML\Specifications\SpecificationChain; use Rubix\ML\NeuralNet\Optimizers\Optimizer; -use Rubix\ML\NeuralNet\CostFunctions\CrossEntropy; +use Rubix\ML\NeuralNet\CostFunctions\MulticlassCrossEntropy; use Rubix\ML\NeuralNet\ActivationFunctions\Softmax; use Rubix\ML\NeuralNet\CostFunctions\ClassificationLoss; use Rubix\ML\Exceptions\InvalidArgumentException; @@ -88,7 +88,7 @@ public function __construct(array $classes, ?ClassificationLoss $costFn = null) ])->check(); $this->classes = $classes; - $this->costFn = $costFn ?? new CrossEntropy(); + $this->costFn = $costFn ?? new MulticlassCrossEntropy(); $this->softmax = new Softmax(); } @@ -213,7 +213,9 @@ public function gradient(NDArray $input, NDArray $output, NDArray $expected) : N { $n = array_product($output->shape()); - if ($this->costFn instanceof CrossEntropy) { + // Optimization specific to softmax + multiclass cross entropy. + // The loss derivative cancels with the softmax derivative, so dZ = (output - expected). + if ($this->costFn instanceof MulticlassCrossEntropy) { return NumPower::divide( NumPower::subtract($output, $expected), $n diff --git a/tests/Classifiers/LogisticRegressionTest.php b/tests/Classifiers/LogisticRegressionTest.php index 7bb247865..a2d1d6839 100644 --- a/tests/Classifiers/LogisticRegressionTest.php +++ b/tests/Classifiers/LogisticRegressionTest.php @@ -17,7 +17,7 @@ use Rubix\ML\Datasets\Generators\Agglomerate; use Rubix\ML\Transformers\ZScaleStandardizer; use Rubix\ML\CrossValidation\Metrics\FBeta; -use Rubix\ML\NeuralNet\CostFunctions\CrossEntropy; +use Rubix\ML\NeuralNet\CostFunctions\BinaryCrossEntropy; use Rubix\ML\Exceptions\InvalidArgumentException; use Rubix\ML\Exceptions\RuntimeException; use PHPUnit\Framework\TestCase; @@ -74,7 +74,7 @@ protected function setUp() : void l2Penalty: 1e-4, epochs: 300, minChange: 1e-4, - costFn: new CrossEntropy() + costFn: new BinaryCrossEntropy() ); $this->metric = new FBeta(); @@ -116,7 +116,7 @@ public function testParams() : void 'l2 penalty' => 1e-4, 'epochs' => 300, 'min change' => 1e-4, - 'cost fn' => new CrossEntropy(), + 'cost fn' => new BinaryCrossEntropy(), ]; $this->assertEquals($expected, $this->estimator->params()); diff --git a/tests/Classifiers/MultilayerPerceptronTest.php b/tests/Classifiers/MultilayerPerceptronTest.php index 895512373..9511875ad 100644 --- a/tests/Classifiers/MultilayerPerceptronTest.php +++ b/tests/Classifiers/MultilayerPerceptronTest.php @@ -24,7 +24,7 @@ use Rubix\ML\Transformers\ZScaleStandardizer; use Rubix\ML\Datasets\Generators\Agglomerate; use Rubix\ML\Classifiers\MultilayerPerceptron; -use Rubix\ML\NeuralNet\CostFunctions\CrossEntropy; +use Rubix\ML\NeuralNet\CostFunctions\MulticlassCrossEntropy; use Rubix\ML\NeuralNet\ActivationFunctions\LeakyReLU; use Rubix\ML\Exceptions\InvalidArgumentException; use Rubix\ML\Exceptions\RuntimeException; @@ -107,7 +107,7 @@ protected function setUp() : void evalInterval: 3, window: 5, holdOut: 0.1, - costFn: new CrossEntropy(), + costFn: new MulticlassCrossEntropy(), metric: new FBeta() ); @@ -162,7 +162,7 @@ public function testParams() : void 'eval interval' => 3, 'window' => 5, 'hold out' => 0.1, - 'cost fn' => new CrossEntropy(), + 'cost fn' => new MulticlassCrossEntropy(), 'metric' => new FBeta(), ]; diff --git a/tests/Classifiers/SoftmaxClassifierTest.php b/tests/Classifiers/SoftmaxClassifierTest.php index 83d67af5e..955793657 100644 --- a/tests/Classifiers/SoftmaxClassifierTest.php +++ b/tests/Classifiers/SoftmaxClassifierTest.php @@ -17,7 +17,7 @@ use Rubix\ML\Transformers\ZScaleStandardizer; use Rubix\ML\Datasets\Generators\Agglomerate; use Rubix\ML\CrossValidation\Metrics\FBeta; -use Rubix\ML\NeuralNet\CostFunctions\CrossEntropy; +use Rubix\ML\NeuralNet\CostFunctions\MulticlassCrossEntropy; use Rubix\ML\Exceptions\InvalidArgumentException; use Rubix\ML\Exceptions\RuntimeException; use PHPUnit\Framework\TestCase; @@ -78,7 +78,7 @@ protected function setUp() : void l2Penalty: 1e-4, epochs: 300, minChange: 1e-4, - costFn: new CrossEntropy() + costFn: new MulticlassCrossEntropy() ); $this->metric = new FBeta(); @@ -120,7 +120,7 @@ public function testParams() : void 'l2 penalty' => 1e-4, 'epochs' => 300, 'min change' => 1e-4, - 'cost fn' => new CrossEntropy(), + 'cost fn' => new MulticlassCrossEntropy(), ]; $this->assertEquals($expected, $this->estimator->params()); diff --git a/tests/NeuralNet/CostFunctions/BinaryCrossEntropyTest.php b/tests/NeuralNet/CostFunctions/BinaryCrossEntropyTest.php new file mode 100644 index 000000000..2028bcd63 --- /dev/null +++ b/tests/NeuralNet/CostFunctions/BinaryCrossEntropyTest.php @@ -0,0 +1,193 @@ +costFn = new BinaryCrossEntropy(); + } + + #[Test] + #[TestDox('Can be cast to a string')] + public function testToString() : void + { + static::assertEquals('Binary Cross Entropy', (string) $this->costFn); + } + + #[Test] + #[TestDox('Throws exception when output and target shapes do not match in compute')] + public function testComputeThrowsExceptionOnShapeMismatch() : void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Output and target must have the same shape.'); + + $output = NumPower::array([[1.0, 2.0, 3.0]]); + $target = NumPower::array([[1.0, 2.0]]); + + $this->costFn->compute($output, $target); + } + + #[Test] + #[TestDox('Throws exception when output and target shapes do not match in differentiate')] + public function testDifferentiateThrowsExceptionOnShapeMismatch() : void + { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage('Output and target must have the same shape.'); + + $output = NumPower::array([[1.0, 2.0, 3.0]]); + $target = NumPower::array([[1.0, 2.0]]); + + $this->costFn->differentiate($output, $target); + } + + #[Test] + #[TestDox('Compute loss score')] + #[DataProvider('computeProvider')] + public function testCompute(NDArray $output, NDArray $target, float $expected) : void + { + $loss = $this->costFn->compute($output, $target); + + if (is_nan($expected)) { + self::assertNan($loss); + } else { + self::assertEqualsWithDelta($expected, $loss, 1e-7); + } + } + + #[Test] + #[TestDox('Calculate gradient of cost function')] + #[DataProvider('differentiateProvider')] + public function testDifferentiate(NDArray $output, NDArray $target, array $expected) : void + { + $gradient = $this->costFn->differentiate($output, $target); + + $gradientArray = $gradient->toArray(); + + self::assertEqualsWithDelta($expected, $gradientArray, 1e-7); + } +} diff --git a/tests/NeuralNet/CostFunctions/CrossEntropyTest.php b/tests/NeuralNet/CostFunctions/MulticlassCrossEntropyTest.php similarity index 88% rename from tests/NeuralNet/CostFunctions/CrossEntropyTest.php rename to tests/NeuralNet/CostFunctions/MulticlassCrossEntropyTest.php index 43ed1cff0..a0a371684 100644 --- a/tests/NeuralNet/CostFunctions/CrossEntropyTest.php +++ b/tests/NeuralNet/CostFunctions/MulticlassCrossEntropyTest.php @@ -12,15 +12,15 @@ use NumPower; use NDArray; use Rubix\ML\Exceptions\InvalidArgumentException; -use Rubix\ML\NeuralNet\CostFunctions\CrossEntropy; +use Rubix\ML\NeuralNet\CostFunctions\MulticlassCrossEntropy; use PHPUnit\Framework\TestCase; use Generator; #[Group('CostFunctions')] -#[CoversClass(CrossEntropy::class)] -class CrossEntropyTest extends TestCase +#[CoversClass(MulticlassCrossEntropy::class)] +class MulticlassCrossEntropyTest extends TestCase { - protected CrossEntropy $costFn; + protected MulticlassCrossEntropy $costFn; public static function computeProvider() : Generator { @@ -85,7 +85,7 @@ public static function differentiateProvider() : Generator [1.0, 0.0, 0.0], ]), [ - [-1.0101009, 1.0101009, 0.0], + [-1.0101010, 0.0, 0.0], ], ]; @@ -97,7 +97,7 @@ public static function differentiateProvider() : Generator [0.0, 1.0, 0.0], ]), [ - [1.2499999, -2.5, 1.6666666], + [0.0, -2.5, 0.0], ], ]; @@ -109,7 +109,7 @@ public static function differentiateProvider() : Generator [1.0, 0.0, 0.0], ]), [ - [-100000000.0, 1.1111111, 9.9999981], + [-100000000.0, 0.0, 0.0], ], ]; @@ -125,23 +125,23 @@ public static function differentiateProvider() : Generator [0.0, 0.0, 1.0], ]), [ - [1.2499999, 1.1111111, -1.4285714], - [0.0, -1.1111111, 1.1111111], - [1.1111111, 1.4285714, -1.6666666], + [0.0, 0.0, -1.4285714], + [0.0, -1.1111111, 0.0], + [0.0, 0.0, -1.6666666], ], ]; } protected function setUp() : void { - $this->costFn = new CrossEntropy(); + $this->costFn = new MulticlassCrossEntropy(); } #[Test] #[TestDox('Can be cast to a string')] public function testToString() : void { - static::assertEquals('Cross Entropy', (string) $this->costFn); + static::assertEquals('Multiclass Cross Entropy', (string) $this->costFn); } #[Test] diff --git a/tests/NeuralNet/FeedForwardTest.php b/tests/NeuralNet/FeedForwardTest.php index cb2d6cae5..2359de533 100644 --- a/tests/NeuralNet/FeedForwardTest.php +++ b/tests/NeuralNet/FeedForwardTest.php @@ -9,7 +9,7 @@ use PHPUnit\Framework\TestCase; use Rubix\ML\Datasets\Labeled; use Rubix\ML\NeuralNet\ActivationFunctions\ReLU; -use Rubix\ML\NeuralNet\CostFunctions\CrossEntropy; +use Rubix\ML\NeuralNet\CostFunctions\MulticlassCrossEntropy; use Rubix\ML\NeuralNet\Layers\Activation; use Rubix\ML\NeuralNet\Layers\Hidden; use Rubix\ML\NeuralNet\Layers\Input; @@ -67,7 +67,7 @@ protected function setUp() : void new Dense(3), ]; - $this->output = new Multiclass(['yes', 'no', 'maybe'], new CrossEntropy()); + $this->output = new Multiclass(['yes', 'no', 'maybe'], new MulticlassCrossEntropy()); $this->network = new FeedForward($this->input, $this->hidden, $this->output, new Adam(0.001)); } diff --git a/tests/NeuralNet/Layers/BinaryTest.php b/tests/NeuralNet/Layers/BinaryTest.php index 517cf3758..59017772c 100644 --- a/tests/NeuralNet/Layers/BinaryTest.php +++ b/tests/NeuralNet/Layers/BinaryTest.php @@ -15,7 +15,7 @@ use Rubix\ML\Deferred; use Rubix\ML\NeuralNet\Layers\Binary; use Rubix\ML\NeuralNet\Optimizers\Stochastic; -use Rubix\ML\NeuralNet\CostFunctions\CrossEntropy; +use Rubix\ML\NeuralNet\CostFunctions\BinaryCrossEntropy; use Rubix\ML\Exceptions\InvalidArgumentException; use PHPUnit\Framework\TestCase; @@ -85,7 +85,7 @@ protected function setUp() : void $this->optimizer = new Stochastic(0.001); - $this->layer = new Binary(classes: ['hot', 'cold'], costFn: new CrossEntropy()); + $this->layer = new Binary(classes: ['hot', 'cold'], costFn: new BinaryCrossEntropy()); } #[Test] @@ -94,7 +94,7 @@ public function testToString() : void { $this->layer->initialize(1); - self::assertEquals('Binary (cost function: Cross Entropy)', (string) $this->layer); + self::assertEquals('Binary (cost function: Binary Cross Entropy)', (string) $this->layer); } #[Test] @@ -111,14 +111,14 @@ public function testInitializeWidth() : void public function testConstructorRejectsInvalidClasses(array $classes) : void { $this->expectException(InvalidArgumentException::class); - new Binary(classes: $classes, costFn: new CrossEntropy()); + new Binary(classes: $classes, costFn: new BinaryCrossEntropy()); } #[Test] #[TestDox('Constructor accepts classes arrays that dedupe to exactly 2 labels')] public function testConstructorAcceptsDuplicateClassesThatDedupeToTwo() : void { - $layer = new Binary(classes: ['hot', 'cold', 'hot'], costFn: new CrossEntropy()); + $layer = new Binary(classes: ['hot', 'cold', 'hot'], costFn: new BinaryCrossEntropy()); // Should initialize without throwing and report correct width $layer->initialize(1); self::assertEquals(1, $layer->width()); diff --git a/tests/NeuralNet/Layers/MulticlassTest.php b/tests/NeuralNet/Layers/MulticlassTest.php index ba272d7b1..5199253e1 100644 --- a/tests/NeuralNet/Layers/MulticlassTest.php +++ b/tests/NeuralNet/Layers/MulticlassTest.php @@ -16,7 +16,7 @@ use Rubix\ML\Deferred; use Rubix\ML\NeuralNet\Layers\Multiclass; use Rubix\ML\NeuralNet\Optimizers\Stochastic; -use Rubix\ML\NeuralNet\CostFunctions\CrossEntropy; +use Rubix\ML\NeuralNet\CostFunctions\MulticlassCrossEntropy; use Rubix\ML\NeuralNet\CostFunctions\RelativeEntropy; use PHPUnit\Framework\TestCase; @@ -97,7 +97,7 @@ protected function setUp() : void $this->layer = new Multiclass( classes: ['hot', 'cold', 'ice cold'], - costFn: new CrossEntropy() + costFn: new MulticlassCrossEntropy() ); } @@ -250,7 +250,7 @@ public function testInfer(array $expected) : void #[TestDox('It returns correct string representation')] public function testToStringReturnsCorrectValue() : void { - $expected = 'Multiclass (cost function: Cross Entropy)'; + $expected = 'Multiclass (cost function: Multiclass Cross Entropy)'; self::assertSame($expected, (string) $this->layer); } diff --git a/tests/NeuralNet/NetworkTest.php b/tests/NeuralNet/NetworkTest.php index 261ed1ca1..fa355302a 100644 --- a/tests/NeuralNet/NetworkTest.php +++ b/tests/NeuralNet/NetworkTest.php @@ -9,7 +9,7 @@ use PHPUnit\Framework\TestCase; use Rubix\ML\Datasets\Labeled; use Rubix\ML\NeuralNet\ActivationFunctions\ReLU; -use Rubix\ML\NeuralNet\CostFunctions\CrossEntropy; +use Rubix\ML\NeuralNet\CostFunctions\MulticlassCrossEntropy; use Rubix\ML\NeuralNet\Layers\Activation; use Rubix\ML\NeuralNet\Layers\Hidden; use Rubix\ML\NeuralNet\Layers\Input; @@ -61,7 +61,7 @@ protected function setUp() : void $this->output = new Multiclass( classes: ['yes', 'no', 'maybe'], - costFn: new CrossEntropy() + costFn: new MulticlassCrossEntropy() ); $this->network = new FeedForward( diff --git a/tests/NeuralNet/SnapshotTest.php b/tests/NeuralNet/SnapshotTest.php index a13b15bc4..7e5ffe6ed 100644 --- a/tests/NeuralNet/SnapshotTest.php +++ b/tests/NeuralNet/SnapshotTest.php @@ -10,7 +10,7 @@ use PHPUnit\Framework\TestCase; use Rubix\ML\Datasets\Labeled; use Rubix\ML\NeuralNet\ActivationFunctions\ELU; -use Rubix\ML\NeuralNet\CostFunctions\CrossEntropy; +use Rubix\ML\NeuralNet\CostFunctions\BinaryCrossEntropy; use Rubix\ML\NeuralNet\Layers\Activation; use Rubix\ML\NeuralNet\Layers\Binary; use Rubix\ML\NeuralNet\Layers\Dense; @@ -266,7 +266,7 @@ protected function createNetwork() : FeedForward ], output: new Binary( classes: ['yes', 'no'], - costFn: new CrossEntropy() + costFn: new BinaryCrossEntropy() ), optimizer: new Stochastic() );