| // Copyright 2014 Google Inc. All Rights Reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| // Author: nevena@google.com (Nevena Lazic) |
| // |
| // In InnerProductLossFunction, the loss of a labeled example |
| // (instance_i, label_i) is only a function of the inner product |
| // <weights, instance_i> and label_i. label_i is a scalar (single element |
| // VectorXf). |
| // |
| // Derived classes need to provide InnerProductExampleLoss, |
| // InnerProductExampleGradient, and InnerProductPredictLabel, and set the |
| // 'curvature_' parameter. |
| // |
| // Current implementations include logistic-regression, averaged-logistic, |
| // linear-regression, poisson-regression, smooth-hinge. |
| |
| #pragma once |
| |
| #include <float.h> |
| #include <math.h> |
| #include <functional> |
| #include <mutex> |
| |
| #include "lossmin/eigen-types.h" |
| #include "lossmin/losses/loss-function.h" |
| |
| namespace lossmin { |
| |
| class InnerProductLossFunction : public LossFunction { |
| public: |
| // Returns the loss for a single example. |
| float ExampleLoss( |
| const Weights &weights, const InstanceSet &instances, |
| const LabelSet &labels, int example) const override; |
| |
| // Adds the gradient of a single example to 'gradient'. |
| void AddExampleGradient( |
| const Weights &weights, const InstanceSet &instances, |
| const LabelSet &labels, int example, float weights_scale, |
| float example_scale, Weights *gradient) const override; |
| |
| // Returns the gradient of a single example. |
| void ExampleGradient( |
| const Weights &weights, const InstanceSet &instances, |
| const LabelSet &labels, int example, float weights_scale, |
| float example_scale, |
| std::vector<std::pair<int, float>> *example_gradient) const override; |
| |
| // Assigns labels to 'instances' given 'weights'. |
| void PredictLabels(const Weights &weights, const InstanceSet &instances, |
| LabelSet *labels) const override; |
| |
| // Returns an upper bound on the loss curvature. |
| float LossCurvature(const InstanceSet &instances) const override; |
| |
| // Returns an upper bound on the per-coordinate curvature. |
| void PerCoordinateCurvature( |
| const InstanceSet &instances, |
| VectorXf *per_coordinate_curvature) const override; |
| |
| virtual float InnerProductExampleLoss(float inner_product, float label) |
| const = 0; |
| |
| virtual float InnerProductExampleGradient(float inner_product, float label) |
| const = 0; |
| |
| virtual float InnerProductPredictLabel(float inner_product) const = 0; |
| |
| // Returns 'curvature_'. |
| virtual float InnerProductCurvature(float inner_product, float label) const { |
| return curvature_; |
| } |
| |
| protected: |
| // Sets the upper bound on the curvature of the loss function. |
| void set_curvature(float curvature) { curvature_ = curvature; } |
| |
| private: |
| // Mutex for synchronous updates of the gradient vector. |
| mutable std::mutex gradient_update_mutex_; |
| |
| // Upper bound on the absolute value of the second derivative of the loss: |
| // |d^2 loss(x) / dx^2| <= curvature_, where 'x' is the inner product |
| // <instance, weights>. Should be set by derived classes. |
| float curvature_; |
| }; |
| |
| // Binary logistic regression with labels in {-1, 1}. |
| class LogisticRegressionLossFunction : public InnerProductLossFunction { |
| public: |
| LogisticRegressionLossFunction() { set_curvature(0.25); } |
| |
| // Returns the logistic loss. |
| float InnerProductExampleLoss(float inner_product, float label) |
| const override { |
| return log1pf(exp(-label * inner_product)); |
| } |
| |
| // Returns the gradient of the logistic loss wrt 'inner_product'. |
| float InnerProductExampleGradient(float inner_product, float label) |
| const override { |
| if (label * inner_product < kInputMin) return -label; |
| return -label / (1.0f + exp(label * inner_product)); |
| } |
| |
| // Assigns a label in {-1, 1} given 'inner_product'. |
| float InnerProductPredictLabel(float inner_product) const override { |
| if (inner_product > 0) return 1.0f; |
| return -1.0f; |
| } |
| |
| // Returns the magnitude of the second derivative of the loss, evaluated at |
| // 'inner_product' and 'label'. |
| float InnerProductCurvature(float inner_product, float label) |
| const override { |
| if (-label * inner_product < kInputMin) return 0.0f; |
| float label_prob = 1.0f / (1.0f + exp(-label * inner_product)); |
| return label_prob * (1.0f - label_prob); |
| } |
| |
| // Min allowed exp input. |
| const float kInputMin = log(FLT_EPSILON); |
| }; |
| |
| // Linear regression with squared error loss. |
| class LinearRegressionLossFunction : public InnerProductLossFunction { |
| public: |
| LinearRegressionLossFunction() { set_curvature(1.0); } |
| |
| // Returns the squared error loss. |
| float InnerProductExampleLoss(float inner_product, float label) |
| const override { |
| return 0.5 * (inner_product - label) * (inner_product - label); |
| } |
| |
| // Returns the gradient of the squared error loss wrt 'inner_product'. |
| float InnerProductExampleGradient(float inner_product, float label) |
| const override { |
| return inner_product - label; |
| } |
| |
| // Assigns a label given 'inner_product'. |
| float InnerProductPredictLabel(float inner_product) const override { |
| return inner_product; |
| } |
| }; |
| |
| // Poisson regression, where the distribution mean is the inner product between |
| // an instance and parameters. |
| class PoissonRegressionLossFunction : public InnerProductLossFunction { |
| public: |
| PoissonRegressionLossFunction() {} |
| |
| // Returns the Poisson regression loss. |
| float InnerProductExampleLoss(float inner_product, float label) |
| const override { |
| return exp(inner_product) - label * inner_product; |
| } |
| |
| // Returns the gradient of the Poisson regression loss wrt 'inner_product'. |
| float InnerProductExampleGradient(float inner_product, float label) |
| const override { |
| return exp(inner_product) - label; |
| } |
| |
| // Returns the mode of a Poisson distribution with mean 'inner_product'. |
| float InnerProductPredictLabel(float inner_product) const override { |
| return floor(inner_product); |
| } |
| |
| // Returns the magnitude of the second derivative of the loss, evaluated at |
| // 'inner_product' and 'label'. |
| float InnerProductCurvature(float inner_product, float label) |
| const override { |
| return exp(inner_product); |
| } |
| }; |
| |
| // Binary logistic regression where 'label' = P(true_label = 1) is in [0, 1], |
| // and the loss is averaged over labels: |
| // label * logloss(inner_product, 1) + (1 - label) * logloss(inner_product, 0). |
| class AveragedLogisticLossFunction : public InnerProductLossFunction { |
| public: |
| AveragedLogisticLossFunction() { set_curvature(0.25); } |
| |
| // Returns the averaged logistic loss. |
| float InnerProductExampleLoss(float inner_product, float label) |
| const override { |
| return (1.0 - label) * log1pf(exp(inner_product)) |
| + label * log1pf(exp(-inner_product)); |
| } |
| |
| // Returns the gradient of the averaged logistic loss wrt 'inner_product'. |
| float InnerProductExampleGradient(float inner_product, float label) |
| const override { |
| if (-inner_product < kInputMin) return -label + 1.0f; |
| return -label + 1.0f / (1.0f + exp(-inner_product)); |
| } |
| |
| // Returns the probability that the label is 1 given the inner product. |
| float InnerProductPredictLabel(float inner_product) const override { |
| if (-inner_product < kInputMin) return 1.0f; |
| return 1.0f / (1.0f + exp(-inner_product)); |
| } |
| |
| // Returns the magnitude of the second derivative of the loss, evaluated at |
| // 'inner_product' and 'label'. |
| float InnerProductCurvature(float inner_product, float label) |
| const override { |
| float label_prob = 1.0f / (1.0f + exp(-inner_product)); |
| if (-inner_product < kInputMin) return 0.0f; |
| return label_prob * (1.0f - label_prob); |
| } |
| |
| // Min allowed exp input. |
| const float kInputMin = log(FLT_EPSILON); |
| }; |
| |
| |
| // Smooth hinge loss. If a = inner_product * label, the loss is |
| // loss = 0.0 a > threshold |
| // 0.5 * (a - threshold)^2 0 < a <= threshold |
| // 0.5 * threshold^2 - threshold * a a <= 0 |
| class SmoothHingeLossFunction : public InnerProductLossFunction { |
| public: |
| SmoothHingeLossFunction() { set_curvature(1.0f); } |
| |
| /* |
| void Setup(const LossFunctionConfigContainer &config) override { |
| if (config.has_smooth_hinge_threshold()) { |
| threshold_ = config.smooth_hinge_threshold(); |
| } |
| set_curvature(threshold_); |
| } |
| */ |
| |
| float InnerProductExampleLoss(float inner_product, float label) |
| const override { |
| float hinge_input = inner_product *= label; |
| if (hinge_input >= threshold_) { |
| return 0.0; |
| } else if (hinge_input <= 0.0f) { |
| return (0.5 * threshold_ * threshold_ - threshold_ * hinge_input); |
| } else { |
| return 0.5 * (hinge_input - threshold_) * (hinge_input - threshold_); |
| } |
| } |
| |
| float InnerProductExampleGradient(float inner_product, float label) |
| const override { |
| float hinge_input = inner_product * label; |
| if (hinge_input >= threshold_) { |
| return 0.0f; |
| } else if (hinge_input <= 0.0f) { |
| return -label * threshold_; |
| } else { |
| return (hinge_input - threshold_) * label; |
| } |
| } |
| |
| // Assigns a label in {-1, 1} given 'inner_product'. |
| float InnerProductPredictLabel(float inner_product) const override { |
| if (inner_product > 0) return 1.0f; |
| return -1.0f; |
| } |
| |
| void set_threshold(float threshold) { threshold_ = threshold; } |
| |
| private: |
| float threshold_ = 1.0f; |
| }; |
| |
| |
| } // namespace lossmin |
| |