lossmin/losses/inner-product-loss-function.h

// Copyright 2014 Google Inc. All Rights Reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// Author: nevena@google.com (Nevena Lazic)
//
// In InnerProductLossFunction, the loss of a labeled example
// (instance_i, label_i) is only a function of the inner product
// <weights, instance_i> and label_i. label_i is a scalar (single element
// VectorXf).
//
// Derived classes need to provide InnerProductExampleLoss,
// InnerProductExampleGradient, and InnerProductPredictLabel, and set the
// 'curvature_' parameter.
//
// Current implementations include logistic-regression, averaged-logistic,
// linear-regression, poisson-regression, smooth-hinge.

#pragma once

#include <float.h>
#include <math.h>
#include <functional>
#include <mutex>

#include "lossmin/eigen-types.h"
#include "lossmin/losses/loss-function.h"

namespace lossmin {

class InnerProductLossFunction : public LossFunction {
 public:
  // Returns the loss for a single example.
  float ExampleLoss(
      const Weights &weights, const InstanceSet &instances,
      const LabelSet &labels, int example) const override;

  // Adds the gradient of a single example to 'gradient'.
  void AddExampleGradient(
      const Weights &weights, const InstanceSet &instances,
      const LabelSet &labels, int example, float weights_scale,
      float example_scale, Weights *gradient) const override;

  // Returns the gradient of a single example.
  void ExampleGradient(
      const Weights &weights, const InstanceSet &instances,
      const LabelSet &labels, int example, float weights_scale,
      float example_scale,
      std::vector<std::pair<int, float>> *example_gradient) const override;

  // Assigns labels to 'instances' given 'weights'.
  void PredictLabels(const Weights &weights, const InstanceSet &instances,
                     LabelSet *labels) const override;

  // Returns an upper bound on the loss curvature.
  float LossCurvature(const InstanceSet &instances) const override;

  // Returns an upper bound on the per-coordinate curvature.
  void PerCoordinateCurvature(
      const InstanceSet &instances,
      VectorXf *per_coordinate_curvature) const override;

  virtual float InnerProductExampleLoss(float inner_product, float label)
      const = 0;

  virtual float InnerProductExampleGradient(float inner_product, float label)
      const = 0;

  virtual float InnerProductPredictLabel(float inner_product) const = 0;

  // Returns 'curvature_'.
  virtual float InnerProductCurvature(float inner_product, float label) const {
    return curvature_;
  }

 protected:
  // Sets the upper bound on the curvature of the loss function.
  void set_curvature(float curvature) { curvature_ = curvature; }

 private:
  // Mutex for synchronous updates of the gradient vector.
  mutable std::mutex gradient_update_mutex_;

  // Upper bound on the absolute value of the second derivative of the loss:
  // |d^2 loss(x) / dx^2| <= curvature_, where 'x' is the inner product
  //  <instance, weights>. Should be set by derived classes.
  float curvature_;
};

// Binary logistic regression with labels in {-1, 1}.
class LogisticRegressionLossFunction : public InnerProductLossFunction {
 public:
  LogisticRegressionLossFunction() { set_curvature(0.25); }

  // Returns the logistic loss.
  float InnerProductExampleLoss(float inner_product, float label)
      const override {
    return log1pf(exp(-label * inner_product));
  }

  // Returns the gradient of the logistic loss wrt 'inner_product'.
  float InnerProductExampleGradient(float inner_product, float label)
      const override {
    if (label * inner_product < kInputMin) return -label;
    return -label / (1.0f + exp(label * inner_product));
  }

  // Assigns a label in {-1, 1} given 'inner_product'.
  float InnerProductPredictLabel(float inner_product) const override {
    if (inner_product > 0) return 1.0f;
    return -1.0f;
  }

  // Returns the magnitude of the second derivative of the loss, evaluated at
  // 'inner_product' and 'label'.
  float InnerProductCurvature(float inner_product, float label)
      const override {
    if (-label * inner_product < kInputMin) return 0.0f;
    float label_prob = 1.0f / (1.0f + exp(-label * inner_product));
    return label_prob * (1.0f - label_prob);
  }

  // Min allowed exp input.
  const float kInputMin = log(FLT_EPSILON);
};

// Linear regression with squared error loss.
class LinearRegressionLossFunction : public InnerProductLossFunction {
 public:
  LinearRegressionLossFunction() { set_curvature(1.0); }

  // Returns the squared error loss.
  float InnerProductExampleLoss(float inner_product, float label)
      const override {
    return 0.5 * (inner_product - label) * (inner_product - label);
  }

  // Returns the gradient of the squared error loss wrt 'inner_product'.
  float InnerProductExampleGradient(float inner_product, float label)
      const override {
    return inner_product - label;
  }

  // Assigns a label given 'inner_product'.
  float InnerProductPredictLabel(float inner_product) const override {
    return inner_product;
  }
};

// Poisson regression, where the distribution mean is the inner product between
// an instance and parameters.
class PoissonRegressionLossFunction : public InnerProductLossFunction {
 public:
  PoissonRegressionLossFunction() {}

  // Returns the Poisson regression loss.
  float InnerProductExampleLoss(float inner_product, float label)
      const override {
    return exp(inner_product) - label * inner_product;
  }

  // Returns the gradient of the Poisson regression loss wrt 'inner_product'.
  float InnerProductExampleGradient(float inner_product, float label)
      const override {
    return exp(inner_product) - label;
  }

  // Returns the mode of a Poisson distribution with mean 'inner_product'.
  float InnerProductPredictLabel(float inner_product) const override {
    return floor(inner_product);
  }

  // Returns the magnitude of the second derivative of the loss, evaluated at
  // 'inner_product' and 'label'.
  float InnerProductCurvature(float inner_product, float label)
      const override {
    return exp(inner_product);
  }
};

// Binary logistic regression where 'label' = P(true_label = 1) is in [0, 1],
// and the loss is averaged over labels:
// label * logloss(inner_product, 1) + (1 - label) * logloss(inner_product, 0).
class AveragedLogisticLossFunction : public InnerProductLossFunction {
 public:
  AveragedLogisticLossFunction() { set_curvature(0.25); }

  // Returns the averaged logistic loss.
  float InnerProductExampleLoss(float inner_product, float label)
      const override {
    return (1.0 - label) * log1pf(exp(inner_product))
        + label * log1pf(exp(-inner_product));
  }

  // Returns the gradient of the averaged logistic loss wrt 'inner_product'.
  float InnerProductExampleGradient(float inner_product, float label)
      const override {
    if (-inner_product < kInputMin) return -label + 1.0f;
    return -label + 1.0f / (1.0f + exp(-inner_product));
  }

  // Returns the probability that the label is 1 given the inner product.
  float InnerProductPredictLabel(float inner_product) const override {
    if (-inner_product < kInputMin) return 1.0f;
    return 1.0f / (1.0f + exp(-inner_product));
  }

  // Returns the magnitude of the second derivative of the loss, evaluated at
  // 'inner_product' and 'label'.
  float InnerProductCurvature(float inner_product, float label)
      const override {
    float label_prob = 1.0f / (1.0f + exp(-inner_product));
    if (-inner_product < kInputMin) return 0.0f;
    return label_prob * (1.0f - label_prob);
  }

  // Min allowed exp input.
  const float kInputMin = log(FLT_EPSILON);
};


// Smooth hinge loss. If a = inner_product * label, the loss is
// loss = 0.0                                   a > threshold
//        0.5 * (a - threshold)^2               0 < a <= threshold
//        0.5 * threshold^2 - threshold * a     a <= 0
class SmoothHingeLossFunction : public InnerProductLossFunction {
 public:
  SmoothHingeLossFunction() { set_curvature(1.0f); }

  /*
  void Setup(const LossFunctionConfigContainer &config) override {
    if (config.has_smooth_hinge_threshold()) {
      threshold_ = config.smooth_hinge_threshold();
    }
    set_curvature(threshold_);
  }
  */

  float InnerProductExampleLoss(float inner_product, float label)
      const override {
    float hinge_input = inner_product *= label;
    if (hinge_input >= threshold_) {
      return 0.0;
    } else if (hinge_input <= 0.0f) {
      return (0.5 * threshold_ * threshold_ - threshold_ * hinge_input);
    } else {
      return 0.5 * (hinge_input - threshold_) * (hinge_input - threshold_);
    }
  }

  float InnerProductExampleGradient(float inner_product, float label)
      const override {
    float hinge_input = inner_product * label;
    if (hinge_input >= threshold_) {
      return 0.0f;
    } else if (hinge_input <= 0.0f) {
      return -label * threshold_;
    } else {
      return (hinge_input - threshold_) * label;
    }
  }

  // Assigns a label in {-1, 1} given 'inner_product'.
  float InnerProductPredictLabel(float inner_product) const override {
    if (inner_product > 0) return 1.0f;
    return -1.0f;
  }

  void set_threshold(float threshold) { threshold_ = threshold; }

 private:
  float threshold_ = 1.0f;
};


}  // namespace lossmin