lossmin/minimizers/loss-minimizer.cc

// Copyright 2014 Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// Author: nevena@google.com (Nevena Lazic)

#include "lossmin/minimizers/loss-minimizer.h"

#include <algorithm>
#include <random>

namespace lossmin {

bool LossMinimizer::Run(int max_epochs, int loss_epochs, int convergence_epochs,
                        Weights *weights, std::vector<float> *loss) {
  // Run for up to 'max_epochs' epochs.
  int epoch;
  for (epoch = 0; epoch < max_epochs; ++epoch) {
    // Compute the loss.
    if (epoch % loss_epochs == 0) {
      loss->push_back(Loss(*weights));
      // LOG(INFO) << epoch << ": " << loss->at(loss->size() - 1);
    }

    // Set the 'check_convergence' flag.
    bool check_convergence = (epoch > 0) && (epoch % convergence_epochs == 0);

    // Run for one epoch to update the parameters.
    EpochUpdate(weights, epoch, check_convergence);

    // We should also periodically check if the algorithm has not stopped;
    // numerical problems can be encountered if
    // the convergence is checked only by CheckConvergence():
    // if the algorithm "stops" for any reason before solving the problem
    // up to the given accuracy.
    if (check_convergence) {
      SimpleConvergenceCheck(*loss);
    }

    // Check the convergence flag.
    if (converged_) {
      break;
    }
  }

  loss->push_back(Loss(*weights));
  // LOG(INFO) << "final loss: " << loss->at(loss->size() - 1);
  num_epochs_run_ = std::min(epoch + 1, max_epochs);
  return converged_;
}

// By default only check if gradient is == 0
void LossMinimizer::ConvergenceCheck(const Weights &weights,
                                     const Weights &gradient) {
  if (gradient.squaredNorm() / weights.size() < convergence_threshold_) {
    set_reached_solution(true);
    set_converged(true);
  }
}

void LossMinimizer::SimpleConvergenceCheck(const std::vector<float> &loss) {
  // Check convergence by verifying that the max relative loss decrease
  // (loss[t-1] - loss[t]) / loss[t-1] is below 'simple_convergence_threshold_'.
  if (loss.size() > num_convergence_epochs_) {
    float loss_difference = 0.0f;
    for (int i = loss.size() - num_convergence_epochs_; i < loss.size(); ++i) {
      if (loss[i - 1] > 0) {
        loss_difference = std::max(loss_difference, 1 - loss[i] / loss[i - 1]);
      } else {
        set_reached_solution(true);
        set_converged(true);
      }
    }
    if (loss_difference < simple_convergence_threshold_) set_converged(true);
  }
}

}  // namespace lossmin