summaryrefslogtreecommitdiff
path: root/training/optimize.h
blob: eddceaadb13a116187b87c168865b087246a2ddf (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
#ifndef _OPTIMIZE_H_
#define _OPTIMIZE_H_

#include <iostream>
#include <vector>
#include <string>
#include <cassert>

#include "lbfgs.h"

// abstract base class for first order optimizers
// order of invocation: new, Load(), Optimize(), Save(), delete
class Optimizer {
 public:
  Optimizer() : eval_(1), has_converged_(false) {}
  virtual ~Optimizer();
  virtual std::string Name() const = 0;
  int EvaluationCount() const { return eval_; }
  bool HasConverged() const { return has_converged_; }

  void Optimize(const double& obj,
                const std::vector<double>& g,
                std::vector<double>* x) {
    assert(g.size() == x->size());
    OptimizeImpl(obj, g, x);
    scitbx::lbfgs::traditional_convergence_test<double> converged(g.size());
    has_converged_ = converged(&(*x)[0], &g[0]);
  }

  void Save(std::ostream* out) const;
  void Load(std::istream* in);
 protected:
  virtual void SaveImpl(std::ostream* out) const;
  virtual void LoadImpl(std::istream* in);
  virtual void OptimizeImpl(const double& obj,
                            const std::vector<double>& g,
                            std::vector<double>* x) = 0;

  int eval_;
 private:
  bool has_converged_;
};

class RPropOptimizer : public Optimizer {
 public:
  explicit RPropOptimizer(int num_vars,
                          double eta_plus = 1.2,
                          double eta_minus = 0.5,
                          double delta_0 = 0.1,
                          double delta_max = 50.0,
                          double delta_min = 1e-6) :
      prev_g_(num_vars, 0.0),
      delta_ij_(num_vars, delta_0),
      eta_plus_(eta_plus),
      eta_minus_(eta_minus),
      delta_max_(delta_max),
      delta_min_(delta_min) {
    assert(eta_plus > 1.0);
    assert(eta_minus > 0.0 && eta_minus < 1.0);
    assert(delta_max > 0.0);
    assert(delta_min > 0.0);
  }
  std::string Name() const;
  void OptimizeImpl(const double& obj,
                    const std::vector<double>& g,
                    std::vector<double>* x);
  void SaveImpl(std::ostream* out) const;
  void LoadImpl(std::istream* in);
 private:
  std::vector<double> prev_g_;
  std::vector<double> delta_ij_;
  const double eta_plus_;
  const double eta_minus_;
  const double delta_max_;
  const double delta_min_;
};

class SGDOptimizer : public Optimizer {
 public:
  explicit SGDOptimizer(int num_vars, double eta = 0.1) : eta_(eta) {
    (void) num_vars;
  }
  std::string Name() const;
  void OptimizeImpl(const double& obj,
                    const std::vector<double>& g,
                    std::vector<double>* x);
 private:
  const double eta_;
};

class LBFGSOptimizer : public Optimizer {
 public:
  explicit LBFGSOptimizer(int num_vars, int memory_buffers = 10);
  std::string Name() const;
  void SaveImpl(std::ostream* out) const;
  void LoadImpl(std::istream* in);
  void OptimizeImpl(const double& obj,
                    const std::vector<double>& g,
                    std::vector<double>* x);
 private:
  scitbx::lbfgs::minimizer<double> opt_;
};

#endif