Added reset to L-BFGS if no progress was made.

tensorflow · DirkToewe · Nov 8, 2018 · Nov 8, 2018 · Nov 9, 2018 · Nov 9, 2018
commit 519df5260c31f5381206ee98ff7012a5c3fa060b
diff --git a/src/optimizers/lbfgs_function_optimizer.ts b/src/optimizers/lbfgs_function_optimizer.ts
@@ -45,6 +45,18 @@ function dotProd( x: Tensor1D, y: Tensor1D ) {
   return z;
 }
 
+export class LineSearchError extends Error {
+  constructor( msg: string ) {
+    super(msg);
+  }
+}
+
+export class LineSearchNoProgressError extends LineSearchError {
+  constructor( msg: string ) {
+    super(msg);
+  }
+}
+
 /** The function type of a linesearch method.
  *
  *  @param fg A function that returns both the value and gradients of the
@@ -151,7 +163,7 @@ export function strongWolfeLineSearch(
       }
 
       if( ! (α < αMax) ) {
-        throw new Error(
+        throw new LineSearchError(
             'strongWolfeLineSearch(): '
           + 'Strong Wolfe condition not satisfiable in range.'
         );
@@ -162,7 +174,7 @@ export function strongWolfeLineSearch(
     }
 
     if( αMin === αMax ) {
-      throw new Error('strongWolfeLineSearch: bracketing failed.');
+      throw new LineSearchError('strongWolfeLineSearch: bracketing failed.');
     }
 
     // STEP 2: BISECTION PHASE
@@ -181,7 +193,9 @@ export function strongWolfeLineSearch(
 
       if( f - f0 > c1*α*p0 || f >= fMin ) {
         if( αMax === α ) {
-          throw new Error('strongWolfeLineSearch(): bisection failed.');
+          throw new LineSearchError(
+            'strongWolfeLineSearch(): bisection failed.'
+          );
         }
         αMax = α;
       }
@@ -198,14 +212,20 @@ export function strongWolfeLineSearch(
         }
 
         if( αMin === α ) {
-          throw new Error('strongWolfeLineSearch(): bisection failed.');
+          throw new LineSearchError(
+            'strongWolfeLineSearch(): bisection failed.'
+          );
         }
         αMin = α;
         fMin = f;
       }
 
       if( αMin === αMax ) {
-        throw new Error('strongWolfeLineSearch(): bisection failed.');
+        const msg = 'strongWolfeLineSearch(): bisection failed.';
+        if( αMin === 0) {
+          throw new LineSearchNoProgressError(msg);
+        }
+        throw new LineSearchError(msg);
       }
     }
   };
@@ -396,15 +416,33 @@ export class LBFGSFunctionOptimizer {
         dGdX = this.dGdX,
         dG   = this.dG;
 
-    const [x,f,g] = ENV.engine.tidy(
-      () => this.lineSearch(
+    const [x,f,g] = ( () => {
+      const stepFunc = () => this.lineSearch(
         this.fg,
         this.x,
         this.f,
         this.g,
         this.negDir(this.g)
-      )
-    );
+      );
+      try {
+        return ENV.engine.tidy(stepFunc);
+      }
+      catch( err ) {
+        if( err instanceof LineSearchNoProgressError ) {
+          // reset line search
+          while( dX.length > 0 ) {
+              dX.pop().dispose();
+            dGdX.pop().dispose();
+            dG  .pop().dispose();
+          }
+          // try one last time
+          return ENV.engine.tidy(stepFunc);
+        }
+        else {
+          throw err;
+        }
+      }
+    })();
 
     const dXi = sub(x,this.x) as Tensor1D,
           dGi = sub(g,this.g) as Tensor1D;

diff --git a/src/optimizers/lbfgs_function_optimizer_test.ts b/src/optimizers/lbfgs_function_optimizer_test.ts
@@ -26,7 +26,9 @@ import {convertToTensor} from '../tensor_util_env';
 import {valueAndGrad} from '../gradients';
 import {ENV} from '../environment';
 import {randomUniform} from '../ops/array_ops';
-import {strongWolfeLineSearch, LBFGSFunctionOptimizer} from './lbfgs_function_optimizer';
+import {strongWolfeLineSearch,
+        LBFGSFunctionOptimizer,
+        LineSearchNoProgressError} from './lbfgs_function_optimizer';
 import {dot} from '../ops/matmul';
 
 function rosenbrock( x: Tensor|TensorLike ): Tensor
@@ -257,10 +259,19 @@ describeWithFlags('lbfgs', ALL_ENVS, () => {
         { initNegDir: g => g.div(DENOM) }
       );
 
-      while( ! opt.g.abs().lessEqual( scalar(2**-12) ).all().dataSync()[0] )
+      const atol = scalar( Math.sqrt(ENV.get('EPSILON')) );
+      while( ! opt.g.abs().lessEqual(atol).all().dataSync()[0] )
       {
         ++nSteps;
-        opt.step();
+        try {
+          opt.step();
+        }
+        catch(err) {
+          if( err instanceof LineSearchNoProgressError ) {
+            break;
+          }
+          throw err;
+        }
       }
 
       expect(nCalls).toBeLessThan(256);