CalvinBackup/iperov-DeepFaceLab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix for clipgrad

Browse Source
This commit is contained in:
iperov
2021-07-15 00:24:38 +04:00
parent 35877dbfd7
commit 2edac3df8c
2 changed files with 4 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
core/leras/optimizers/AdaBelief.py
View File

@@ -50,11 +50,11 @@ class AdaBelief(nn.OptimizerBase):
updates = []
if self.clipnorm > 0.0:
norm = tf.sqrt( sum([tf.reduce_sum(tf.square(g)) for g,v in grads_vars]))
norm = tf.sqrt( sum([tf.reduce_sum(tf.square(tf.cast(g, tf.float32))) for g,v in grads_vars]))
updates += [ state_ops.assign_add( self.iterations, 1) ]
for i, (g,v) in enumerate(grads_vars):
if self.clipnorm > 0.0:
g = self.tf_clip_norm(g, self.clipnorm, norm)
g = self.tf_clip_norm(g, self.clipnorm, tf.cast(norm, g.dtype) )
ms = self.ms_dict[ v.name ]
vs = self.vs_dict[ v.name ]

4
core/leras/optimizers/RMSprop.py
View File

@@ -47,11 +47,11 @@ class RMSprop(nn.OptimizerBase):
updates = []
if self.clipnorm > 0.0:
norm = tf.sqrt( sum([tf.reduce_sum(tf.square(g)) for g,v in grads_vars]))
norm = tf.sqrt( sum([tf.reduce_sum(tf.square(tf.cast(g, tf.float32))) for g,v in grads_vars]))
updates += [ state_ops.assign_add( self.iterations, 1) ]
for i, (g,v) in enumerate(grads_vars):
if self.clipnorm > 0.0:
g = self.tf_clip_norm(g, self.clipnorm, norm)
g = self.tf_clip_norm(g, self.clipnorm, tf.cast(norm, g.dtype) )
a = self.accumulators_dict[ v.name ]