CalvinBackup/iperov-DeepFaceLab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix for clipgrad

Browse Source
This commit is contained in:
iperov
2021-07-15 00:24:38 +04:00
parent 35877dbfd7
commit 2edac3df8c
2 changed files with 4 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
core/leras/optimizers/AdaBelief.py
+2 -2
View File
@@ -50,11 +50,11 @@ class AdaBelief(nn.OptimizerBase):
updates = [] updates = []
if self.clipnorm > 0.0: if self.clipnorm > 0.0:
norm = tf.sqrt( sum([tf.reduce_sum(tf.square(g)) for g,v in grads_vars])) norm = tf.sqrt( sum([tf.reduce_sum(tf.square(tf.cast(g, tf.float32))) for g,v in grads_vars]))
updates += [ state_ops.assign_add( self.iterations, 1) ] updates += [ state_ops.assign_add( self.iterations, 1) ]
for i, (g,v) in enumerate(grads_vars): for i, (g,v) in enumerate(grads_vars):
if self.clipnorm > 0.0: if self.clipnorm > 0.0:
g = self.tf_clip_norm(g, self.clipnorm, norm) g = self.tf_clip_norm(g, self.clipnorm, tf.cast(norm, g.dtype) )
ms = self.ms_dict[ v.name ] ms = self.ms_dict[ v.name ]
vs = self.vs_dict[ v.name ] vs = self.vs_dict[ v.name ]
core/leras/optimizers/RMSprop.py
+2 -2
View File
@@ -47,11 +47,11 @@ class RMSprop(nn.OptimizerBase):
updates = [] updates = []
if self.clipnorm > 0.0: if self.clipnorm > 0.0:
norm = tf.sqrt( sum([tf.reduce_sum(tf.square(g)) for g,v in grads_vars])) norm = tf.sqrt( sum([tf.reduce_sum(tf.square(tf.cast(g, tf.float32))) for g,v in grads_vars]))
updates += [ state_ops.assign_add( self.iterations, 1) ] updates += [ state_ops.assign_add( self.iterations, 1) ]
for i, (g,v) in enumerate(grads_vars): for i, (g,v) in enumerate(grads_vars):
if self.clipnorm > 0.0: if self.clipnorm > 0.0:
g = self.tf_clip_norm(g, self.clipnorm, norm) g = self.tf_clip_norm(g, self.clipnorm, tf.cast(norm, g.dtype) )
a = self.accumulators_dict[ v.name ] a = self.accumulators_dict[ v.name ]