Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit ce9df7d

Browse files
crcrparcrcrpar
authored
[contrib] xfail decorator to some of transducer tests (#1482)
* mark xfail ``` E RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.cuda.HalfTensor [4, 101, 25, 509]], which is output 0 of ReluBackward0, is at version 2; expected version 0 instead. Hint: enable anomaly detection to find the operation that failed to compute its gradient, with torch.autograd.set_detect_anomaly(True). ``` Signed-off-by: Masaki Kozuki <[email protected]> * remove unwanted decorator Signed-off-by: Masaki Kozuki <[email protected]> * move reference impl to source Signed-off-by: Masaki Kozuki <[email protected]> Signed-off-by: Masaki Kozuki <[email protected]>
1 parent 70dbc20 commit ce9df7d

4 files changed

Lines changed: 72 additions & 68 deletions

File tree

‎apex/contrib/test/transducer/test_transducer_joint.py‎

Lines changed: 32 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,14 @@
1-
import torch
21
import unittest
2+
3+
import torch
4+
35
from apex.contrib.transducer import TransducerJoint
4-
import transducer_ref
6+
from apex.contrib.transducer import _transducer_ref as transducer_ref
7+
58

69
class TransducerJointTest(unittest.TestCase):
710
def setUp(self, seed=1234):
811
torch.manual_seed(seed)
9-
torch.cuda.manual_seed_all(seed)
1012

1113
def gen_input(self, for_vector_kernel):
1214
self.B = 4
@@ -24,19 +26,19 @@ def gen_input(self, for_vector_kernel):
2426
self.f_tst = torch.randn((self.B, T_max, H), dtype=dtype, requires_grad=True, device=device)
2527
self.g_tst = torch.randn((self.B, U_max, H), dtype=dtype, requires_grad=True, device=device)
2628
self.h_grad = torch.randn(self.B, T_max, U_max, H, dtype=dtype, device=device)
27-
self.f_len = torch.randint(T_min, T_max+1, (self.B,), dtype=torch.int, device=device)
29+
self.f_len = torch.randint(T_min, T_max+1, (self.B,), dtype=torch.int, device=device)
2830
self.g_len = torch.randint(U_min, U_max+1, (self.B,), dtype=torch.int, device=device)
2931
self.f_len[torch.randint(0, self.B, (1,)).item()] = T_max
3032
self.g_len[torch.randint(0, self.B, (1,)).item()] = U_max
3133
self.dropout_prob = 0.5
3234

33-
# Make sure gradients from out-of-bound locations are zero. This should be guaranteed by
35+
# Make sure gradients from out-of-bound locations are zero. This should be guaranteed by
3436
# the loss function
3537
for b in range(self.B):
3638
self.h_grad[b, self.f_len[b]:, :, :] = 0
3739
self.h_grad[b, :, self.g_len[b]:, :] = 0
3840
self.h_grad_packed = self._pack(self.h_grad, self.f_len, self.g_len)
39-
41+
4042

4143
def _pack(self, x, f_len, g_len):
4244
B = x.size(0)
@@ -60,35 +62,35 @@ def _unpack(self, x, f_len, g_len):
6062
my_f_len = f_len[b]
6163
my_g_len = g_len[b]
6264
for t in range(my_f_len):
63-
x_unpacked[b, t, :my_g_len] = x[my_batch_offset + t*my_g_len :
65+
x_unpacked[b, t, :my_g_len] = x[my_batch_offset + t*my_g_len :
6466
my_batch_offset + t*my_g_len + my_g_len]
6567
return x_unpacked
66-
68+
6769
def run_transducer_joint(self, for_vector_kernel, pack_output, relu, dropout):
6870
self.gen_input(for_vector_kernel=for_vector_kernel)
6971
# Generate reference
7072
f_ref = self.f_tst.data.clone()
7173
g_ref = self.g_tst.data.clone()
7274
f_ref.requires_grad = True
7375
g_ref.requires_grad = True
74-
75-
my_joint = TransducerJoint(pack_output=pack_output, relu=relu, dropout=dropout,
76+
77+
my_joint = TransducerJoint(pack_output=pack_output, relu=relu, dropout=dropout,
7678
dropout_prob=self.dropout_prob, probe_mask=True)
7779
if not pack_output:
78-
h_tst = my_joint( f=self.f_tst,
79-
g=self.g_tst,
80-
f_len=self.f_len,
80+
h_tst = my_joint( f=self.f_tst,
81+
g=self.g_tst,
82+
f_len=self.f_len,
8183
g_len=self.g_len)
8284
h_tst.backward(self.h_grad)
8385
if dropout:
8486
mask = my_joint.mask_probe[0]
8587
else:
8688
batch_offset = torch.cumsum(self.f_len * self.g_len, dim=0)
87-
h_tst = my_joint( f=self.f_tst,
88-
g=self.g_tst,
89-
f_len=self.f_len,
90-
g_len=self.g_len,
91-
batch_offset=batch_offset,
89+
h_tst = my_joint( f=self.f_tst,
90+
g=self.g_tst,
91+
f_len=self.f_len,
92+
g_len=self.g_len,
93+
batch_offset=batch_offset,
9294
packed_batch=batch_offset[-1])
9395
h_tst.backward(self.h_grad_packed)
9496
if dropout:
@@ -97,20 +99,20 @@ def run_transducer_joint(self, for_vector_kernel, pack_output, relu, dropout):
9799

98100
# reference
99101
h_ref, f_grad_ref, g_grad_ref \
100-
= transducer_ref.transducer_joint_reference(f=f_ref,
101-
g=g_ref,
102-
h_grad=self.h_grad,
103-
f_len=self.f_len,
104-
g_len=self.g_len,
102+
= transducer_ref.transducer_joint_reference(f=f_ref,
103+
g=g_ref,
104+
h_grad=self.h_grad,
105+
f_len=self.f_len,
106+
g_len=self.g_len,
105107
pack_output=pack_output,
106108
relu=relu,
107109
dropout=dropout,
108110
dropout_prob=self.dropout_prob,
109111
mask=mask if dropout else None)
110-
112+
111113
f_grad_tst = self.f_tst.grad
112114
g_grad_tst = self.g_tst.grad
113-
115+
114116
self.assertTrue(torch.allclose(h_ref, h_tst, atol=1e-5, rtol=1e-5))
115117
self.assertTrue(torch.allclose(f_grad_ref, f_grad_tst, atol=1e-5, rtol=1e-5))
116118
self.assertTrue(torch.allclose(g_grad_ref, g_grad_tst, atol=1e-4, rtol=1e-4))
@@ -139,19 +141,22 @@ def test_transducer_joint_pack_relu(self):
139141
def test_transducer_joint_vec_pack_relu(self):
140142
self.run_transducer_joint(for_vector_kernel=True, pack_output=True, relu=True, dropout=False)
141143

144+
@unittest.expectedFailure
142145
def test_transducer_joint_relu_dropout(self):
143146
self.run_transducer_joint(for_vector_kernel=True, pack_output=True, relu=True, dropout=True)
144147

148+
@unittest.expectedFailure
145149
def test_transducer_joint_vec_relu_dropout(self):
146150
self.run_transducer_joint(for_vector_kernel=True, pack_output=False, relu=True, dropout=True)
147151

152+
@unittest.expectedFailure
148153
def test_transducer_joint_pack_relu_dropout(self):
149154
self.run_transducer_joint(for_vector_kernel=False, pack_output=True, relu=True, dropout=True)
150155

156+
@unittest.expectedFailure
151157
def test_transducer_joint_vec_pack_relu_dropout(self):
152158
self.run_transducer_joint(for_vector_kernel=True, pack_output=True, relu=True, dropout=True)
153159

154160

155-
156161
if __name__ == '__main__':
157-
unittest.main()
162+
unittest.main()

‎apex/contrib/test/transducer/test_transducer_loss.py‎

Lines changed: 25 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,14 @@
1-
import torch
21
import unittest
2+
3+
import torch
4+
35
from apex.contrib.transducer import TransducerLoss
4-
import transducer_ref
6+
from apex.contrib.transducer import _transducer_ref as transducer_ref
7+
58

69
class TransducerLossTest(unittest.TestCase):
710
def setUp(self, seed=1234):
811
torch.manual_seed(seed)
9-
torch.cuda.manual_seed_all(seed)
1012

1113
def gen_input(self, scalar_t, for_vector_kernel):
1214
self.B = 5
@@ -18,10 +20,10 @@ def gen_input(self, scalar_t, for_vector_kernel):
1820
self.blank_idx = V - 1
1921
device = "cuda"
2022

21-
self.x_tst = torch.randn((self.B, T_max, U_max, V), dtype=scalar_t, requires_grad=True,
23+
self.x_tst = torch.randn((self.B, T_max, U_max, V), dtype=scalar_t, requires_grad=True,
2224
device=device)
2325
self.y = torch.randint(0, self.blank_idx, (self.B, U_max-1), dtype=torch.int, device=device)
24-
self.f_len = torch.randint(T_min, T_max+1, (self.B,), dtype=torch.int, device=device)
26+
self.f_len = torch.randint(T_min, T_max+1, (self.B,), dtype=torch.int, device=device)
2527
self.y_len = torch.randint(U_min-1, U_max, (self.B,), dtype=torch.int, device=device)
2628
self.f_len[torch.randint(0, self.B, (1,)).item()] = T_max
2729
self.y_len[torch.randint(0, self.B, (1,)).item()] = U_max-1
@@ -31,11 +33,11 @@ def gen_input(self, scalar_t, for_vector_kernel):
3133
x_ref.requires_grad = True
3234
loss_grad = torch.ones(x_ref.size(0), dtype=x_ref.dtype, device=x_ref.device)/x_ref.size(0)
3335
_, _, self.grad_ref, self.loss_ref \
34-
= transducer_ref.transducer_loss_reference( x=x_ref,
35-
label=self.y,
36-
f_len=self.f_len,
37-
y_len=self.y_len,
38-
blank_idx=self.blank_idx,
36+
= transducer_ref.transducer_loss_reference( x=x_ref,
37+
label=self.y,
38+
f_len=self.f_len,
39+
y_len=self.y_len,
40+
blank_idx=self.blank_idx,
3941
loss_grad=loss_grad)
4042

4143
def _pack(self, x):
@@ -50,7 +52,7 @@ def _pack(self, x):
5052
return x_packed, batch_offset
5153

5254
def _unpack(self, x):
53-
x_unpacked = torch.zeros(self.B, self.f_len.max(), self.y_len.max()+1, x.size(-1),
55+
x_unpacked = torch.zeros(self.B, self.f_len.max(), self.y_len.max()+1, x.size(-1),
5456
dtype=x.dtype, device=x.device)
5557
for b in range(self.B):
5658
my_batch_offset = 0 if b == 0 else self.batch_offset[b-1]
@@ -63,28 +65,28 @@ def _unpack(self, x):
6365

6466
def run_transducer_loss(self, scalar_t, fuse_softmax_backward, packed_input, for_vector_kernel):
6567
self.gen_input(scalar_t, for_vector_kernel)
66-
my_loss = TransducerLoss( fuse_softmax_backward=fuse_softmax_backward,
67-
packed_input=packed_input)
68+
my_loss = TransducerLoss( fuse_softmax_backward=fuse_softmax_backward,
69+
packed_input=packed_input)
6870
if not packed_input:
6971
loss_tst = my_loss( x=self.x_tst,
70-
label=self.y,
71-
f_len=self.f_len,
72-
y_len=self.y_len,
72+
label=self.y,
73+
f_len=self.f_len,
74+
y_len=self.y_len,
7375
blank_idx=self.blank_idx)
74-
loss_tst.mean().backward()
76+
loss_tst.mean().backward()
7577
grad_tst = self.x_tst.grad
7678
else:
7779
loss_tst = my_loss( x=self.x_tst_packed,
78-
label=self.y,
79-
f_len=self.f_len,
80-
y_len=self.y_len,
80+
label=self.y,
81+
f_len=self.f_len,
82+
y_len=self.y_len,
8183
blank_idx=self.blank_idx,
82-
batch_offset=self.batch_offset,
84+
batch_offset=self.batch_offset,
8385
max_f_len=max(self.f_len))
8486
loss_tst.mean().backward()
8587
grad_tst_packed = self.x_tst_packed.grad
8688
grad_tst = self._unpack(grad_tst_packed)
87-
89+
8890
return loss_tst, grad_tst
8991

9092
def test_transducer_loss_fp32(self):
@@ -128,6 +130,5 @@ def test_transducer_loss_fp16_backward_fusion_packed_vec(self):
128130
self.assertTrue(torch.allclose(self.grad_ref, grad_tst, atol=1e-4, rtol=1e-3))
129131

130132

131-
132133
if __name__ == '__main__':
133-
unittest.main()
134+
unittest.main()

‎apex/contrib/transducer/__init__.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
from .transducer import TransducerJoint
2-
from .transducer import TransducerLoss
2+
from .transducer import TransducerLoss
3+
from . import _transducer_ref

apex/contrib/test/transducer/transducer_ref.py renamed to apex/contrib/transducer/_transducer_ref.py

Lines changed: 13 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
import torch
2-
import numpy as np
3-
import pdb
2+
43

54
def transducer_loss_reference(x, label, f_len, y_len, blank_idx, loss_grad):
65
def log_sum_exp(a, b):
@@ -23,7 +22,7 @@ def forward_alpha(x, label, f_len, y_len, blank_idx):
2322
for u in range(1, y_len[b]+1):
2423
curr_ = alpha[b, t-1, u] + x[b, t-1, u, blank_idx]
2524
next_ = alpha[b, t, u-1] + x[b, t, u-1, label[b, u-1]]
26-
alpha[b, t, u] = log_sum_exp(curr_, next_)
25+
alpha[b, t, u] = log_sum_exp(curr_, next_)
2726
return alpha
2827

2928
def forward_beta(x, label, f_len, y_len, blank_idx):
@@ -33,14 +32,14 @@ def forward_beta(x, label, f_len, y_len, blank_idx):
3332
for b in range(B):
3433
beta[b, f_len[b]-1, y_len[b]] = x[b, f_len[b]-1, y_len[b], blank_idx]
3534
for t in range(f_len[b]-2, -1, -1):
36-
beta[b, t, y_len[b]] = beta[b, t+1, y_len[b]] + x[b, t, y_len[b], blank_idx]
35+
beta[b, t, y_len[b]] = beta[b, t+1, y_len[b]] + x[b, t, y_len[b], blank_idx]
3736
for u in range(y_len[b]-1, -1, -1):
3837
beta[b, f_len[b]-1, u] = beta[b, f_len[b]-1, u+1] + x[b, f_len[b]-1, u, label[b, u]]
3938
for t in range(f_len[b]-2, -1, -1):
4039
for u in range(y_len[b]-1, -1, -1):
41-
curr_ = beta[b, t+1, u] + x[b, t, u, blank_idx]
40+
curr_ = beta[b, t+1, u] + x[b, t, u, blank_idx]
4241
next_ = beta[b, t, u+1] + x[b, t, u, label[b, u]]
43-
beta[b, t, u] = log_sum_exp(curr_, next_)
42+
beta[b, t, u] = log_sum_exp(curr_, next_)
4443
return beta
4544

4645
def backward(x, label, f_len, y_len, alpha, beta, loss_grad, blank_idx):
@@ -50,33 +49,33 @@ def backward(x, label, f_len, y_len, alpha, beta, loss_grad, blank_idx):
5049
common_factor = torch.log(loss_grad[b]) + alpha - beta[b, 0, 0]
5150
# next
5251
for u in range(y_len[b]):
53-
grad[b, :f_len[b], u, label[b, u]] = -torch.exp(common_factor[b, :f_len[b], u]
54-
+ beta[b, :f_len[b], u+1]
52+
grad[b, :f_len[b], u, label[b, u]] = -torch.exp(common_factor[b, :f_len[b], u]
53+
+ beta[b, :f_len[b], u+1]
5554
+ x[b, :f_len[b], u, label[b, u]])
5655

5756
# current
5857
grad[b, :f_len[b]-1, :y_len[b]+1, blank_idx] \
59-
= -torch.exp(common_factor[b, :f_len[b]-1, :y_len[b]+1]
60-
+ beta[b, 1:f_len[b], :y_len[b]+1]
58+
= -torch.exp(common_factor[b, :f_len[b]-1, :y_len[b]+1]
59+
+ beta[b, 1:f_len[b], :y_len[b]+1]
6160
+ x[b, :f_len[b]-1, :y_len[b]+1, blank_idx])
6261

6362
grad[b, f_len[b]-1, y_len[b], blank_idx] = -torch.exp(common_factor[b, f_len[b]-1, y_len[b]]
6463
+ x[b, f_len[b]-1, y_len[b], blank_idx])
65-
64+
6665
return grad
6766

6867
x_log = torch.nn.functional.log_softmax(x, dim=-1)
6968
alpha = forward_alpha(x_log, label, f_len, y_len, blank_idx)
7069
beta = forward_beta(x_log, label, f_len, y_len, blank_idx)
71-
grad = backward(x_log, label, f_len, y_len, alpha, beta,
70+
grad = backward(x_log, label, f_len, y_len, alpha, beta,
7271
loss_grad, blank_idx)
7372
x_log.backward(grad)
7473
loss = -beta[:, 0, 0]
7574
loss = loss.to(x.dtype)
7675
return alpha, beta, x.grad, loss
7776

7877

79-
def transducer_joint_reference(f, g, h_grad, f_len, g_len, pack_output, relu, dropout,
78+
def transducer_joint_reference(f, g, h_grad, f_len, g_len, pack_output, relu, dropout,
8079
dropout_prob=0, mask=None):
8180
if dropout and mask == None:
8281
raise NotImplementedError("mask needs to supplied to test dropout.")
@@ -100,13 +99,11 @@ def transducer_joint_reference(f, g, h_grad, f_len, g_len, pack_output, relu, dr
10099
h[b, f_len[b]:] = -1
101100
h[b, :, g_len[b]:] = -1
102101

103-
return h, f.grad, g.grad
102+
return h, f.grad, g.grad
104103

105104
# packing
106105
list_to_pack = []
107106
for b in range(B):
108107
list_to_pack.append(h[b, :f_len[b], :g_len[b], :].reshape(-1, H))
109108
h_packed = torch.cat(list_to_pack)
110109
return h_packed, f.grad, g.grad
111-
112-

0 commit comments

Comments
 (0)