fix

2 months ago · 5ecc27e150
parent f98384aef6
commit 5ecc27e150
1 changed files with 0 additions and 1 deletions
--- a/colossalai/shardformer/layer/attn.py
+++ b/colossalai/shardformer/layer/attn.py
@ -479,7 +479,6 @@ class RingAttention(torch.autograd.Function):
        num_inner_group = num_ring_size // inner_ring_size
        if tp_size > 1:
            # inner_ring_size = 2
            for i in range(num_rings):
                for j in range(num_inner_group):
                    # find inner ring group in one sp groups