Fix deadlock in DataChannel with mutex unlock

Fulfill request changes in PR 3009

Close answer and assert errors in detach tests

Author: pabloFuente

datachannel.go

@@ -464,10 +464,12 @@ func (d *DataChannel) DetachWithDeadline() (datachannel.ReadWriteCloserDeadliner
 	d.mu.Lock()
 
 	if !d.api.settingEngine.detach.DataChannels {
+		d.mu.Unlock()
 		return nil, errDetachNotEnabled
 	}
 
 	if d.dataChannel == nil {
+		d.mu.Unlock()
 		return nil, errDetachBeforeOpened
 	}
 

datachannel_go_test.go

@@ -822,3 +822,30 @@ func TestDataChannelClose(t *testing.T) {
 		assert.NoError(t, answerPC.Close())
 	})
 }
+
+func TestDataChannel_DetachErrors(t *testing.T) {
+	t.Run("error errDetachNotEnabled", func(t *testing.T) {
+		s := SettingEngine{}
+		offer, answer, err := NewAPI(WithSettingEngine(s)).newPair(Configuration{})
+		assert.NoError(t, err)
+		dc, err := offer.CreateDataChannel("data", nil)
+		assert.NoError(t, err)
+		_, err = dc.Detach()
+		assert.ErrorIs(t, err, errDetachNotEnabled)
+		assert.NoError(t, offer.Close())
+		assert.NoError(t, answer.Close())
+	})
+
+	t.Run("error errDetachBeforeOpened", func(t *testing.T) {
+		s := SettingEngine{}
+		s.DetachDataChannels()
+		offer, answer, err := NewAPI(WithSettingEngine(s)).newPair(Configuration{})
+		assert.NoError(t, err)
+		dc, err := offer.CreateDataChannel("data", nil)
+		assert.NoError(t, err)
+		_, err = dc.Detach()
+		assert.ErrorIs(t, err, errDetachBeforeOpened)
+		assert.NoError(t, offer.Close())
+		assert.NoError(t, answer.Close())
+	})
+}