8000 Merge pull request #207 from yonatann/unified-plan · flutter-webrtc/flutter-webrtc@c818612 · GitHub
[go: up one dir, main page]
Skip to content

Commit c818612

Browse files
yonatannyonatann
authored
Merge pull request #207 from yonatann/unified-plan
Implemented audio only recording in media recorder for android and ios
2 parents 0d7a91a + b25d741 commit c818612

16 files changed

+417
-47
lines changed

‎.gitignore

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
.DS_Store
44
.packages
55
.vscode/.DS_Store
6+
.dart_tool/
67
example/pubspec.lock
78
pubspec.lock
89
example/ios/Podfile.lock
@@ -13,3 +14,21 @@ example/android/gradle*
1314
WorkspaceSettings.xcsettings
1415
example/.flutter-plugins
1516
example/android/local.properties
17+
android/.gradle
18+
android/.settings
19+
android/gradle
20+
android/gradlew
21+
android/gradlew.bat
22+
android/local.properties
23+
example/android/.settings
24+
example/.flutter-plugins-dependencies
25+
example/android/.project/
26+
example/android/app/.classpath
27+
example/ios/Flutter/flutter_export_environment.sh
28+
example/ios/Flutter/Generated.xcconfig
29+
example/ios/Pods/.symlinks
30+
example/ios/Pods/Local Podspecs/
31+
example/ios/Runner/GeneratedPluginRegistrant.h
32+
example/ios/Runner/GeneratedPluginRegistrant.m
33+
example/ios/Runner.xcworkspace/xcuserdata/
34+
example/android/.project

‎android/.classpath

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<classpath>
3+
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8/"/>
4+
<classpathentry kind="con" path="org.eclipse.buildship.core.gradleclasspathcontainer"/>
5+
<classpathentry kind="output" path="bin/default"/>
6+
</classpath>

‎android/.project

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<projectDescription>
3+
<name>flutter_webrtc</name>
4+
<comment>Project android_ created by Buildship.</comment>
5+
<projects>
6+
</projects>
7+
<buildSpec>
8+
<buildCommand>
9+
<name>org.eclipse.jdt.core.javabuilder</name>
10+
<arguments>
11+
</arguments>
12+
</buildCommand>
13+
<buildCommand>
14+
<name>org.eclipse.buildship.core.gradleprojectbuilder</name>
15+
<arguments>
16+
</arguments>
17+
</buildCommand>
18+
</buildSpec>
19+
<natures>
20+
<nature>org.eclipse.jdt.core.javanature</nature>
21+
<nature>org.eclipse.buildship.core.gradleprojectnature</nature>
22+
</natures>
23+
</projectDescription>

‎android/src/main/java/com/cloudwebrtc/webrtc/FlutterWebRTCPlugin.java

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -421,8 +421,7 @@ public void onMethodCall(MethodCall call, Result notSafeResult) {
421421
}
422422
} else if (call.method.equals("stopRecordToFile")) {
423423
Integer recorderId = call.argument("recorderId");
424-
getUserMediaImpl.stopRecording(recorderId);
425-
result.success(null);
424+
getUserMediaImpl.stopRecording(recorderId, result);
426425
} else if (call.method.equals("captureFrame")) {
427426
String path = call.argument("path");
428427
String videoTrackId = call.argument("trackId");
@@ -477,6 +476,10 @@ public void onMethodCall(MethodCall call, Result notSafeResult) {
477476
String kind = call.argument("kind");
478477
String streamId = call.argument("streamId");
479478
createSender(peerConnectionId, kind, streamId, result);
479+
} else if (call.method.equals("closeSender")) {
480+
String peerConnectionId = call.argument("peerConnectionId");
481+
String senderId = call.argument("senderId");
482+
stopSender(peerConnectionId, senderId, result);
480483
} else if (call.method.equals("addTrack")) {
481484
String peerConnectionId = call.argument("peerConnectionId");
482485
String trackId = call.argument("trackId");
@@ -1000,7 +1003,7 @@ private void mediaStreamTrackStop(final String id) {
10001003
private void mediaStreamTrackSetEnabled(final String id, final boolean enabled) {
10011004
MediaStreamTrack track = getTrackForId(id);
10021005
if (track == null) {
1003-
Log.d(TAG, "mediaStreamTrackSetEnabled() track is null");
1006+
Log.d(TAG, "mediaStreamTrackSetEnabled() track is null " + id);
10041007
return;
10051008
} else if (track.enabled() == enabled) {
10061009
return;
@@ -1386,8 +1389,21 @@ private void createSender(String peerConnectionId, String kind, String streamId,
13861389
}
13871390
}
13881391

1389-
private void addTrack(String peerConnectionId, String trackId, List<String> streamIds, Result result) {
1390-
PeerConnectionObserver pco = mPeerConnectionObservers.get(peerConnectionId);
1392+
1393+
private void stopSender(String peerConnectionId, String senderId, Result result) {
1394+
PeerConnectionObserver pco
1395+
= mPeerConnectionObservers.get(peerConnectionId);
1396+
if (pco == null || pco.getPeerConnection() == null) {
1397+
Log.d(TAG, "removeTrack() peerConnection is null");
1398+
result.error("removeTrack", "removeTrack() peerConnection is null", null);
1399+
} else {
1400+
pco.closeSender(senderId, result);
1401+
}
1402+
}
1403+
1404+
private void addTrack(String peerConnectionId, String trackId, List<String> streamIds, Result result){
1405+
PeerConnectionObserver pco
1406+
= mPeerConnectionObservers.get(peerConnectionId);
13911407
MediaStreamTrack track = localTracks.get(trackId);
13921408
if (track == null) {
13931409
result.error("addTrack", "addTrack() track is null", null);

‎android/src/main/java/com/cloudwebrtc/webrtc/GetUserMediaImpl.java

Lines changed: 3 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -585,6 +585,7 @@ private void getUserMedia(
585585
}
586586

587587
String id = track.id();
588+
Log.d(TAG, "MediaStream Track id: " + id);
588589

589590
if (track instanceof AudioTrack) {
590591
mediaStream.addTrack((AudioTrack) track);
@@ -792,19 +793,11 @@ else if (audioChannel == AudioChannel.OUTPUT) {
792793
mediaRecorders.append(id, mediaRecorder);
793794
}
794795

795-
void stopRecording(Integer id) {
796+
void stopRecording(Integer id, Result result) {
796797
MediaRecorderImpl mediaRecorder = mediaRecorders.get(id);
797798
if (mediaRecorder != null) {
798-
mediaRecorder.stopRecording();
799+
mediaRecorder.stopRecording(result);
799800
mediaRecorders.remove(id);
800-
File file = mediaRecorder.getRecordFile();
801-
if (file != null) {
802-
ContentValues values = new ContentValues(3);
803-
values.put(MediaStore.Video.Media.TITLE, file.getName());
804-
values.put(MediaStore.Video.Media.MIME_TYPE, "video/mp4");
805-
values.put(MediaStore.Video.Media.DATA, file.getAbsolutePath());
806-
applicationContext.getContentResolver().insert(MediaStore.Video.Media.EXTERNAL_CONTENT_URI, values);
807-
}
808801
}
809802
}
810803

‎android/src/main/java/com/cloudwebrtc/webrtc/PeerConnectionObserver.java

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -92,8 +92,8 @@ void close() {
9292
remoteTracks.clear();
9393
dataChannels.clear();
9494
transceivers.clear();
95-
senders.clear();;
96-
receivers.clear();;
95+
senders.clear();
96+
receivers.clear();
9797
}
9898

9999
void dispose() {
@@ -641,6 +641,14 @@ public void createSender(String kind, String streamId, Result result){
641641
result.success(rtpSenderToMap(sender));
642642
}
643643

644+
public void closeSender(String senderId, Result result) {
645+
RtpSender sender = senders.get(senderId);
646+
sender.dispose();
647+
Map<String, Object> params = new HashMap<>();
648+
params.put("result", true);
649+
result.success(params);
650+
}
651+
644652
public void addTrack(MediaStreamTrack track, List<String> streamIds, Result result){
645653
RtpSender sender = peerConnection.addTrack(track, streamIds);
646654
senders.put(sender.id(),sender);
@@ -654,8 +662,8 @@ public void removeTrack(String senderId, Result result){
654662
return;
655663
}
656664
boolean res = peerConnection.removeTrack(sender);
657-
ConstraintsMap params = new ConstraintsMap();
658-
params.putBoolean("result", res);
665+
Map<String, Object> params = new HashMap<>();
666+
params.put("result", res);
659667
result.success(params);
660668
}
661669

‎android/src/main/java/com/cloudwebrtc/webrtc/record/AudioFileRenderer.java

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
package com.cloudwebrtc.webrtc.record;
2+
3+
import org.webrtc.audio.JavaAudioDeviceModule;
4+
5+
import java.io.File;
6+
import java.io.FileNotFoundException;
7+
import java.io.FileOutputStream;
8+
import java.io.IOException;
9+
import java.io.OutputStream;
10+
import java.nio.ByteBuffer;
11+
12+
import android.media.AudioFormat;
13+
import android.media.MediaCodec;
14+
import android.media.MediaCodecInfo;
15+
import android.media.MediaFormat;
16+
import android.media.MediaMuxer;
17+
import android.os.Handler;
18+
import android.os.HandlerThread;
19+
import android.util.Log;
20+
21+
import io.flutter.plugin.common.MethodChannel;
22+
23+
import static android.media.MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4;
24+
25+
public class AudioFileRenderer implements JavaAudioDeviceModule.SamplesReadyCallback {
26+
private static final String TAG = "AudioFileRenderer";
27+
private boolean isRunning = true;
28+
29+
private final HandlerThread audioThread;
30+
private final Handler audioThreadHandler;
31+
private MediaCodec audioEncoder;
32+
private ByteBuffer[] audioInputBuffers;
33+
private ByteBuffer[] audioOutputBuffers;
34+
private MediaCodec.BufferInfo audioBufferInfo;
35+
private MediaMuxer mediaMuxer;
36+
private int trackIndex = -1;
37+
private int audioTrackIndex = -1;
38+
private long presTime = 0L;
39+
private volatile boolean muxerStarted = false;
40+
41+
AudioFileRenderer(File outputFile) throws IOException {
42+
audioThread = new HandlerThread(TAG + "AudioThread");
43+
audioThread.start();
44+
audioThreadHandler = new Handler(audioThread.getLooper());
45+
46+
mediaMuxer = new MediaMuxer(outputFile.getPath(),
47+
MUXER_OUTPUT_MPEG_4);
48+
}
49+
50+
@Override
51+
public void onWebRtcAudioRecordSamplesReady(JavaAudioDeviceModule.AudioSamples audioSamples) {
52+
audioThreadHandler.post(() -> {
53+
if (audioEncoder == null) try {
54+
audioEncoder = MediaCodec.createEncoderByType("audio/mp4a-latm");
55+
MediaFormat format = new MediaFormat();
56+
format.setString(MediaFormat.KEY_MIME, "audio/mp4a-latm");
57+
format.setInteger(MediaFormat.KEY_CHANNEL_COUNT, audioSamples.getChannelCount());
58+
format.setInteger(MediaFormat.KEY_SAMPLE_RATE, audioSamples.getSampleRate());
59+
format.setInteger(MediaFormat.KEY_BIT_RATE, 64 * 1024);
60+
format.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC);
61+
audioEncoder.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
62+
audioEncoder.start();
63+
audioInputBuffers = audioEncoder.getInputBuffers();
64+
audioOutputBuffers = audioEncoder.getOutputBuffers();
65+
} catch (IOException exception) {
66+
Log.wtf(TAG, exception);
67+
}
68+
int bufferIndex = audioEncoder.dequeueInputBuffer(0);
69+
if (bufferIndex >= 0) {
70+
ByteBuffer buffer = audioInputBuffers[bufferIndex];
71+
buffer.clear();
72+
byte[] data = audioSamples.getData();
73+
buffer.put(data);
74+
audioEncoder.queueInputBuffer(bufferIndex, 0, data.length, presTime, 0);
75+
presTime += data.length * 125 / 12; // 1000000 microseconds / 48000hz / 2 bytes
76+
}
77+
drainAudio();
78+
});
79+
}
80+
81+
private void drainAudio() {
82+
if (audioBufferInfo == null)
83+
audioBufferInfo = new MediaCodec.BufferInfo();
84+
while (true) {
85+
int encoderStatus = audioEncoder.dequeueOutputBuffer(audioBufferInfo, 10000);
86+
if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
87+
break;
88+
} else if (encoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
89+
// not expected for an encoder
90+
audioOutputBuffers = audioEncoder.getOutputBuffers();
91+
Log.d(TAG, "encoder output buffers changed");
92+
} else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
93+
// not expected for an encoder
94+
MediaFormat newFormat = audioEncoder.getOutputFormat();
95+
96+
Log.d(TAG, "encoder output format changed: " + newFormat);
97+
audioTrackIndex = mediaMuxer.addTrack(newFormat);
98+
if (audioTrackIndex != -1 && !muxerStarted) {
99+
mediaMuxer.start();
100+
muxerStarted = true;
101+
}
102+
if (!muxerStarted)
103+
break;
104+
} else if (encoderStatus < 0) {
105+
Log.d(TAG, "unexpected result fr om encoder.dequeueOutputBuffer: " + encoderStatus);
106+
} else { // encoderStatus >= 0
107+
try {
108+
ByteBuffer encodedData = audioOutputBuffers[encoderStatus];
109+
if (encodedData == null) {
110+
Log.e(TAG, "encoderOutputBuffer " + encoderStatus + " was null");
111+
break;
112+
}
113+
// It's usually necessary to adjust the ByteBuffer values to match BufferInfo.
114+
encodedData.position(audioBufferInfo.offset);
115+
encodedData.limit(audioBufferInfo.offset + audioBufferInfo.size);
116+
if (muxerStarted)
117+
mediaMuxer.writeSampleData(audioTrackIndex, encodedData, audioBufferInfo);
118+
isRunning = isRunning && (audioBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) == 0;
119+
audioEncoder.releaseOutputBuffer(encoderStatus, false);
120+
if ((audioBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
121+
break;
122+
}
123+
} catch (Exception e) {
124+
Log.wtf(TAG, e);
125+
break;
126+
}
127+
}
128+< 10000 /span>
}
129+
}
130+
131+
/**
132+
* Release all resources. All already posted frames will be rendered first.
133+
*/
134+
void release(MethodChannel.Result result) {
135+
isRunning = false;
136+
if (audioThreadHandler != null)
137+
audioThreadHandler.post(() -> {
138+
if (audioEncoder != null) {
139+
audioEncoder.stop();
140+
audioEncoder.release();
141+
}
142+
try {
143+
mediaMuxer.stop();
144+
mediaMuxer.release();
145+
} catch (Exception e) {
146+
// do nothing...
147+
}
148+
audioThread.quit();
149+
150+
result.success(null);
151+
});
152+
}
153+
}

‎android/src/main/java/com/cloudwebrtc/webrtc/record/AudioSamplesInterceptor.java

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,27 @@
11
package com.cloudwebrtc.webrtc.record;
22

33
import android.annotation.SuppressLint;
4+
import android.util.Log;
45

56
import org.webrtc.audio.JavaAudioDeviceModule.SamplesReadyCallback;
67
import org.webrtc.audio.JavaAudioDeviceModule.AudioSamples;
78

8-
import java.util.HashMap;
9+
import java.util.concurrent.ConcurrentHashMap;
910

1011
/** JavaAudioDeviceModule allows attaching samples callback only on building
1112
* We don't want to instantiate VideoFileRenderer and codecs at this step
1213
* It's simple dummy class, it does nothing until samples are necessary */
1314
@SuppressWarnings("WeakerAccess")
1415
public class AudioSamplesInterceptor implements SamplesReadyCallback {
15-
16+
public static int id = 0;
17+
private int _id;
18+
private static final String TAG = "AudioSamplesInterceptor";
1619
@SuppressLint("UseSparseArrays")
17-
protected final HashMap<Integer, SamplesReadyCallback> callbacks = new HashMap<>();
20+
protected final ConcurrentHashMap<Integer, SamplesReadyCallback> callbacks = new ConcurrentHashMap<>();
21+
22+
public AudioSamplesInterceptor() {
23+
this._id = id++;
24+
}
1825

1926
@Override
2027
public void onWebRtcAudioRecordSamplesReady(AudioSamples audioSamples) {
@@ -25,10 +32,12 @@ public void onWebRtcAudioRecordSamplesReady(AudioSamples audioSamples) {
2532

2633
public void attachCallback(Integer id, SamplesReadyCallback callback) throws Exception {
2734
callbacks.put(id, callback);
35+
Log.d(TAG, _id + " Attached callback "+callbacks.size());
2836
}
2937

3038
public void detachCallback(Integer id) {
3139
callbacks.remove(id);
40+
Log.d(TAG, _id + " Detached callback "+callbacks.size());
3241
}
3342

3443
}

0 commit comments

Comments
 (0)
0