1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
|
using Gst;
namespace Dino.Plugins.Rtp {
public static extern Buffer adjust_to_running_time(Base.Transform transform, Buffer buf);
}
public class Dino.Plugins.Rtp.EchoProbe : Audio.Filter {
private static StaticPadTemplate sink_template = {"sink", PadDirection.SINK, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}};
private static StaticPadTemplate src_template = {"src", PadDirection.SRC, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}};
public Audio.Info audio_info { get; private set; }
public signal void on_new_buffer(owned Buffer buffer);
public signal void on_new_delay(int delay);
private uint period_samples;
private uint period_size;
public int delay { get; private set; }
private Base.Adapter adapter = new Base.Adapter();
static construct {
add_static_pad_template(sink_template);
add_static_pad_template(src_template);
set_static_metadata("Acoustic Echo Canceller probe", "Generic/Audio", "Gathers playback buffers for echo cancellation", "Dino Team <contact@dino.im>");
}
construct {
set_passthrough(true);
}
public override bool setup(Audio.Info info) {
audio_info = info;
period_samples = info.rate / 100; // 10ms buffers
period_size = period_samples * info.bpf;
return true;
}
public override bool src_event(owned Event event) {
Query query = new Query.latency();
if (event.type == EventType.LATENCY && srcpad != null && srcpad.query(query)) {
ClockTime upstream_latency;
query.parse_latency(null, out upstream_latency, null);
int delay = this.delay;
if (upstream_latency != CLOCK_TIME_NONE) {
delay = (int) (upstream_latency / MSECOND);
} else {
delay = 0;
}
if (delay != this.delay) {
debug("Delay adjusted from %ms to %dms", this.delay, delay);
this.delay = delay;
on_new_delay(delay);
}
}
return base.src_event((owned) event);
}
public override FlowReturn transform_ip(Buffer buf) {
lock (adapter) {
adapter.push(adjust_to_running_time(this, buf));
while (adapter.available() > period_size) {
on_new_buffer(adapter.take_buffer(period_size));
}
}
return FlowReturn.OK;
}
public override bool stop() {
adapter.clear();
return true;
}
}
public class Dino.Plugins.Rtp.VoiceProcessor : Audio.Filter {
private static StaticPadTemplate sink_template = {"sink", PadDirection.SINK, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}};
private static StaticPadTemplate src_template = {"src", PadDirection.SRC, PadPresence.ALWAYS, {null, "audio/x-raw,rate=48000,channels=1,layout=interleaved,format=S16LE"}};
public Audio.Info audio_info { get; private set; }
private ulong process_outgoing_buffer_handler_id;
private ulong process_stream_delay_handler_id;
private uint adjust_delay_timeout_id;
private uint period_samples;
private uint period_size;
private Base.Adapter adapter = new Base.Adapter();
private EchoProbe? echo_probe;
private Audio.StreamVolume? stream_volume;
private ClockTime last_reverse;
private void* native;
static construct {
add_static_pad_template(sink_template);
add_static_pad_template(src_template);
set_static_metadata("Voice Processor (AGC, AEC, filters, etc.)", "Generic/Audio", "Pre-processes voice with WebRTC Audio Processing Library", "Dino Team <contact@dino.im>");
}
construct {
set_passthrough(false);
}
public VoiceProcessor(EchoProbe? echo_probe = null, Audio.StreamVolume? stream_volume = null) {
this.echo_probe = echo_probe;
this.stream_volume = stream_volume;
}
private static extern void* init_native(int stream_delay);
private static extern void destroy_native(void* native);
private static extern void analyze_reverse_stream(void* native, Audio.Info info, Buffer buffer);
private static extern void process_stream(void* native, Audio.Info info, Buffer buffer);
private static extern void adjust_stream_delay(void* native);
private static extern void set_stream_delay(void* native, int delay);
private static extern void notify_gain_level(void* native, int gain_level);
private static extern int get_suggested_gain_level(void* native);
private static extern bool get_stream_has_voice(void* native);
public override bool setup(Audio.Info info) {
debug("VoiceProcessor.setup(%s)", info.to_caps().to_string());
audio_info = info;
period_samples = info.rate / 100; // 10ms buffers
period_size = period_samples * info.bpf;
adapter.clear();
return true;
}
public override bool start() {
native = init_native(echo_probe.delay);
if (process_outgoing_buffer_handler_id == 0 && echo_probe != null) {
process_outgoing_buffer_handler_id = echo_probe.on_new_buffer.connect(process_outgoing_buffer);
}
if (process_stream_delay_handler_id == 0 && echo_probe != null) {
process_stream_delay_handler_id = echo_probe.on_new_delay.connect(process_stream_delay);
}
if (stream_volume == null && sinkpad.get_peer() != null && sinkpad.get_peer().get_parent_element() is Audio.StreamVolume) {
stream_volume = sinkpad.get_peer().get_parent_element() as Audio.StreamVolume;
}
return true;
}
private bool adjust_delay() {
if (native != null) {
adjust_stream_delay(native);
return Source.CONTINUE;
} else {
adjust_delay_timeout_id = 0;
return Source.REMOVE;
}
}
private void process_outgoing_buffer(owned Buffer buffer) {
if (buffer.pts != uint64.MAX) {
last_reverse = buffer.pts;
}
if (native != null) {
buffer = (Buffer) buffer.make_writable();
analyze_reverse_stream(native, echo_probe.audio_info, buffer);
}
if (adjust_delay_timeout_id == 0 && echo_probe != null) {
adjust_delay_timeout_id = Timeout.add(1000, adjust_delay);
}
}
private void process_stream_delay(int stream_delay) {
if (native != null) {
set_stream_delay(native, stream_delay);
}
}
public override FlowReturn submit_input_buffer(bool is_discont, Buffer input) {
lock (adapter) {
if (is_discont) {
adapter.clear();
}
adapter.push(adjust_to_running_time(this, input));
}
return FlowReturn.OK;
}
public override FlowReturn generate_output(out Buffer output_buffer) {
lock (adapter) {
if (adapter.available() >= period_size) {
output_buffer = (Gst.Buffer) adapter.take_buffer(period_size).make_writable();
int old_gain_level = 0;
if (stream_volume != null) {
old_gain_level = (int) (stream_volume.get_volume(Audio.StreamVolumeFormat.LINEAR) * 255.0);
notify_gain_level(native, old_gain_level);
}
process_stream(native, audio_info, output_buffer);
if (stream_volume != null) {
int new_gain_level = get_suggested_gain_level(native);
if (old_gain_level != new_gain_level) {
debug("Gain: %i -> %i", old_gain_level, new_gain_level);
stream_volume.set_volume(Audio.StreamVolumeFormat.LINEAR, ((double)new_gain_level) / 255.0);
}
}
}
}
return FlowReturn.OK;
}
public override bool stop() {
if (process_outgoing_buffer_handler_id != 0) {
echo_probe.disconnect(process_outgoing_buffer_handler_id);
process_outgoing_buffer_handler_id = 0;
}
if (adjust_delay_timeout_id != 0) {
Source.remove(adjust_delay_timeout_id);
adjust_delay_timeout_id = 0;
}
adapter.clear();
destroy_native(native);
native = null;
return true;
}
}
|
