diff options
| -rw-r--r-- | TODO | 5 | ||||
| -rw-r--r-- | include/sound.hpp | 7 | ||||
| -rw-r--r-- | src/main.cpp | 36 | ||||
| -rw-r--r-- | src/sound.cpp | 20 |
4 files changed, 52 insertions, 16 deletions
@@ -292,9 +292,6 @@ We can use dri2connect/dri3open to get the /dev/dri/card device. Note that this Add support for QVBR (QP with target bitrate). Maybe use VBR instead, since nvidia doesn't support QVBR and neither does vulkan. -KDE Plasma Wayland seems to use overlay planes now in non-fullscreen mode(limited to 1 overlay plane per gpu). Check if this is the case in the latest kde on arch linux. - If it is, then support it in kms capture. - Check if pipewire audio link-factory is available before attempting to use app audio or merging audio with pipewire. Also do the same in supports_app_audio check in gpu-screen-recorder --info output. @@ -427,5 +424,3 @@ Add the option to use application audio with exectable name, or automatically ch In the ui it should also list the application name beside the app audio. Region capture works incorrectly on gnome with hidpi, especially with 150% scaling. Same in gsr ui. - -Support overlay planes with kms capture. COSMIC uses an overlay plane for the top panel, so its not visible in the video. diff --git a/include/sound.hpp b/include/sound.hpp index c7164f8..6d903d8 100644 --- a/include/sound.hpp +++ b/include/sound.hpp @@ -71,6 +71,13 @@ int sound_device_get_by_name(SoundDevice *device, const char *node_name, const c void sound_device_close(SoundDevice *device); /* + Discards the audio that has been captured so far. + Call this before the first call to sound_device_read_next_chunk to not get audio that was captured before that point, + since the sound device can be created a while before audio capture starts. +*/ +void sound_device_flush(SoundDevice *device); + +/* Returns the next chunk of audio into @buffer. Returns the number of frames read, or a negative value on failure. */ diff --git a/src/main.cpp b/src/main.cpp index fcef91f..d64d7f4 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1521,6 +1521,10 @@ static int init_filter_graph(AVCodecContext* audio_codec_context, AVFilterGraph* goto fail; } + /* Make sure the sink always outputs frames with the exact amount of samples the audio encoder wants, + otherwise the audio encoder rejects the frame and that piece of audio is lost */ + av_buffersink_set_frame_size(abuffersink_ctx, audio_codec_context->frame_size); + *graph = filter_graph; *sink = abuffersink_ctx; @@ -4196,9 +4200,13 @@ int main(int argc, char **argv) { const double audio_fps = (double)audio_track.codec_context->sample_rate / (double)audio_track.codec_context->frame_size; const int64_t timeout_ms = std::round(1000.0 / audio_fps); const double timeout_sec = 1000.0 / audio_fps / 1000.0; - bool first_frame = true; int64_t num_received_frames = 0; + // The sound device is opened before the recording starts, so it can contain old audio from before the recording started. + // Discard it so the recording doesn't start with old audio. + if(audio_device.sound_device.handle) + sound_device_flush(&audio_device.sound_device); + while(running) { void *sound_buffer; int sound_buffer_size = -1; @@ -4246,15 +4254,13 @@ int main(int argc, char **argv) { // despite nvidia shadowplay and xbox game bar producing variable frame rate videos. // So we have to make sure we produce frames at the same relative rate as the video. if((num_missing_frames >= 1 && got_audio_data) || num_missing_frames >= 5 || !audio_device.sound_device.handle) { - // TODO: - //audio_track.frame->data[0] = empty_audio; - if(first_frame || num_missing_frames >= 5) { - if(needs_audio_conversion) - swr_convert(swr, &audio_device.frame->data[0], audio_track.codec_context->frame_size, (const uint8_t**)&empty_audio, audio_track.codec_context->frame_size); - else - audio_device.frame->data[0] = empty_audio; - } - first_frame = false; + // Fill the missing frames with silence. Duplicating the previous audio frame to fill the gap instead + // sounds like a stutter and it's especially noticeable at the start of the recording when the audio device + // hasn't started to deliver audio at a stable rate yet, which repeats the first audio frame multiple times. + if(needs_audio_conversion) + swr_convert(swr, &audio_device.frame->data[0], audio_track.codec_context->frame_size, (const uint8_t**)&empty_audio, audio_track.codec_context->frame_size); + else + audio_device.frame->data[0] = empty_audio; // TODO: Check if duplicate frame can be saved just by writing it with a different pts instead of sending it again std::lock_guard<std::mutex> lock(audio_filter_mutex); @@ -4283,12 +4289,20 @@ int main(int argc, char **argv) { if(!audio_device.sound_device.handle) { av_usleep(timeout_ms * 1000); } else if(got_audio_data) { + // The frame has to be made writable again if the frame was already sent to the audio filter above (when filling missing frames) + // because the audio filter only references the frame data instead of copying it. Without this the sent frames data would be + // overwritten with the audio data below, causing the audio to repeat instead of the missing frames being silent. + ret = av_frame_make_writable(audio_device.frame); + if (ret < 0) { + fprintf(stderr, "Failed to make audio frame writable\n"); + break; + } + // TODO: Instead of converting audio, get float audio from alsa. Or does alsa do conversion internally to get this format? if(needs_audio_conversion) swr_convert(swr, &audio_device.frame->data[0], audio_track.codec_context->frame_size, (const uint8_t**)&sound_buffer, audio_track.codec_context->frame_size); else audio_device.frame->data[0] = (uint8_t*)sound_buffer; - first_frame = false; std::lock_guard<std::mutex> lock(audio_filter_mutex); diff --git a/src/sound.cpp b/src/sound.cpp index eedfbf9..4e04d8f 100644 --- a/src/sound.cpp +++ b/src/sound.cpp @@ -520,6 +520,26 @@ void sound_device_close(SoundDevice *device) { device->handle = NULL; } +void sound_device_flush(SoundDevice *device) { + pa_handle *p = (pa_handle*)device->handle; + if(!p || !p->stream) + return; + + if(pa_stream_get_state(p->stream) != PA_STREAM_READY) + return; + + pa_operation *op = pa_stream_flush(p->stream, NULL, NULL); + if(!op) + return; + + /* The flush operation should finish immediately, the timeout is to not freeze if pulseaudio is in a bad state */ + const double start_time = clock_get_monotonic_seconds(); + while(pa_operation_get_state(op) == PA_OPERATION_RUNNING && clock_get_monotonic_seconds() - start_time < 0.1) { + pa_mainloop_iterate(p->mainloop, 1, NULL); + } + pa_operation_unref(op); +} + int sound_device_read_next_chunk(SoundDevice *device, void **buffer, double timeout_sec, double *latency_seconds) { pa_handle *pa = (pa_handle*)device->handle; if(pa_sound_device_read(pa, timeout_sec) < 0) { |