aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authordec05eba <dec05eba@protonmail.com>2026-08-01 21:49:08 +0200
committerdec05eba <dec05eba@protonmail.com>2026-08-01 21:49:08 +0200
commitbd12a0ede46bbd92510f1cd82b04383e4a9e83c8 (patch)
tree4cb025520db4c75e91e238207c3799ef15c227b1
parente91f1c0b75968c576fd5adb1738737cb19dd6094 (diff)
Move audio capture from main.cpp into audio_capture module and replace std::thread with pthread
Audio device and mix threads are now pthreads with explicit context structs, and the paused time tracking moved into a gsr_recording_clock.
-rw-r--r--include/recorder/audio_capture.h92
-rw-r--r--include/recorder/recording_clock.h29
-rw-r--r--meson.build2
-rw-r--r--src/main.cpp582
-rw-r--r--src/recorder/audio_capture.c610
-rw-r--r--src/recorder/recording_clock.c61
6 files changed, 857 insertions, 519 deletions
diff --git a/include/recorder/audio_capture.h b/include/recorder/audio_capture.h
new file mode 100644
index 0000000..68390aa
--- /dev/null
+++ b/include/recorder/audio_capture.h
@@ -0,0 +1,92 @@
+#ifndef GSR_RECORDER_AUDIO_CAPTURE_H
+#define GSR_RECORDER_AUDIO_CAPTURE_H
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <signal.h>
+#include <pthread.h>
+#include "../sound.h"
+#include "recording_clock.h"
+#include "../encoder/encoder.h"
+#include "audio_input.h"
+
+#ifdef GSR_APP_AUDIO
+#include "../pipewire_audio.h"
+#endif
+
+#include <libavcodec/avcodec.h>
+#include <libavfilter/avfilter.h>
+
+#define GSR_MAX_AUDIO_SOURCES_PER_TRACK 32
+
+typedef struct gsr_audio_capture gsr_audio_capture;
+typedef struct gsr_audio_track gsr_audio_track;
+typedef struct gsr_audio_device_capture gsr_audio_device_capture;
+
+typedef struct {
+ gsr_audio_capture *audio_capture;
+ gsr_audio_track *track;
+ gsr_audio_device_capture *device;
+} gsr_audio_device_thread_userdata;
+
+struct gsr_audio_device_capture {
+ SoundDevice sound_device;
+ gsr_audio_input audio_input;
+ AVFilterContext *src_filter_ctx;
+ AVFrame *frame;
+ pthread_t thread;
+ bool thread_created;
+ gsr_audio_device_thread_userdata thread_userdata;
+};
+
+/* TODO: Instead of having a thread for each audio device, have one thread for all of them and read the data with non-blocking read */
+struct gsr_audio_track {
+ char name[GSR_AUDIO_TRACK_NAME_MAX_SIZE];
+ AVCodecContext *codec_context;
+ gsr_audio_device_capture *audio_devices;
+ size_t num_audio_devices;
+ AVFilterGraph *graph;
+ AVFilterContext *sink;
+ int stream_index;
+ int64_t pts;
+};
+
+struct gsr_audio_capture {
+ gsr_audio_track *tracks;
+ size_t num_tracks;
+ size_t capacity_tracks;
+
+ pthread_mutex_t filter_mutex;
+ bool filter_mutex_initialized;
+ pthread_t amix_thread;
+ bool amix_thread_created;
+ uint8_t *empty_audio;
+
+ gsr_encoder *encoder;
+ gsr_recording_clock *clock;
+ const volatile sig_atomic_t *running;
+};
+
+/* Returns a |gsr_error| value */
+int gsr_audio_capture_init(gsr_audio_capture *self, gsr_encoder *encoder, gsr_recording_clock *clock, const volatile sig_atomic_t *running);
+void gsr_audio_capture_deinit(gsr_audio_capture *self);
+
+bool gsr_audio_capture_add_track(gsr_audio_capture *self, const gsr_audio_track *track);
+/* Allocates the silence buffer and starts one thread per audio device, plus the amix thread when |uses_amix| is set */
+int gsr_audio_capture_start(gsr_audio_capture *self, int audio_max_frame_size, bool uses_amix);
+void gsr_audio_capture_join_threads(gsr_audio_capture *self);
+void gsr_audio_capture_lock_filter(gsr_audio_capture *self);
+void gsr_audio_capture_unlock_filter(gsr_audio_capture *self);
+
+/* Returns 0 on success, or a negative value on failure. |src_filter_ctx| must have room for |num_sources| items */
+int gsr_audio_init_filter_graph(AVCodecContext *audio_codec_context, AVFilterGraph **graph, AVFilterContext **sink, AVFilterContext **src_filter_ctx, size_t num_sources);
+
+/* Returns a |gsr_error| value. Opens the sound devices of one audio track */
+int gsr_audio_track_init_device_inputs(gsr_audio_track *self, const gsr_merged_audio_inputs *merged_audio_inputs, AVCodecContext *audio_codec_context, int num_channels, double num_audio_frames_shift, AVFilterContext **src_filter_ctx, bool use_amix);
+#ifdef GSR_APP_AUDIO
+/* Returns a |gsr_error| value. Creates a combined sink that application audio is routed to */
+int gsr_audio_track_init_application_input(gsr_audio_track *self, const gsr_merged_audio_inputs *merged_audio_inputs, AVCodecContext *audio_codec_context, int num_channels, double num_audio_frames_shift, gsr_pipewire_audio *pipewire_audio);
+#endif
+void gsr_audio_track_deinit(gsr_audio_track *self);
+
+#endif /* GSR_RECORDER_AUDIO_CAPTURE_H */
diff --git a/include/recorder/recording_clock.h b/include/recorder/recording_clock.h
new file mode 100644
index 0000000..4e302b6
--- /dev/null
+++ b/include/recorder/recording_clock.h
@@ -0,0 +1,29 @@
+#ifndef GSR_RECORDER_RECORDING_CLOCK_H
+#define GSR_RECORDER_RECORDING_CLOCK_H
+
+#include <stdbool.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Monotonic clock that excludes the time the recording has been paused. Safe to use from multiple threads */
+typedef struct gsr_recording_clock gsr_recording_clock;
+
+gsr_recording_clock* gsr_recording_clock_create(void);
+void gsr_recording_clock_destroy(gsr_recording_clock *self);
+
+/* Sets the time the recording started to now */
+void gsr_recording_clock_start(gsr_recording_clock *self);
+double gsr_recording_clock_get_start_time(const gsr_recording_clock *self);
+/* Returns the current time, excluding the time the recording has been paused */
+double gsr_recording_clock_get_time(const gsr_recording_clock *self);
+
+void gsr_recording_clock_set_paused(gsr_recording_clock *self, bool paused);
+bool gsr_recording_clock_is_paused(const gsr_recording_clock *self);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* GSR_RECORDER_RECORDING_CLOCK_H */
diff --git a/meson.build b/meson.build
index c751f9e..d56b21c 100644
--- a/meson.build
+++ b/meson.build
@@ -39,6 +39,8 @@ src = [
'src/recorder/capture_source.c',
'src/recorder/capture_setup.c',
'src/recorder/audio_input.c',
+ 'src/recorder/audio_capture.c',
+ 'src/recorder/recording_clock.c',
'src/egl.c',
'src/cuda.c',
'src/window_texture.c',
diff --git a/src/main.cpp b/src/main.cpp
index 1fb4d38..b3056c3 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -7,6 +7,7 @@ extern "C" {
#include "../include/recorder/capture_source.h"
#include "../include/recorder/capture_setup.h"
#include "../include/recorder/audio_input.h"
+#include "../include/recorder/audio_capture.h"
#include "../include/recorder/error.h"
#include "../include/capture/nvfbc.h"
#include "../include/capture/xcomposite.h"
@@ -195,26 +196,7 @@ static void run_recording_saved_script_async(const char *script_file, const char
}
}
-struct AudioDeviceData {
- SoundDevice sound_device;
- gsr_audio_input audio_input;
- AVFilterContext *src_filter_ctx = nullptr;
- AVFrame *frame = nullptr;
- std::thread thread; // TODO: Instead of having a thread for each track, have one thread for all threads and read the data with non-blocking read
-};
-
// TODO: Cleanup
-struct AudioTrack {
- std::string name;
- AVCodecContext *codec_context = nullptr;
-
- std::vector<AudioDeviceData> audio_devices;
- AVFilterGraph *graph = nullptr;
- AVFilterContext *sink = nullptr;
- int stream_index = 0;
- int64_t pts = 0;
-};
-
static bool add_hdr_metadata_to_video_stream(gsr_capture *cap, AVStream *video_stream) {
size_t light_metadata_size = 0;
size_t mastering_display_metadata_size = 0;
@@ -286,7 +268,7 @@ static void set_format_context_options(AVFormatContext *av_format_context) {
}
struct RecordingStartAudio {
- const AudioTrack *audio_track;
+ const gsr_audio_track *audio_track;
AVStream *stream;
};
@@ -296,7 +278,7 @@ struct RecordingStartResult {
std::vector<RecordingStartAudio> audio_inputs;
};
-static RecordingStartResult start_recording_create_streams(const char *filename, const args_parser &arg_parser, AVCodecContext *video_codec_context, const std::vector<AudioTrack> &audio_tracks, bool hdr, gsr_video_sources *video_sources) {
+static RecordingStartResult start_recording_create_streams(const char *filename, const args_parser &arg_parser, AVCodecContext *video_codec_context, const gsr_audio_capture *audio_capture, bool hdr, gsr_video_sources *video_sources) {
AVFormatContext *av_format_context;
avformat_alloc_output_context2(&av_format_context, nullptr, arg_parser.settings.container_format, filename);
set_format_context_options(av_format_context);
@@ -305,12 +287,13 @@ static RecordingStartResult start_recording_create_streams(const char *filename,
avcodec_parameters_from_context(video_stream->codecpar, video_codec_context);
RecordingStartResult result;
- result.audio_inputs.reserve(audio_tracks.size());
+ result.audio_inputs.reserve(audio_capture->num_tracks);
- for(const AudioTrack &audio_track : audio_tracks) {
+ for(size_t audio_track_index = 0; audio_track_index < audio_capture->num_tracks; ++audio_track_index) {
+ const gsr_audio_track &audio_track = audio_capture->tracks[audio_track_index];
AVStream *audio_stream = create_stream(av_format_context, audio_track.codec_context);
- if(!audio_track.name.empty() && !arg_parser.settings.exclude_metadata)
- av_dict_set(&audio_stream->metadata, "title", audio_track.name.c_str(), 0);
+ if(!audio_track.name[0] == '\0' && !arg_parser.settings.exclude_metadata)
+ av_dict_set(&audio_stream->metadata, "title", audio_track.name, 0);
avcodec_parameters_from_context(audio_stream->codecpar, audio_track.codec_context);
result.audio_inputs.push_back({&audio_track, audio_stream});
}
@@ -394,7 +377,7 @@ struct AudioPtsOffset {
int stream_index = 0;
};
-static bool save_replay_async(AVCodecContext *video_codec_context, int video_stream_index, const std::vector<AudioTrack> &audio_tracks, gsr_encoder *encoder, const args_parser &arg_parser, const std::string &file_extension, bool date_folders, bool hdr, gsr_video_sources *video_sources, int current_save_replay_seconds) {
+static bool save_replay_async(AVCodecContext *video_codec_context, int video_stream_index, const gsr_audio_capture *audio_capture, gsr_encoder *encoder, const args_parser &arg_parser, const std::string &file_extension, bool date_folders, bool hdr, gsr_video_sources *video_sources, int current_save_replay_seconds) {
if(save_replay_thread.valid())
return true;
@@ -420,15 +403,16 @@ static bool save_replay_async(AVCodecContext *video_codec_context, int video_str
const int64_t video_pts_offset = gsr_replay_buffer_iterator_get_packet(cloned_replay_buffer, video_start_iterator)->pts;
std::vector<AudioPtsOffset> audio_pts_offsets;
- audio_pts_offsets.reserve(audio_tracks.size());
- for(const AudioTrack &audio_track : audio_tracks) {
+ audio_pts_offsets.reserve(audio_capture->num_tracks);
+ for(size_t audio_track_index = 0; audio_track_index < audio_capture->num_tracks; ++audio_track_index) {
+ const gsr_audio_track &audio_track = audio_capture->tracks[audio_track_index];
const gsr_replay_buffer_iterator audio_start_iterator = gsr_replay_buffer_find_keyframe(cloned_replay_buffer, video_start_iterator, audio_track.stream_index, false);
const int64_t audio_pts_offset = audio_start_iterator.packet_index == (size_t)-1 ? 0 : gsr_replay_buffer_iterator_get_packet(cloned_replay_buffer, audio_start_iterator)->pts;
audio_pts_offsets.push_back(AudioPtsOffset{audio_pts_offset, audio_track.stream_index});
}
std::string output_filepath = create_new_recording_filepath_from_timestamp(arg_parser.settings.filename, "Replay", file_extension, date_folders);
- RecordingStartResult recording_start_result = start_recording_create_streams(output_filepath.c_str(), arg_parser, video_codec_context, audio_tracks, hdr, video_sources);
+ RecordingStartResult recording_start_result = start_recording_create_streams(output_filepath.c_str(), arg_parser, video_codec_context, audio_capture, hdr, video_sources);
if(!recording_start_result.av_format_context) {
pthread_mutex_lock(&encoder->replay_mutex);
gsr_replay_buffer_destroy(cloned_replay_buffer);
@@ -489,7 +473,7 @@ static bool save_replay_async(AVCodecContext *video_codec_context, int video_str
continue;
}
- const AudioTrack *audio_track = recording_start_audio->audio_track;
+ const gsr_audio_track *audio_track = recording_start_audio->audio_track;
stream = recording_start_audio->stream;
codec_context = audio_track->codec_context;
@@ -525,140 +509,6 @@ static bool save_replay_async(AVCodecContext *video_codec_context, int video_str
return true;
}
-static int init_filter_graph(AVCodecContext* audio_codec_context, AVFilterGraph** graph, AVFilterContext** sink, std::vector<AVFilterContext*>& src_filter_ctx, size_t num_sources) {
- char ch_layout[64];
- int err = 0;
- ch_layout[0] = '\0';
-
- // C89-style variable declaration to
- // avoid problems because of goto
- AVFilterGraph* filter_graph = nullptr;
- AVFilterContext* mix_ctx = nullptr;
-
- const AVFilter* mix_filter = nullptr;
- const AVFilter* abuffersink = nullptr;
- AVFilterContext* abuffersink_ctx = nullptr;
- char args[512] = { 0 };
-#if LIBAVFILTER_VERSION_INT >= AV_VERSION_INT(7, 107, 100)
- bool normalize = false;
-#endif
-
- filter_graph = avfilter_graph_alloc();
- if (!filter_graph) {
- fprintf(stderr, "Unable to create filter graph.\n");
- err = AVERROR(ENOMEM);
- goto fail;
- }
-
- for(size_t i = 0; i < num_sources; ++i) {
- const AVFilter *abuffer = avfilter_get_by_name("abuffer");
- if (!abuffer) {
- fprintf(stderr, "Could not find the abuffer filter.\n");
- err = AVERROR_FILTER_NOT_FOUND;
- goto fail;
- }
-
- AVFilterContext *abuffer_ctx = avfilter_graph_alloc_filter(filter_graph, abuffer, NULL);
- if (!abuffer_ctx) {
- fprintf(stderr, "Could not allocate the abuffer instance.\n");
- err = AVERROR(ENOMEM);
- goto fail;
- }
-
- #if LIBAVCODEC_VERSION_MAJOR < 60
- av_get_channel_layout_string(ch_layout, sizeof(ch_layout), 0, AV_CH_LAYOUT_STEREO);
- #else
- av_channel_layout_describe(&audio_codec_context->ch_layout, ch_layout, sizeof(ch_layout));
- #endif
- av_opt_set (abuffer_ctx, "channel_layout", ch_layout, AV_OPT_SEARCH_CHILDREN);
- av_opt_set (abuffer_ctx, "sample_fmt", av_get_sample_fmt_name(audio_codec_context->sample_fmt), AV_OPT_SEARCH_CHILDREN);
- av_opt_set_q (abuffer_ctx, "time_base", audio_codec_context->time_base, AV_OPT_SEARCH_CHILDREN);
- av_opt_set_int(abuffer_ctx, "sample_rate", audio_codec_context->sample_rate, AV_OPT_SEARCH_CHILDREN);
- av_opt_set_int(abuffer_ctx, "bit_rate", audio_codec_context->bit_rate, AV_OPT_SEARCH_CHILDREN);
-
- err = avfilter_init_str(abuffer_ctx, NULL);
- if (err < 0) {
- fprintf(stderr, "Could not initialize the abuffer filter.\n");
- goto fail;
- }
-
- src_filter_ctx.push_back(abuffer_ctx);
- }
-
- mix_filter = avfilter_get_by_name("amix");
- if (!mix_filter) {
- av_log(NULL, AV_LOG_ERROR, "Could not find the mix filter.\n");
- err = AVERROR_FILTER_NOT_FOUND;
- goto fail;
- }
-
-#if LIBAVFILTER_VERSION_INT >= AV_VERSION_INT(7, 107, 100)
- snprintf(args, sizeof(args), "inputs=%d:normalize=%s", (int)num_sources, normalize ? "true" : "false");
-#else
- snprintf(args, sizeof(args), "inputs=%d", (int)num_sources);
- gsr_log(GSR_LOG_LEVEL_WARNING, "your ffmpeg version doesn't support disabling normalizing of mixed audio. Volume might be lower than expected");
-#endif
-
- err = avfilter_graph_create_filter(&mix_ctx, mix_filter, "amix", args, NULL, filter_graph);
- if (err < 0) {
- av_log(NULL, AV_LOG_ERROR, "Cannot create audio amix filter\n");
- goto fail;
- }
-
- abuffersink = avfilter_get_by_name("abuffersink");
- if (!abuffersink) {
- fprintf(stderr, "Could not find the abuffersink filter.\n");
- err = AVERROR_FILTER_NOT_FOUND;
- goto fail;
- }
-
- abuffersink_ctx = avfilter_graph_alloc_filter(filter_graph, abuffersink, "sink");
- if (!abuffersink_ctx) {
- fprintf(stderr, "Could not allocate the abuffersink instance.\n");
- err = AVERROR(ENOMEM);
- goto fail;
- }
-
- err = avfilter_init_str(abuffersink_ctx, NULL);
- if (err < 0) {
- fprintf(stderr, "Could not initialize the abuffersink instance.\n");
- goto fail;
- }
-
- err = 0;
- for(size_t i = 0; i < src_filter_ctx.size(); ++i) {
- AVFilterContext *src_ctx = src_filter_ctx[i];
- if (err >= 0)
- err = avfilter_link(src_ctx, 0, mix_ctx, i);
- }
- if (err >= 0)
- err = avfilter_link(mix_ctx, 0, abuffersink_ctx, 0);
- if (err < 0) {
- av_log(NULL, AV_LOG_ERROR, "Error connecting filters\n");
- goto fail;
- }
-
- err = avfilter_graph_config(filter_graph, NULL);
- if (err < 0) {
- av_log(NULL, AV_LOG_ERROR, "Error configuring the filter graph\n");
- goto fail;
- }
-
- /* Make sure the sink always outputs frames with the exact amount of samples the audio encoder wants,
- otherwise the audio encoder rejects the frame and that piece of audio is lost */
- av_buffersink_set_frame_size(abuffersink_ctx, audio_codec_context->frame_size);
-
- *graph = filter_graph;
- *sink = abuffersink_ctx;
-
- return 0;
-
-fail:
- avfilter_graph_free(&filter_graph);
- src_filter_ctx.clear(); // possibly unnecessary?
- return err;
-}
-
static void list_system_info(bool wayland) {
printf("display_server|%s\n", wayland ? "wayland" : "x11");
bool supports_app_audio = false;
@@ -1149,101 +999,7 @@ static void capture_image_to_file(args_parser &arg_parser, gsr_egl *egl, gsr_win
// OH, YOU MISSPELLED THE AUDIO INPUT? FUCK YOU
// Should use amix if more than 1 audio device and 0 application audio, merged
/* Returns -1 if none is available */
-static std::vector<AudioDeviceData> create_device_audio_inputs(const gsr_merged_audio_inputs *merged_audio_inputs, AVCodecContext *audio_codec_context, int num_channels, double num_audio_frames_shift, std::vector<AVFilterContext*> &src_filter_ctx, bool use_amix) {
- std::vector<AudioDeviceData> audio_track_audio_devices;
- for(size_t i = 0; i < merged_audio_inputs->num_items; ++i) {
- const gsr_audio_input &audio_input = merged_audio_inputs->items[i];
- AVFilterContext *src_ctx = nullptr;
- if(use_amix)
- src_ctx = src_filter_ctx[i];
-
- AudioDeviceData audio_device;
- audio_device.audio_input = audio_input;
- audio_device.src_filter_ctx = src_ctx;
-
- if(audio_input.name[0] == '\0') {
- audio_device.sound_device.handle = NULL;
- audio_device.sound_device.frames = 0;
- } else {
- char description[GSR_AUDIO_INPUT_NAME_MAX_SIZE + 8];
- snprintf(description, sizeof(description), "gsr-%s", audio_input.name);
- if(sound_device_get_by_name(&audio_device.sound_device, description, audio_input.name, description, num_channels, audio_codec_context->frame_size, audio_codec_context_get_audio_format(audio_codec_context)) != 0) {
- gsr_log(GSR_LOG_LEVEL_ERROR, "failed to get \"%s\" audio device", audio_input.name);
- _exit(1);
- }
- }
-
- audio_device.frame = create_audio_frame(audio_codec_context);
- if(!audio_device.frame)
- _exit(1);
- audio_device.frame->pts = -audio_codec_context->frame_size * num_audio_frames_shift;
-
- audio_track_audio_devices.push_back(std::move(audio_device));
- }
- return audio_track_audio_devices;
-}
-
#ifdef GSR_APP_AUDIO
-static AudioDeviceData create_application_audio_audio_input(const gsr_merged_audio_inputs &merged_audio_inputs, AVCodecContext *audio_codec_context, int num_channels, double num_audio_frames_shift, gsr_pipewire_audio *pipewire_audio) {
- AudioDeviceData audio_device;
- audio_device.frame = create_audio_frame(audio_codec_context);
- if(!audio_device.frame)
- _exit(1);
- audio_device.frame->pts = -audio_codec_context->frame_size * num_audio_frames_shift;
-
- char random_str[8];
- if(!generate_random_characters_standard_alphabet(random_str, sizeof(random_str))) {
- gsr_log(GSR_LOG_LEVEL_ERROR, "failed to generate random string");
- _exit(1);
- }
-
- std::string combined_sink_name = "gsr-combined-";
- combined_sink_name.append(random_str, sizeof(random_str));
- combined_sink_name += ".monitor";
-
- if(sound_device_get_by_name(&audio_device.sound_device, combined_sink_name.c_str(), "", "gpu-screen-recorder", num_channels, audio_codec_context->frame_size, audio_codec_context_get_audio_format(audio_codec_context)) != 0) {
- gsr_log(GSR_LOG_LEVEL_ERROR, "failed to setup audio recording to combined sink");
- _exit(1);
- }
-
- std::vector<const char*> audio_devices_sources;
- for(size_t i = 0; i < merged_audio_inputs.num_items; ++i) {
- const gsr_audio_input &audio_input = merged_audio_inputs.items[i];
- if(audio_input.type == GSR_AUDIO_INPUT_TYPE_DEVICE)
- audio_devices_sources.push_back(audio_input.name);
- }
-
- bool app_audio_inverted = false;
- std::vector<const char*> app_names;
- for(size_t i = 0; i < merged_audio_inputs.num_items; ++i) {
- const gsr_audio_input &audio_input = merged_audio_inputs.items[i];
- if(audio_input.type == GSR_AUDIO_INPUT_TYPE_APPLICATION) {
- app_names.push_back(audio_input.name);
- app_audio_inverted = audio_input.inverted;
- }
- }
-
- if(!audio_devices_sources.empty()) {
- if(!gsr_pipewire_audio_add_link_from_sources_to_stream(pipewire_audio, audio_devices_sources.data(), audio_devices_sources.size(), combined_sink_name.c_str())) {
- gsr_log(GSR_LOG_LEVEL_ERROR, "failed to add application audio link");
- _exit(1);
- }
- }
-
- if(app_audio_inverted) {
- if(!gsr_pipewire_audio_add_link_from_apps_to_stream_inverted(pipewire_audio, app_names.data(), app_names.size(), combined_sink_name.c_str())) {
- gsr_log(GSR_LOG_LEVEL_ERROR, "failed to add application audio link");
- _exit(1);
- }
- } else {
- if(!gsr_pipewire_audio_add_link_from_apps_to_stream(pipewire_audio, app_names.data(), app_names.size(), combined_sink_name.c_str())) {
- gsr_log(GSR_LOG_LEVEL_ERROR, "failed to add application audio link");
- _exit(1);
- }
- }
-
- return audio_device;
-}
#endif
static bool get_image_format_from_filename(const char *filename, gsr_image_format *image_format) {
@@ -1619,7 +1375,6 @@ int main(int argc, char **argv) {
}
AVStream *video_stream = nullptr;
- std::vector<AudioTrack> audio_tracks;
if(arg_parser.settings.video_encoder == GSR_VIDEO_ENCODER_HW_CPU && arg_parser.settings.video_codec != (gsr_video_codec)GSR_VIDEO_CODEC_AUTO && arg_parser.settings.video_codec != GSR_VIDEO_CODEC_H264) {
gsr_log(GSR_LOG_LEVEL_ERROR, "-encoder cpu was specified but a codec other than h264 was specified. -encoder cpu supports only h264 at the moment");
@@ -1652,6 +1407,10 @@ int main(int argc, char **argv) {
video_frame->chroma_location = video_codec_context->chroma_sample_location;
const size_t estimated_replay_buffer_packets = calculate_estimated_replay_buffer_packets(arg_parser.settings.replay_buffer_size_secs, arg_parser.settings.fps, arg_parser.settings.audio_codec, &requested_audio_inputs);
+ gsr_recording_clock *recording_clock = gsr_recording_clock_create();
+ if(!recording_clock)
+ _exit(1);
+
gsr_encoder encoder;
if(!gsr_encoder_init(&encoder, arg_parser.settings.replay_storage, estimated_replay_buffer_packets, arg_parser.settings.replay_buffer_size_secs, arg_parser.settings.filename)) {
gsr_log(GSR_LOG_LEVEL_ERROR, "failed to create encoder");
@@ -1716,6 +1475,10 @@ int main(int argc, char **argv) {
}
}
+ gsr_audio_capture audio_capture;
+ if(gsr_audio_capture_init(&audio_capture, &encoder, recording_clock, &running) != GSR_ERROR_OK)
+ _exit(1);
+
int audio_max_frame_size = 1024;
int audio_stream_index = VIDEO_STREAM_INDEX + 1;
for(size_t audio_track_index = 0; audio_track_index < requested_audio_inputs.num_items; ++audio_track_index) {
@@ -1732,7 +1495,7 @@ int main(int argc, char **argv) {
gsr_log(GSR_LOG_LEVEL_ERROR, "added too many audio sources");
}
- if(audio_stream && !merged_audio_inputs.track_name[0] == '\0' && !arg_parser.settings.exclude_metadata)
+ if(audio_stream && merged_audio_inputs.track_name[0] != '\0' && !arg_parser.settings.exclude_metadata)
av_dict_set(&audio_stream->metadata, "title", merged_audio_inputs.track_name, 0);
if(!open_audio(audio_codec_context, arg_parser.settings.ffmpeg_audio_opts))
@@ -1748,12 +1511,16 @@ int main(int argc, char **argv) {
//audio_frame->sample_rate = audio_codec_context->sample_rate;
- std::vector<AVFilterContext*> src_filter_ctx;
+ AVFilterContext *src_filter_ctx[GSR_MAX_AUDIO_SOURCES_PER_TRACK];
AVFilterGraph *graph = nullptr;
AVFilterContext *sink = nullptr;
if(use_amix) {
- int err = init_filter_graph(audio_codec_context, &graph, &sink, src_filter_ctx, merged_audio_inputs.num_items);
- if(err < 0) {
+ if(merged_audio_inputs.num_items > GSR_MAX_AUDIO_SOURCES_PER_TRACK) {
+ gsr_log(GSR_LOG_LEVEL_ERROR, "too many audio sources for one audio track, the maximum is %d", GSR_MAX_AUDIO_SOURCES_PER_TRACK);
+ _exit(1);
+ }
+
+ if(gsr_audio_init_filter_graph(audio_codec_context, &graph, &sink, src_filter_ctx, merged_audio_inputs.num_items) < 0) {
gsr_log(GSR_LOG_LEVEL_ERROR, "failed to create audio filter");
_exit(1);
}
@@ -1767,28 +1534,34 @@ int main(int argc, char **argv) {
const double audio_startup_time_seconds = force_no_audio_offset ? 0 : audio_codec_get_desired_delay(arg_parser.settings.audio_codec, arg_parser.settings.fps);// * ((double)audio_codec_context->frame_size / 1024.0);
const double num_audio_frames_shift = audio_startup_time_seconds / timeout_sec;
- std::vector<AudioDeviceData> audio_track_audio_devices;
+ gsr_audio_track audio_track;
+ memset(&audio_track, 0, sizeof(audio_track));
+ snprintf(audio_track.name, sizeof(audio_track.name), "%s", merged_audio_inputs.track_name);
+ audio_track.codec_context = audio_codec_context;
+ audio_track.graph = graph;
+ audio_track.sink = sink;
+ audio_track.stream_index = audio_stream_index;
+ audio_track.pts = -audio_codec_context->frame_size * num_audio_frames_shift;
+
+ int audio_track_result = GSR_ERROR_OK;
if(gsr_audio_inputs_has_app_audio(&merged_audio_inputs)) {
assert(!use_amix);
#ifdef GSR_APP_AUDIO
- audio_track_audio_devices.push_back(create_application_audio_audio_input(merged_audio_inputs, audio_codec_context, num_channels, num_audio_frames_shift, &pipewire_audio));
+ audio_track_result = gsr_audio_track_init_application_input(&audio_track, &merged_audio_inputs, audio_codec_context, num_channels, num_audio_frames_shift, &pipewire_audio);
#endif
} else {
- audio_track_audio_devices = create_device_audio_inputs(&merged_audio_inputs, audio_codec_context, num_channels, num_audio_frames_shift, src_filter_ctx, use_amix);
+ audio_track_result = gsr_audio_track_init_device_inputs(&audio_track, &merged_audio_inputs, audio_codec_context, num_channels, num_audio_frames_shift, src_filter_ctx, use_amix);
}
- AudioTrack audio_track;
- audio_track.name = merged_audio_inputs.track_name;
- audio_track.codec_context = audio_codec_context;
- audio_track.audio_devices = std::move(audio_track_audio_devices);
- audio_track.graph = graph;
- audio_track.sink = sink;
- audio_track.stream_index = audio_stream_index;
- audio_track.pts = -audio_codec_context->frame_size * num_audio_frames_shift;
- audio_tracks.push_back(std::move(audio_track));
+ if(audio_track_result != GSR_ERROR_OK)
+ _exit(gsr_error_to_exit_code(audio_track_result));
+
+ if(!gsr_audio_capture_add_track(&audio_capture, &audio_track))
+ _exit(1);
++audio_stream_index;
- audio_max_frame_size = std::max(audio_max_frame_size, audio_codec_context->frame_size);
+ if(audio_codec_context->frame_size > audio_max_frame_size)
+ audio_max_frame_size = audio_codec_context->frame_size;
}
//av_dump_format(av_format_context, 0, filename, 1);
@@ -1810,230 +1583,17 @@ int main(int argc, char **argv) {
int damage_fps_counter = 0;
bool paused = false;
- std::atomic<double> paused_time_offset(0.0);
- double paused_time_start = 0.0;
bool replay_recording = false;
RecordingStartResult replay_recording_start_result;
std::vector<size_t> replay_recording_items;
std::string replay_recording_filepath;
bool force_iframe_frame = false; // Only needed for video since audio frames are always iframes
- std::mutex audio_filter_mutex;
+ gsr_recording_clock_start(recording_clock);
+ const double record_start_time = gsr_recording_clock_get_start_time(recording_clock);
- const double record_start_time = clock_get_monotonic_seconds();
-
- const size_t audio_buffer_size = audio_max_frame_size * 4 * 2; // max 4 bytes/sample, 2 channels
- uint8_t *empty_audio = (uint8_t*)malloc(audio_buffer_size);
- if(!empty_audio) {
- gsr_log(GSR_LOG_LEVEL_ERROR, "failed to create empty audio");
+ if(gsr_audio_capture_start(&audio_capture, audio_max_frame_size, uses_amix) != GSR_ERROR_OK)
_exit(1);
- }
- memset(empty_audio, 0, audio_buffer_size);
-
- for(AudioTrack &audio_track : audio_tracks) {
- for(AudioDeviceData &audio_device : audio_track.audio_devices) {
- audio_device.thread = std::thread([&]() mutable {
- const AVSampleFormat sound_device_sample_format = audio_format_to_sample_format(audio_codec_context_get_audio_format(audio_track.codec_context));
- // TODO: Always do conversion for now. This fixes issue with stuttering audio on pulseaudio with opus + multiple audio sources merged
- const bool needs_audio_conversion = true;//audio_track.codec_context->sample_fmt != sound_device_sample_format;
- SwrContext *swr = nullptr;
- if(needs_audio_conversion) {
- swr = swr_alloc();
- if(!swr) {
- fprintf(stderr, "Failed to create SwrContext\n");
- _exit(1);
- }
- #if LIBAVUTIL_VERSION_MAJOR <= 56
- av_opt_set_channel_layout(swr, "in_channel_layout", AV_CH_LAYOUT_STEREO, 0);
- av_opt_set_channel_layout(swr, "out_channel_layout", AV_CH_LAYOUT_STEREO, 0);
- #elif LIBAVUTIL_VERSION_MAJOR >= 59
- av_opt_set_chlayout(swr, "in_chlayout", &audio_track.codec_context->ch_layout, 0);
- av_opt_set_chlayout(swr, "out_chlayout", &audio_track.codec_context->ch_layout, 0);
- #else
- av_opt_set_chlayout(swr, "in_channel_layout", &audio_track.codec_context->ch_layout, 0);
- av_opt_set_chlayout(swr, "out_channel_layout", &audio_track.codec_context->ch_layout, 0);
- #endif
- av_opt_set_int(swr, "in_sample_rate", audio_track.codec_context->sample_rate, 0);
- av_opt_set_int(swr, "out_sample_rate", audio_track.codec_context->sample_rate, 0);
- av_opt_set_sample_fmt(swr, "in_sample_fmt", sound_device_sample_format, 0);
- av_opt_set_sample_fmt(swr, "out_sample_fmt", audio_track.codec_context->sample_fmt, 0);
- swr_init(swr);
- }
-
- const double audio_fps = (double)audio_track.codec_context->sample_rate / (double)audio_track.codec_context->frame_size;
- const int64_t timeout_ms = std::round(1000.0 / audio_fps);
- const double timeout_sec = 1000.0 / audio_fps / 1000.0;
- int64_t num_received_frames = 0;
-
- // The sound device is opened before the recording starts, so it can contain old audio from before the recording started.
- // Discard it so the recording doesn't start with old audio.
- if(audio_device.sound_device.handle)
- sound_device_flush(&audio_device.sound_device);
-
- while(running) {
- void *sound_buffer;
- int sound_buffer_size = -1;
- const double time_before_read_seconds = clock_get_monotonic_seconds();
- if(audio_device.sound_device.handle) {
- // TODO: use this instead of calculating time to read. But this can fluctuate and we dont want to go back in time,
- // also it's 0.0 for some users???
- double latency_seconds = 0.0;
- sound_buffer_size = sound_device_read_next_chunk(&audio_device.sound_device, &sound_buffer, timeout_sec * 2.0, &latency_seconds);
- }
-
- const bool got_audio_data = sound_buffer_size >= 0;
- //fprintf(stderr, "got audio data: %s\n", got_audio_data ? "yes" : "no");
- //fprintf(stderr, "time to read: %f, %s, %f\n", time_to_read_seconds, got_audio_data ? "yes" : "no", timeout_sec);
- const double this_audio_frame_time = clock_get_monotonic_seconds() - paused_time_offset;
-
- if(paused) {
- if(!audio_device.sound_device.handle)
- av_usleep(timeout_ms * 1000);
-
- continue;
- }
-
- int ret = av_frame_make_writable(audio_device.frame);
- if (ret < 0) {
- fprintf(stderr, "Failed to make audio frame writable\n");
- break;
- }
-
- // TODO: Is this |received_audio_time| really correct?
- const int64_t num_expected_frames = std::floor((this_audio_frame_time - record_start_time) / timeout_sec);
- int64_t num_missing_frames = std::max((int64_t)0LL, num_expected_frames - num_received_frames);
-
- if(got_audio_data)
- num_missing_frames = std::max((int64_t)0LL, num_missing_frames - 1);
-
- if(!audio_device.sound_device.handle)
- num_missing_frames = std::max((int64_t)1, num_missing_frames);
-
- // Fucking hell is there a better way to do this? I JUST WANT TO KEEP VIDEO AND AUDIO SYNCED HOLY FUCK I WANT TO KILL MYSELF NOW.
- // THIS PIECE OF SHIT WANTS EMPTY FRAMES OTHERWISE VIDEO PLAYS TOO FAST TO KEEP UP WITH AUDIO OR THE AUDIO PLAYS TOO EARLY.
- // BUT WE CANT USE DELAYS TO GIVE DUMMY DATA BECAUSE PULSEAUDIO MIGHT GIVE AUDIO A BIG DELAYED!!!
- // This garbage is needed because we want to produce constant frame rate videos instead of variable frame rate
- // videos because bad software such as video editing software and VLC do not support variable frame rate software,
- // despite nvidia shadowplay and xbox game bar producing variable frame rate videos.
- // So we have to make sure we produce frames at the same relative rate as the video.
- if((num_missing_frames >= 1 && got_audio_data) || num_missing_frames >= 5 || !audio_device.sound_device.handle) {
- // Fill the missing frames with silence. Duplicating the previous audio frame to fill the gap instead
- // sounds like a stutter and it's especially noticeable at the start of the recording when the audio device
- // hasn't started to deliver audio at a stable rate yet, which repeats the first audio frame multiple times.
- if(needs_audio_conversion)
- swr_convert(swr, &audio_device.frame->data[0], audio_track.codec_context->frame_size, (const uint8_t**)&empty_audio, audio_track.codec_context->frame_size);
- else
- audio_device.frame->data[0] = empty_audio;
-
- // TODO: Check if duplicate frame can be saved just by writing it with a different pts instead of sending it again
- std::lock_guard<std::mutex> lock(audio_filter_mutex);
- for(int i = 0; i < num_missing_frames; ++i) {
- if(audio_track.graph) {
- // TODO: av_buffersrc_add_frame
- if(av_buffersrc_write_frame(audio_device.src_filter_ctx, audio_device.frame) < 0) {
- gsr_log(GSR_LOG_LEVEL_ERROR, "failed to add audio frame to filter");
- }
- } else {
- ret = avcodec_send_frame(audio_track.codec_context, audio_device.frame);
- if(ret >= 0) {
- // TODO: Move to separate thread because this could write to network (for example when livestreaming)
- gsr_encoder_receive_packets(&encoder, audio_track.codec_context, audio_device.frame->pts, audio_track.stream_index);
- } else {
- fprintf(stderr, "Failed to encode audio!\n");
- }
- audio_track.pts += audio_track.codec_context->frame_size;
- }
-
- audio_device.frame->pts += audio_track.codec_context->frame_size;
- num_received_frames++;
- }
- }
-
- if(!audio_device.sound_device.handle) {
- av_usleep(timeout_ms * 1000);
- } else if(got_audio_data) {
- // The frame has to be made writable again if the frame was already sent to the audio filter above (when filling missing frames)
- // because the audio filter only references the frame data instead of copying it. Without this the sent frames data would be
- // overwritten with the audio data below, causing the audio to repeat instead of the missing frames being silent.
- ret = av_frame_make_writable(audio_device.frame);
- if (ret < 0) {
- fprintf(stderr, "Failed to make audio frame writable\n");
- break;
- }
-
- // TODO: Instead of converting audio, get float audio from alsa. Or does alsa do conversion internally to get this format?
- if(needs_audio_conversion)
- swr_convert(swr, &audio_device.frame->data[0], audio_track.codec_context->frame_size, (const uint8_t**)&sound_buffer, audio_track.codec_context->frame_size);
- else
- audio_device.frame->data[0] = (uint8_t*)sound_buffer;
-
- std::lock_guard<std::mutex> lock(audio_filter_mutex);
-
- if(audio_track.graph) {
- // TODO: av_buffersrc_add_frame
- if(av_buffersrc_write_frame(audio_device.src_filter_ctx, audio_device.frame) < 0) {
- gsr_log(GSR_LOG_LEVEL_ERROR, "failed to add audio frame to filter");
- }
- } else {
- ret = avcodec_send_frame(audio_track.codec_context, audio_device.frame);
- if(ret >= 0) {
- // TODO: Move to separate thread because this could write to network (for example when livestreaming)
- gsr_encoder_receive_packets(&encoder, audio_track.codec_context, audio_device.frame->pts, audio_track.stream_index);
- } else {
- fprintf(stderr, "Failed to encode audio!\n");
- }
- audio_track.pts += audio_track.codec_context->frame_size;
- }
-
- audio_device.frame->pts += audio_track.codec_context->frame_size;
- num_received_frames++;
- } else {
- // TODO: Maybe sleep for time_to_sleep_until_next_frame/4? for better latency
- const double time_after_read_seconds = clock_get_monotonic_seconds();
- const double time_to_read_seconds = time_after_read_seconds - time_before_read_seconds;
- const double time_to_sleep_until_next_frame = timeout_sec - time_to_read_seconds;
- if(time_to_sleep_until_next_frame > 0.0)
- av_usleep(time_to_sleep_until_next_frame * 1000ULL * 1000ULL);
- }
- }
-
- if(swr)
- swr_free(&swr);
- });
- }
- }
-
- std::thread amix_thread;
- if(uses_amix) {
- amix_thread = std::thread([&]() {
- AVFrame *aframe = av_frame_alloc();
- while(running) {
- {
- std::lock_guard<std::mutex> lock(audio_filter_mutex);
- for(AudioTrack &audio_track : audio_tracks) {
- if(!audio_track.sink)
- continue;
-
- int err = 0;
- while ((err = av_buffersink_get_frame(audio_track.sink, aframe)) >= 0) {
- aframe->pts = audio_track.pts;
- err = avcodec_send_frame(audio_track.codec_context, aframe);
- if(err >= 0){
- // TODO: Move to separate thread because this could write to network (for example when livestreaming)
- gsr_encoder_receive_packets(&encoder, audio_track.codec_context, aframe->pts, audio_track.stream_index);
- } else {
- fprintf(stderr, "Failed to encode audio!\n");
- }
- av_frame_unref(aframe);
- audio_track.pts += audio_track.codec_context->frame_size;
- }
- }
- }
- av_usleep(5 * 1000); // 5 milliseconds
- }
- av_frame_free(&aframe);
- });
- }
// Set update_fps to 24 to test if duplicate/delayed frames cause video/audio desync or too fast/slow video.
//const double update_fps = fps + 190;
@@ -2146,7 +1706,7 @@ int main(int argc, char **argv) {
damage_fps_counter = 0;
}
- const double this_video_frame_time = clock_get_monotonic_seconds() - paused_time_offset;
+ const double this_video_frame_time = gsr_recording_clock_get_time(recording_clock);
const int64_t expected_frames = std::floor((this_video_frame_time - record_start_time) / target_fps);
const int64_t num_missed_frames = expected_frames - video_pts_counter;
@@ -2169,8 +1729,8 @@ int main(int argc, char **argv) {
if(video_source.capture->capture_has_synchronous_task) {
capture_has_synchronous_task = video_source.capture->capture_has_synchronous_task(video_source.capture);
if(capture_has_synchronous_task) {
- paused_time_start = clock_get_monotonic_seconds();
paused = true;
+ gsr_recording_clock_set_paused(recording_clock, true);
}
}
}
@@ -2202,8 +1762,8 @@ int main(int argc, char **argv) {
}
if(capture_has_synchronous_task) {
- paused_time_offset = paused_time_offset + (clock_get_monotonic_seconds() - paused_time_start);
paused = false;
+ gsr_recording_clock_set_paused(recording_clock, false);
}
gsr_egl_swap_buffers(&egl);
@@ -2250,17 +1810,10 @@ int main(int argc, char **argv) {
}
if(toggle_pause == 1 && !arg_parser.settings.is_replaying) {
- const bool new_paused_state = !paused;
- if(new_paused_state) {
- paused_time_start = clock_get_monotonic_seconds();
- fprintf(stderr, "Paused\n");
- } else {
- paused_time_offset = paused_time_offset + (clock_get_monotonic_seconds() - paused_time_start);
- fprintf(stderr, "Unpaused\n");
- }
-
- toggle_pause = 0;
paused = !paused;
+ gsr_recording_clock_set_paused(recording_clock, paused);
+ gsr_log(GSR_LOG_LEVEL_INFO, paused ? "Paused" : "Unpaused");
+ toggle_pause = 0;
}
if(toggle_replay_recording && !arg_parser.settings.replay_recording_directory) {
@@ -2273,10 +1826,10 @@ int main(int argc, char **argv) {
toggle_replay_recording = 0;
const bool new_replay_recording_state = !replay_recording;
if(new_replay_recording_state) {
- std::lock_guard<std::mutex> lock(audio_filter_mutex);
+ gsr_audio_capture_lock_filter(&audio_capture);
replay_recording_items.clear();
replay_recording_filepath = create_new_recording_filepath_from_timestamp(arg_parser.settings.replay_recording_directory, "Video", file_extension, arg_parser.settings.date_folders);
- replay_recording_start_result = start_recording_create_streams(replay_recording_filepath.c_str(), arg_parser, video_codec_context, audio_tracks, hdr, video_sources);
+ replay_recording_start_result = start_recording_create_streams(replay_recording_filepath.c_str(), arg_parser, video_codec_context, &audio_capture, hdr, video_sources);
if(replay_recording_start_result.av_format_context) {
const size_t video_recording_destination_id = gsr_encoder_add_recording_destination(&encoder, video_codec_context, replay_recording_start_result.av_format_context, replay_recording_start_result.video_stream, video_frame->pts);
if(arg_parser.settings.write_first_frame_ts && video_recording_destination_id != (size_t)-1) {
@@ -2343,7 +1896,7 @@ int main(int argc, char **argv) {
save_replay_seconds = 0;
save_replay_output_filepath.clear();
- const bool replay_start_result = save_replay_async(video_codec_context, VIDEO_STREAM_INDEX, audio_tracks, &encoder, arg_parser, file_extension, arg_parser.settings.date_folders, hdr, video_sources, current_save_replay_seconds);
+ const bool replay_start_result = save_replay_async(video_codec_context, VIDEO_STREAM_INDEX, &audio_capture, &encoder, arg_parser, file_extension, arg_parser.settings.date_folders, hdr, video_sources, current_save_replay_seconds);
if(!replay_start_result) {
printf("gsr error: Failed to save replay\n");
fflush(stdout);
@@ -2356,7 +1909,7 @@ int main(int argc, char **argv) {
}
}
- const double time_at_frame_end = clock_get_monotonic_seconds() - paused_time_offset;
+ const double time_at_frame_end = gsr_recording_clock_get_time(recording_clock);
const double time_elapsed_total = time_at_frame_end - record_start_time;
const int64_t frames_elapsed = std::floor(time_elapsed_total / target_fps);
const double time_at_next_frame = (frames_elapsed + 1) * target_fps;
@@ -2409,15 +1962,7 @@ int main(int argc, char **argv) {
}
}
- for(AudioTrack &audio_track : audio_tracks) {
- for(auto &audio_device : audio_track.audio_devices) {
- audio_device.thread.join();
- sound_device_close(&audio_device.sound_device);
- }
- }
-
- if(amix_thread.joinable())
- amix_thread.join();
+ gsr_audio_capture_join_threads(&audio_capture);
// TODO: Replace this with start_recording_create_steams
if(!arg_parser.settings.is_replaying && av_write_trailer(av_format_context) != 0) {
@@ -2451,7 +1996,6 @@ int main(int argc, char **argv) {
//gsr_window_destroy(&window);
//av_frame_free(&video_frame);
- free(empty_audio);
args_parser_deinit(&arg_parser);
// We do an _exit here because cuda uses at_exit to do _something_ that causes the program to freeze,
// but only on some nvidia driver versions on some gpus (RTX?), and _exit exits the program without calling
diff --git a/src/recorder/audio_capture.c b/src/recorder/audio_capture.c
new file mode 100644
index 0000000..ee1c75f
--- /dev/null
+++ b/src/recorder/audio_capture.c
@@ -0,0 +1,610 @@
+#include "../../include/recorder/audio_capture.h"
+#include "../../include/recorder/error.h"
+#include "../../include/recorder/audio_codec.h"
+#include "../../include/utils.h"
+#include "../../include/log.h"
+
+#include <string.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <math.h>
+
+#include <libavutil/opt.h>
+#include <libavutil/time.h>
+#include <libswresample/swresample.h>
+#include <libavfilter/buffersink.h>
+#include <libavfilter/buffersrc.h>
+
+int gsr_audio_init_filter_graph(AVCodecContext *audio_codec_context, AVFilterGraph **graph, AVFilterContext **sink, AVFilterContext **src_filter_ctx, size_t num_sources) {
+ char ch_layout[64];
+ int err = 0;
+ ch_layout[0] = '\0';
+
+ // C89-style variable declaration to
+ // avoid problems because of goto
+ AVFilterGraph* filter_graph = NULL;
+ AVFilterContext* mix_ctx = NULL;
+
+ const AVFilter* mix_filter = NULL;
+ const AVFilter* abuffersink = NULL;
+ AVFilterContext* abuffersink_ctx = NULL;
+ char args[512] = { 0 };
+#if LIBAVFILTER_VERSION_INT >= AV_VERSION_INT(7, 107, 100)
+ bool normalize = false;
+#endif
+
+ filter_graph = avfilter_graph_alloc();
+ if (!filter_graph) {
+ gsr_log(GSR_LOG_LEVEL_ERROR, "Unable to create filter graph");
+ err = AVERROR(ENOMEM);
+ goto fail;
+ }
+
+ for(size_t i = 0; i < num_sources; ++i) {
+ const AVFilter *abuffer = avfilter_get_by_name("abuffer");
+ if (!abuffer) {
+ gsr_log(GSR_LOG_LEVEL_ERROR, "Could not find the abuffer filter");
+ err = AVERROR_FILTER_NOT_FOUND;
+ goto fail;
+ }
+
+ AVFilterContext *abuffer_ctx = avfilter_graph_alloc_filter(filter_graph, abuffer, NULL);
+ if (!abuffer_ctx) {
+ gsr_log(GSR_LOG_LEVEL_ERROR, "Could not allocate the abuffer instance");
+ err = AVERROR(ENOMEM);
+ goto fail;
+ }
+
+ #if LIBAVCODEC_VERSION_MAJOR < 60
+ av_get_channel_layout_string(ch_layout, sizeof(ch_layout), 0, AV_CH_LAYOUT_STEREO);
+ #else
+ av_channel_layout_describe(&audio_codec_context->ch_layout, ch_layout, sizeof(ch_layout));
+ #endif
+ av_opt_set (abuffer_ctx, "channel_layout", ch_layout, AV_OPT_SEARCH_CHILDREN);
+ av_opt_set (abuffer_ctx, "sample_fmt", av_get_sample_fmt_name(audio_codec_context->sample_fmt), AV_OPT_SEARCH_CHILDREN);
+ av_opt_set_q (abuffer_ctx, "time_base", audio_codec_context->time_base, AV_OPT_SEARCH_CHILDREN);
+ av_opt_set_int(abuffer_ctx, "sample_rate", audio_codec_context->sample_rate, AV_OPT_SEARCH_CHILDREN);
+ av_opt_set_int(abuffer_ctx, "bit_rate", audio_codec_context->bit_rate, AV_OPT_SEARCH_CHILDREN);
+
+ err = avfilter_init_str(abuffer_ctx, NULL);
+ if (err < 0) {
+ gsr_log(GSR_LOG_LEVEL_ERROR, "Could not initialize the abuffer filter");
+ goto fail;
+ }
+
+ src_filter_ctx[i] = abuffer_ctx;
+ }
+
+ mix_filter = avfilter_get_by_name("amix");
+ if (!mix_filter) {
+ av_log(NULL, AV_LOG_ERROR, "Could not find the mix filter.\n");
+ err = AVERROR_FILTER_NOT_FOUND;
+ goto fail;
+ }
+
+#if LIBAVFILTER_VERSION_INT >= AV_VERSION_INT(7, 107, 100)
+ snprintf(args, sizeof(args), "inputs=%d:normalize=%s", (int)num_sources, normalize ? "true" : "false");
+#else
+ snprintf(args, sizeof(args), "inputs=%d", (int)num_sources);
+ gsr_log(GSR_LOG_LEVEL_WARNING, "your ffmpeg version doesn't support disabling normalizing of mixed audio. Volume might be lower than expected");
+#endif
+
+ err = avfilter_graph_create_filter(&mix_ctx, mix_filter, "amix", args, NULL, filter_graph);
+ if (err < 0) {
+ av_log(NULL, AV_LOG_ERROR, "Cannot create audio amix filter\n");
+ goto fail;
+ }
+
+ abuffersink = avfilter_get_by_name("abuffersink");
+ if (!abuffersink) {
+ gsr_log(GSR_LOG_LEVEL_ERROR, "Could not find the abuffersink filter");
+ err = AVERROR_FILTER_NOT_FOUND;
+ goto fail;
+ }
+
+ abuffersink_ctx = avfilter_graph_alloc_filter(filter_graph, abuffersink, "sink");
+ if (!abuffersink_ctx) {
+ gsr_log(GSR_LOG_LEVEL_ERROR, "Could not allocate the abuffersink instance");
+ err = AVERROR(ENOMEM);
+ goto fail;
+ }
+
+ err = avfilter_init_str(abuffersink_ctx, NULL);
+ if (err < 0) {
+ gsr_log(GSR_LOG_LEVEL_ERROR, "Could not initialize the abuffersink instance");
+ goto fail;
+ }
+
+ err = 0;
+ for(size_t i = 0; i < num_sources; ++i) {
+ AVFilterContext *src_ctx = src_filter_ctx[i];
+ if (err >= 0)
+ err = avfilter_link(src_ctx, 0, mix_ctx, i);
+ }
+ if (err >= 0)
+ err = avfilter_link(mix_ctx, 0, abuffersink_ctx, 0);
+ if (err < 0) {
+ av_log(NULL, AV_LOG_ERROR, "Error connecting filters\n");
+ goto fail;
+ }
+
+ err = avfilter_graph_config(filter_graph, NULL);
+ if (err < 0) {
+ av_log(NULL, AV_LOG_ERROR, "Error configuring the filter graph\n");
+ goto fail;
+ }
+
+ /* Make sure the sink always outputs frames with the exact amount of samples the audio encoder wants,
+ otherwise the audio encoder rejects the frame and that piece of audio is lost */
+ av_buffersink_set_frame_size(abuffersink_ctx, audio_codec_context->frame_size);
+
+ *graph = filter_graph;
+ *sink = abuffersink_ctx;
+
+ return 0;
+
+fail:
+ avfilter_graph_free(&filter_graph);
+ memset(src_filter_ctx, 0, num_sources * sizeof(AVFilterContext*)); // possibly unnecessary?
+ return err;
+}
+
+static void* audio_device_thread(void *userdata) {
+ const gsr_audio_device_thread_userdata *thread_userdata = userdata;
+ gsr_audio_capture *self = thread_userdata->audio_capture;
+ gsr_audio_track *track = thread_userdata->track;
+ gsr_audio_device_capture *device = thread_userdata->device;
+ gsr_recording_clock *clock = self->clock;
+ const volatile sig_atomic_t *running = self->running;
+
+ const enum AVSampleFormat sound_device_sample_format = audio_format_to_sample_format(audio_codec_context_get_audio_format(track->codec_context));
+ /* TODO: Always do conversion for now. This fixes issue with stuttering audio on pulseaudio with opus + multiple audio sources merged */
+ const bool needs_audio_conversion = true;
+ SwrContext *swr = NULL;
+ if(needs_audio_conversion) {
+ swr = swr_alloc();
+ if(!swr) {
+ gsr_log(GSR_LOG_LEVEL_ERROR, "failed to create SwrContext");
+ return NULL;
+ }
+ #if LIBAVUTIL_VERSION_MAJOR <= 56
+ av_opt_set_channel_layout(swr, "in_channel_layout", AV_CH_LAYOUT_STEREO, 0);
+ av_opt_set_channel_layout(swr, "out_channel_layout", AV_CH_LAYOUT_STEREO, 0);
+ #elif LIBAVUTIL_VERSION_MAJOR >= 59
+ av_opt_set_chlayout(swr, "in_chlayout", &track->codec_context->ch_layout, 0);
+ av_opt_set_chlayout(swr, "out_chlayout", &track->codec_context->ch_layout, 0);
+ #else
+ av_opt_set_chlayout(swr, "in_channel_layout", &track->codec_context->ch_layout, 0);
+ av_opt_set_chlayout(swr, "out_channel_layout", &track->codec_context->ch_layout, 0);
+ #endif
+ av_opt_set_int(swr, "in_sample_rate", track->codec_context->sample_rate, 0);
+ av_opt_set_int(swr, "out_sample_rate", track->codec_context->sample_rate, 0);
+ av_opt_set_sample_fmt(swr, "in_sample_fmt", sound_device_sample_format, 0);
+ av_opt_set_sample_fmt(swr, "out_sample_fmt", track->codec_context->sample_fmt, 0);
+ swr_init(swr);
+ }
+
+ const double audio_fps = (double)track->codec_context->sample_rate / (double)track->codec_context->frame_size;
+ const int64_t timeout_ms = llround(1000.0 / audio_fps);
+ const double timeout_sec = 1000.0 / audio_fps / 1000.0;
+ int64_t num_received_frames = 0;
+
+ /* The sound device is opened before the recording starts, so it can contain old audio from before the recording started.
+ Discard it so the recording doesn't start with old audio. */
+ if(device->sound_device.handle)
+ sound_device_flush(&device->sound_device);
+
+ while(*running) {
+ void *sound_buffer;
+ int sound_buffer_size = -1;
+ const double time_before_read_seconds = clock_get_monotonic_seconds();
+ if(device->sound_device.handle) {
+ // TODO: use this instead of calculating time to read. But this can fluctuate and we dont want to go back in time,
+ // also it's 0.0 for some users???
+ double latency_seconds = 0.0;
+ sound_buffer_size = sound_device_read_next_chunk(&device->sound_device, &sound_buffer, timeout_sec * 2.0, &latency_seconds);
+ }
+
+ const bool got_audio_data = sound_buffer_size >= 0;
+ //fprintf(stderr, "got audio data: %s\n", got_audio_data ? "yes" : "no");
+ //fprintf(stderr, "time to read: %f, %s, %f\n", time_to_read_seconds, got_audio_data ? "yes" : "no", timeout_sec);
+ const double this_audio_frame_time = gsr_recording_clock_get_time(clock);
+
+ if(gsr_recording_clock_is_paused(clock)) {
+ if(!device->sound_device.handle)
+ av_usleep(timeout_ms * 1000);
+
+ continue;
+ }
+
+ int ret = av_frame_make_writable(device->frame);
+ if (ret < 0) {
+ gsr_log(GSR_LOG_LEVEL_ERROR, "Failed to make audio frame writable");
+ break;
+ }
+
+ // TODO: Is this |received_audio_time| really correct?
+ const int64_t num_expected_frames = floor((this_audio_frame_time - gsr_recording_clock_get_start_time(clock)) / timeout_sec);
+ int64_t num_missing_frames = num_expected_frames > num_received_frames ? num_expected_frames - num_received_frames : 0;
+
+ if(got_audio_data)
+ num_missing_frames = num_missing_frames > 1 ? num_missing_frames - 1 : 0;
+
+ if(!device->sound_device.handle)
+ num_missing_frames = num_missing_frames < 1 ? 1 : num_missing_frames;
+
+ // Fucking hell is there a better way to do this? I JUST WANT TO KEEP VIDEO AND AUDIO SYNCED HOLY FUCK I WANT TO KILL MYSELF NOW.
+ // THIS PIECE OF SHIT WANTS EMPTY FRAMES OTHERWISE VIDEO PLAYS TOO FAST TO KEEP UP WITH AUDIO OR THE AUDIO PLAYS TOO EARLY.
+ // BUT WE CANT USE DELAYS TO GIVE DUMMY DATA BECAUSE PULSEAUDIO MIGHT GIVE AUDIO A BIG DELAYED!!!
+ // This garbage is needed because we want to produce constant frame rate videos instead of variable frame rate
+ // videos because bad software such as video editing software and VLC do not support variable frame rate software,
+ // despite nvidia shadowplay and xbox game bar producing variable frame rate videos.
+ // So we have to make sure we produce frames at the same relative rate as the video.
+ if((num_missing_frames >= 1 && got_audio_data) || num_missing_frames >= 5 || !device->sound_device.handle) {
+ // Fill the missing frames with silence. Duplicating the previous audio frame to fill the gap instead
+ // sounds like a stutter and it's especially noticeable at the start of the recording when the audio device
+ // hasn't started to deliver audio at a stable rate yet, which repeats the first audio frame multiple times.
+ if(needs_audio_conversion)
+ swr_convert(swr, &device->frame->data[0], track->codec_context->frame_size, (const uint8_t**)&self->empty_audio, track->codec_context->frame_size);
+ else
+ device->frame->data[0] = self->empty_audio;
+
+ // TODO: Check if duplicate frame can be saved just by writing it with a different pts instead of sending it again
+ pthread_mutex_lock(&self->filter_mutex);
+ for(int i = 0; i < num_missing_frames; ++i) {
+ if(track->graph) {
+ // TODO: av_buffersrc_add_frame
+ if(av_buffersrc_write_frame(device->src_filter_ctx, device->frame) < 0) {
+ gsr_log(GSR_LOG_LEVEL_ERROR, "failed to add audio frame to filter");
+ }
+ } else {
+ ret = avcodec_send_frame(track->codec_context, device->frame);
+ if(ret >= 0) {
+ // TODO: Move to separate thread because this could write to network (for example when livestreaming)
+ gsr_encoder_receive_packets(self->encoder, track->codec_context, device->frame->pts, track->stream_index);
+ } else {
+ gsr_log(GSR_LOG_LEVEL_ERROR, "Failed to encode audio");
+ }
+ track->pts += track->codec_context->frame_size;
+ }
+
+ device->frame->pts += track->codec_context->frame_size;
+ num_received_frames++;
+ }
+ pthread_mutex_unlock(&self->filter_mutex);
+ }
+
+ if(!device->sound_device.handle) {
+ av_usleep(timeout_ms * 1000);
+ } else if(got_audio_data) {
+ // The frame has to be made writable again if the frame was already sent to the audio filter above (when filling missing frames)
+ // because the audio filter only references the frame data instead of copying it. Without this the sent frames data would be
+ // overwritten with the audio data below, causing the audio to repeat instead of the missing frames being silent.
+ ret = av_frame_make_writable(device->frame);
+ if (ret < 0) {
+ gsr_log(GSR_LOG_LEVEL_ERROR, "Failed to make audio frame writable");
+ break;
+ }
+
+ // TODO: Instead of converting audio, get float audio from alsa. Or does alsa do conversion internally to get this format?
+ if(needs_audio_conversion)
+ swr_convert(swr, &device->frame->data[0], track->codec_context->frame_size, (const uint8_t**)&sound_buffer, track->codec_context->frame_size);
+ else
+ device->frame->data[0] = (uint8_t*)sound_buffer;
+
+ pthread_mutex_lock(&self->filter_mutex);
+
+ if(track->graph) {
+ // TODO: av_buffersrc_add_frame
+ if(av_buffersrc_write_frame(device->src_filter_ctx, device->frame) < 0) {
+ gsr_log(GSR_LOG_LEVEL_ERROR, "failed to add audio frame to filter");
+ }
+ } else {
+ ret = avcodec_send_frame(track->codec_context, device->frame);
+ if(ret >= 0) {
+ // TODO: Move to separate thread because this could write to network (for example when livestreaming)
+ gsr_encoder_receive_packets(self->encoder, track->codec_context, device->frame->pts, track->stream_index);
+ } else {
+ gsr_log(GSR_LOG_LEVEL_ERROR, "Failed to encode audio");
+ }
+ track->pts += track->codec_context->frame_size;
+ }
+
+ device->frame->pts += track->codec_context->frame_size;
+ num_received_frames++;
+ pthread_mutex_unlock(&self->filter_mutex);
+ } else {
+ // TODO: Maybe sleep for time_to_sleep_until_next_frame/4? for better latency
+ const double time_after_read_seconds = clock_get_monotonic_seconds();
+ const double time_to_read_seconds = time_after_read_seconds - time_before_read_seconds;
+ const double time_to_sleep_until_next_frame = timeout_sec - time_to_read_seconds;
+ if(time_to_sleep_until_next_frame > 0.0)
+ av_usleep(time_to_sleep_until_next_frame * 1000ULL * 1000ULL);
+ }
+ }
+
+ if(swr)
+ swr_free(&swr);
+
+ return NULL;
+}
+
+static void* amix_thread(void *userdata) {
+ gsr_audio_capture *self = userdata;
+ AVFrame *aframe = av_frame_alloc();
+ while(*self->running) {
+ pthread_mutex_lock(&self->filter_mutex);
+ for(size_t i = 0; i < self->num_tracks; ++i) {
+ gsr_audio_track *track = &self->tracks[i];
+ if(!track->sink)
+ continue;
+
+ int err = 0;
+ while((err = av_buffersink_get_frame(track->sink, aframe)) >= 0) {
+ aframe->pts = track->pts;
+ err = avcodec_send_frame(track->codec_context, aframe);
+ if(err >= 0) {
+ /* TODO: Move to separate thread because this could write to network (for example when livestreaming) */
+ gsr_encoder_receive_packets(self->encoder, track->codec_context, aframe->pts, track->stream_index);
+ } else {
+ gsr_log(GSR_LOG_LEVEL_ERROR, "Failed to encode audio");
+ }
+ av_frame_unref(aframe);
+ track->pts += track->codec_context->frame_size;
+ }
+ }
+ pthread_mutex_unlock(&self->filter_mutex);
+ av_usleep(5 * 1000); /* 5 milliseconds */
+ }
+ av_frame_free(&aframe);
+ return NULL;
+}
+
+int gsr_audio_capture_init(gsr_audio_capture *self, gsr_encoder *encoder, gsr_recording_clock *clock, const volatile sig_atomic_t *running) {
+ memset(self, 0, sizeof(*self));
+ self->encoder = encoder;
+ self->clock = clock;
+ self->running = running;
+
+ if(pthread_mutex_init(&self->filter_mutex, NULL) != 0) {
+ gsr_log(GSR_LOG_LEVEL_ERROR, "gsr_audio_capture_init: failed to initialize mutex");
+ return GSR_ERROR_GENERIC;
+ }
+ self->filter_mutex_initialized = true;
+
+ return GSR_ERROR_OK;
+}
+
+void gsr_audio_capture_deinit(gsr_audio_capture *self) {
+ gsr_audio_capture_join_threads(self);
+
+ for(size_t i = 0; i < self->num_tracks; ++i) {
+ gsr_audio_track_deinit(&self->tracks[i]);
+ }
+
+ if(self->tracks) {
+ free(self->tracks);
+ self->tracks = NULL;
+ }
+ self->num_tracks = 0;
+ self->capacity_tracks = 0;
+
+ if(self->empty_audio) {
+ free(self->empty_audio);
+ self->empty_audio = NULL;
+ }
+
+ if(self->filter_mutex_initialized) {
+ pthread_mutex_destroy(&self->filter_mutex);
+ self->filter_mutex_initialized = false;
+ }
+}
+
+bool gsr_audio_capture_add_track(gsr_audio_capture *self, const gsr_audio_track *track) {
+ if(!gsr_array_ensure_capacity((void**)&self->tracks, self->num_tracks, &self->capacity_tracks, sizeof(gsr_audio_track)))
+ return false;
+
+ self->tracks[self->num_tracks] = *track;
+ ++self->num_tracks;
+ return true;
+}
+
+int gsr_audio_capture_start(gsr_audio_capture *self, int audio_max_frame_size, bool uses_amix) {
+ const size_t audio_buffer_size = audio_max_frame_size * 4 * 2; /* max 4 bytes/sample, 2 channels */
+ self->empty_audio = calloc(1, audio_buffer_size);
+ if(!self->empty_audio) {
+ gsr_log(GSR_LOG_LEVEL_ERROR, "failed to create empty audio");
+ return GSR_ERROR_GENERIC;
+ }
+
+ for(size_t i = 0; i < self->num_tracks; ++i) {
+ gsr_audio_track *track = &self->tracks[i];
+ for(size_t j = 0; j < track->num_audio_devices; ++j) {
+ gsr_audio_device_capture *device = &track->audio_devices[j];
+ device->thread_userdata.audio_capture = self;
+ device->thread_userdata.track = track;
+ device->thread_userdata.device = device;
+ if(pthread_create(&device->thread, NULL, audio_device_thread, &device->thread_userdata) != 0) {
+ gsr_log(GSR_LOG_LEVEL_ERROR, "failed to create audio thread");
+ return GSR_ERROR_GENERIC;
+ }
+ device->thread_created = true;
+ }
+ }
+
+ if(uses_amix) {
+ if(pthread_create(&self->amix_thread, NULL, amix_thread, self) != 0) {
+ gsr_log(GSR_LOG_LEVEL_ERROR, "failed to create audio mix thread");
+ return GSR_ERROR_GENERIC;
+ }
+ self->amix_thread_created = true;
+ }
+
+ return GSR_ERROR_OK;
+}
+
+void gsr_audio_capture_join_threads(gsr_audio_capture *self) {
+ for(size_t i = 0; i < self->num_tracks; ++i) {
+ gsr_audio_track *track = &self->tracks[i];
+ for(size_t j = 0; j < track->num_audio_devices; ++j) {
+ gsr_audio_device_capture *device = &track->audio_devices[j];
+ if(device->thread_created) {
+ pthread_join(device->thread, NULL);
+ device->thread_created = false;
+ }
+ }
+ }
+
+ if(self->amix_thread_created) {
+ pthread_join(self->amix_thread, NULL);
+ self->amix_thread_created = false;
+ }
+}
+
+void gsr_audio_capture_lock_filter(gsr_audio_capture *self) {
+ pthread_mutex_lock(&self->filter_mutex);
+}
+
+void gsr_audio_capture_unlock_filter(gsr_audio_capture *self) {
+ pthread_mutex_unlock(&self->filter_mutex);
+}
+
+static int audio_track_alloc_devices(gsr_audio_track *self, size_t num_devices) {
+ self->audio_devices = calloc(num_devices, sizeof(gsr_audio_device_capture));
+ if(!self->audio_devices) {
+ gsr_log(GSR_LOG_LEVEL_ERROR, "failed to allocate audio devices");
+ return GSR_ERROR_GENERIC;
+ }
+ return GSR_ERROR_OK;
+}
+
+int gsr_audio_track_init_device_inputs(gsr_audio_track *self, const gsr_merged_audio_inputs *merged_audio_inputs, AVCodecContext *audio_codec_context, int num_channels, double num_audio_frames_shift, AVFilterContext **src_filter_ctx, bool use_amix) {
+ const int alloc_result = audio_track_alloc_devices(self, merged_audio_inputs->num_items);
+ if(alloc_result != GSR_ERROR_OK)
+ return alloc_result;
+
+ for(size_t i = 0; i < merged_audio_inputs->num_items; ++i) {
+ const gsr_audio_input *audio_input = &merged_audio_inputs->items[i];
+ gsr_audio_device_capture *device = &self->audio_devices[i];
+ device->audio_input = *audio_input;
+ device->src_filter_ctx = use_amix ? src_filter_ctx[i] : NULL;
+
+ if(audio_input->name[0] == '\0') {
+ device->sound_device.handle = NULL;
+ device->sound_device.frames = 0;
+ } else {
+ char description[GSR_AUDIO_INPUT_NAME_MAX_SIZE + 8];
+ snprintf(description, sizeof(description), "gsr-%s", audio_input->name);
+ if(sound_device_get_by_name(&device->sound_device, description, audio_input->name, description, num_channels, audio_codec_context->frame_size, audio_codec_context_get_audio_format(audio_codec_context)) != 0) {
+ gsr_log(GSR_LOG_LEVEL_ERROR, "failed to get \"%s\" audio device", audio_input->name);
+ return GSR_ERROR_GENERIC;
+ }
+ }
+
+ device->frame = create_audio_frame(audio_codec_context);
+ if(!device->frame)
+ return GSR_ERROR_GENERIC;
+ device->frame->pts = -audio_codec_context->frame_size * num_audio_frames_shift;
+
+ ++self->num_audio_devices;
+ }
+
+ return GSR_ERROR_OK;
+}
+
+#ifdef GSR_APP_AUDIO
+int gsr_audio_track_init_application_input(gsr_audio_track *self, const gsr_merged_audio_inputs *merged_audio_inputs, AVCodecContext *audio_codec_context, int num_channels, double num_audio_frames_shift, gsr_pipewire_audio *pipewire_audio) {
+ const int alloc_result = audio_track_alloc_devices(self, 1);
+ if(alloc_result != GSR_ERROR_OK)
+ return alloc_result;
+
+ gsr_audio_device_capture *device = &self->audio_devices[0];
+ device->frame = create_audio_frame(audio_codec_context);
+ if(!device->frame)
+ return GSR_ERROR_GENERIC;
+ device->frame->pts = -audio_codec_context->frame_size * num_audio_frames_shift;
+ ++self->num_audio_devices;
+
+ char random_str[8];
+ if(!generate_random_characters_standard_alphabet(random_str, sizeof(random_str))) {
+ gsr_log(GSR_LOG_LEVEL_ERROR, "failed to generate random string");
+ return GSR_ERROR_GENERIC;
+ }
+
+ char combined_sink_name[64];
+ snprintf(combined_sink_name, sizeof(combined_sink_name), "gsr-combined-%.*s.monitor", (int)sizeof(random_str), random_str);
+
+ if(sound_device_get_by_name(&device->sound_device, combined_sink_name, "", "gpu-screen-recorder", num_channels, audio_codec_context->frame_size, audio_codec_context_get_audio_format(audio_codec_context)) != 0) {
+ gsr_log(GSR_LOG_LEVEL_ERROR, "failed to setup audio recording to combined sink");
+ return GSR_ERROR_GENERIC;
+ }
+
+ const char **audio_devices_sources = calloc(merged_audio_inputs->num_items, sizeof(const char*));
+ const char **app_names = calloc(merged_audio_inputs->num_items, sizeof(const char*));
+ if(!audio_devices_sources || !app_names) {
+ free(audio_devices_sources);
+ free(app_names);
+ gsr_log(GSR_LOG_LEVEL_ERROR, "failed to allocate application audio names");
+ return GSR_ERROR_GENERIC;
+ }
+
+ size_t num_audio_devices_sources = 0;
+ size_t num_app_names = 0;
+ bool app_audio_inverted = false;
+ for(size_t i = 0; i < merged_audio_inputs->num_items; ++i) {
+ const gsr_audio_input *audio_input = &merged_audio_inputs->items[i];
+ if(audio_input->type == GSR_AUDIO_INPUT_TYPE_DEVICE) {
+ audio_devices_sources[num_audio_devices_sources] = audio_input->name;
+ ++num_audio_devices_sources;
+ } else if(audio_input->type == GSR_AUDIO_INPUT_TYPE_APPLICATION) {
+ app_names[num_app_names] = audio_input->name;
+ ++num_app_names;
+ app_audio_inverted = audio_input->inverted;
+ }
+ }
+
+ int result = GSR_ERROR_OK;
+ if(num_audio_devices_sources > 0) {
+ if(!gsr_pipewire_audio_add_link_from_sources_to_stream(pipewire_audio, audio_devices_sources, num_audio_devices_sources, combined_sink_name)) {
+ gsr_log(GSR_LOG_LEVEL_ERROR, "failed to add application audio link");
+ result = GSR_ERROR_GENERIC;
+ }
+ }
+
+ if(result == GSR_ERROR_OK) {
+ const bool link_added = app_audio_inverted
+ ? gsr_pipewire_audio_add_link_from_apps_to_stream_inverted(pipewire_audio, app_names, num_app_names, combined_sink_name)
+ : gsr_pipewire_audio_add_link_from_apps_to_stream(pipewire_audio, app_names, num_app_names, combined_sink_name);
+ if(!link_added) {
+ gsr_log(GSR_LOG_LEVEL_ERROR, "failed to add application audio link");
+ result = GSR_ERROR_GENERIC;
+ }
+ }
+
+ free(audio_devices_sources);
+ free(app_names);
+ return result;
+}
+#endif
+
+void gsr_audio_track_deinit(gsr_audio_track *self) {
+ for(size_t i = 0; i < self->num_audio_devices; ++i) {
+ gsr_audio_device_capture *device = &self->audio_devices[i];
+ sound_device_close(&device->sound_device);
+ if(device->frame)
+ av_frame_free(&device->frame);
+ }
+
+ if(self->audio_devices) {
+ free(self->audio_devices);
+ self->audio_devices = NULL;
+ }
+ self->num_audio_devices = 0;
+
+ if(self->graph)
+ avfilter_graph_free(&self->graph);
+
+ if(self->codec_context)
+ avcodec_free_context(&self->codec_context);
+}
+
diff --git a/src/recorder/recording_clock.c b/src/recorder/recording_clock.c
new file mode 100644
index 0000000..cb23d56
--- /dev/null
+++ b/src/recorder/recording_clock.c
@@ -0,0 +1,61 @@
+#include "../../include/recorder/recording_clock.h"
+#include "../../include/utils.h"
+#include "../../include/log.h"
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdatomic.h>
+
+struct gsr_recording_clock {
+ double record_start_time;
+ _Atomic double paused_time_offset;
+ double paused_time_start;
+ atomic_bool paused;
+};
+
+gsr_recording_clock* gsr_recording_clock_create(void) {
+ gsr_recording_clock *self = calloc(1, sizeof(gsr_recording_clock));
+ if(!self) {
+ gsr_log(GSR_LOG_LEVEL_ERROR, "gsr_recording_clock_create: failed to allocate recording clock");
+ return NULL;
+ }
+
+ atomic_init(&self->paused_time_offset, 0.0);
+ atomic_init(&self->paused, false);
+ self->record_start_time = clock_get_monotonic_seconds();
+ return self;
+}
+
+void gsr_recording_clock_destroy(gsr_recording_clock *self) {
+ if(self)
+ free(self);
+}
+
+void gsr_recording_clock_start(gsr_recording_clock *self) {
+ self->record_start_time = clock_get_monotonic_seconds();
+}
+
+double gsr_recording_clock_get_start_time(const gsr_recording_clock *self) {
+ return self->record_start_time;
+}
+
+double gsr_recording_clock_get_time(const gsr_recording_clock *self) {
+ return clock_get_monotonic_seconds() - atomic_load(&self->paused_time_offset);
+}
+
+void gsr_recording_clock_set_paused(gsr_recording_clock *self, bool paused) {
+ if(paused == atomic_load(&self->paused))
+ return;
+
+ if(paused) {
+ self->paused_time_start = clock_get_monotonic_seconds();
+ } else {
+ atomic_store(&self->paused_time_offset, atomic_load(&self->paused_time_offset) + (clock_get_monotonic_seconds() - self->paused_time_start));
+ }
+
+ atomic_store(&self->paused, paused);
+}
+
+bool gsr_recording_clock_is_paused(const gsr_recording_clock *self) {
+ return atomic_load(&self->paused);
+}