Use mbedtls instead of openssl

9 files changed, 162 insertions, 67 deletions

diff --git a/README.md b/README.md
index 9403e23..055bddd 100644
--- a/README.md
+++ b/README.md
@@ -76,8 +76,10 @@ When building GPU Screen Recorder with portal support (`-Dportal=true` meson opt
 * libpipewire (and libspa which is usually part of libpipewire)
 
 ## Building with a statically linked ffmpeg
-The `-Dffmpeg_static=true` meson option (disabled by default) downloads ffmpeg and the libraries that ffmpeg depends on (x264, opus, openssl, srt and nv-codec-headers), builds them from source with lto and only the components that GPU Screen Recorder uses and then links them statically into GPU Screen Recorder.
-This removes the runtime dependency on the system ffmpeg and on all of those libraries. `make`, `nasm`, `cmake` and `perl` are needed to build them and the first `meson setup` takes a few minutes longer because of it.
+The `-Dffmpeg_static=true` meson option (disabled by default) downloads ffmpeg and the libraries that ffmpeg depends on (x264, opus, mbedtls, srt and nv-codec-headers), builds them from source with lto and only the components that GPU Screen Recorder uses and then links them statically into GPU Screen Recorder.
+This removes the runtime dependency on the system ffmpeg and on all of those libraries. `make`, `nasm`, `cmake` and `python3` are needed to build them and the first `meson setup` takes a few minutes longer because of it.
+
+mbedtls is used as the tls backend instead of openssl because it's much smaller. mbedtls has no built-in default certificate location, so ffmpeg is patched to look for the certificate store of the system (and to honor the `SSL_CERT_FILE` and `SSL_CERT_DIR` environment variables) the same way that openssl does.
 
 The only libraries that ffmpeg is still dynamically linked to are libva, libdrm (which GPU Screen Recorder also uses directly) and the c/c++ runtime.
 
diff --git a/extra/build_ffmpeg.sh b/extra/build_ffmpeg.sh
index 06c3344..052a822 100755
--- a/extra/build_ffmpeg.sh
+++ b/extra/build_ffmpeg.sh
@@ -69,58 +69,24 @@ case $library in
             make install
         }
         ;;
-    openssl)
-        require_program perl
-        # The certificate directory of the system openssl is used so that certificate
-        # verification keeps working the same way as it does with the system openssl.
-        openssl_dir=$(openssl version -d 2>/dev/null | sed -n 's/^OPENSSLDIR: "\(.*\)"$/\1/p')
-        if [ -z "$openssl_dir" ]; then
-            openssl_dir=/etc/ssl
-        fi
-        set -- --prefix="$prefix" \
-            --libdir=lib \
-            --openssldir="$openssl_dir" \
-            no-shared \
-            no-apps \
-            no-docs \
-            no-tests \
-            no-legacy \
-            no-engine \
-            no-comp \
-            no-quic \
-            no-ssl3 \
-            no-weak-ssl-ciphers \
-            no-cms \
-            no-ct \
-            no-ts \
-            no-ocsp \
-            no-srp \
-            no-psk \
-            no-dsa \
-            no-ec2m \
-            no-gost \
-            no-idea \
-            no-md2 \
-            no-md4 \
-            no-mdc2 \
-            no-rc2 \
-            no-rc4 \
-            no-rc5 \
-            no-bf \
-            no-cast \
-            no-seed \
-            no-camellia \
-            no-whirlpool \
-            no-rmd160 \
-            no-sm2 \
-            no-sm3 \
-            no-sm4 \
-            no-siphash \
-            -O3 -flto -fPIC
+    mbedtls)
+        require_program cmake
+        require_program python3
+        set -- -DCMAKE_BUILD_TYPE=Release \
+            -DCMAKE_INSTALL_PREFIX="$prefix" \
+            -DCMAKE_INSTALL_LIBDIR=lib \
+            -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=ON \
+            -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+            -DENABLE_TESTING=OFF \
+            -DENABLE_PROGRAMS=OFF \
+            -DUSE_SHARED_MBEDTLS_LIBRARY=OFF \
+            -DUSE_STATIC_MBEDTLS_LIBRARY=ON
         build() {
-            "$source_dir/Configure" "$@"
-            make -j"$jobs"
-            make install_sw
+            # The whip muxer needs dtls-srtp, which mbedtls doesn't enable by default.
+            python3 "$source_dir/scripts/config.py" -f "$source_dir/include/mbedtls/mbedtls_config.h" set MBEDTLS_SSL_DTLS_SRTP
+            cmake -S "$source_dir" -B "$build_dir" "$@"
+            cmake --build "$build_dir" -j"$jobs"
+            cmake --install "$build_dir"
         }
         ;;
     srt)
@@ -136,9 +102,9 @@ case $library in
             -DENABLE_EXAMPLES=OFF \
             -DENABLE_TESTING=OFF \
             -DENABLE_UNITTESTS=OFF \
-            -DUSE_ENCLIB=openssl \
-            -DOPENSSL_ROOT_DIR="$prefix" \
-            -DOPENSSL_USE_STATIC_LIBS=ON
+            -DUSE_ENCLIB=mbedtls \
+            -DMBEDTLS_PREFIX="$prefix" \
+            -DSSL_REQUIRED_MODULES=mbedtls
         build() {
             cmake -S "$source_dir" -B "$build_dir" "$@"
             cmake --build "$build_dir" -j"$jobs"
@@ -192,7 +158,7 @@ case $library in
             --enable-libx264 \
             --enable-libopus \
             --enable-libsrt \
-            --enable-openssl \
+            --enable-mbedtls \
             --enable-ffnvcodec \
             --enable-nvenc \
             --enable-cuda \
diff --git a/subprojects/ffmpeg.wrap b/subprojects/ffmpeg.wrap
index 56e8e65..c98e19a 100644
--- a/subprojects/ffmpeg.wrap
+++ b/subprojects/ffmpeg.wrap
@@ -4,4 +4,4 @@ source_url = https://ffmpeg.org/releases/ffmpeg-8.1.tar.xz
 source_filename = ffmpeg-8.1.tar.xz
 source_hash = b072aed6871998cce9b36e7774033105ca29e33632be5b6347f3206898e0756a
 patch_directory = ffmpeg
-diff_files = ffmpeg-nvenc-runtime-api-version.patch
+diff_files = ffmpeg-nvenc-runtime-api-version.patch, ffmpeg-mbedtls-skip-non-dtls-packets.patch, ffmpeg-mbedtls-default-ca-certs.patch
diff --git a/subprojects/mbedtls.wrap b/subprojects/mbedtls.wrap
new file mode 100644
index 0000000..dfd3c32
--- /dev/null
+++ b/subprojects/mbedtls.wrap
@@ -0,0 +1,6 @@
+[wrap-file]
+directory = mbedtls-3.6.7
+source_url = https://github.com/Mbed-TLS/mbedtls/releases/download/mbedtls-3.6.7/mbedtls-3.6.7.tar.bz2
+source_filename = mbedtls-3.6.7.tar.bz2
+source_hash = a7e8bcbec0e6f761b4af24f25677626b35f762f68eef79c08677a363212d11f6
+patch_directory = mbedtls
diff --git a/subprojects/openssl.wrap b/subprojects/openssl.wrap
deleted file mode 100644
index fe386af..0000000
--- a/subprojects/openssl.wrap
+++ /dev/null
@@ -1,6 +0,0 @@
-[wrap-file]
-directory = openssl-3.6.3
-source_url = https://github.com/openssl/openssl/releases/download/openssl-3.6.3/openssl-3.6.3.tar.gz
-source_filename = openssl-3.6.3.tar.gz
-source_hash = 243a86649cf6f23eeb6a2ff2456e09e5d77dd9018a54d3d96b0c6bdd6ba6c7f1
-patch_directory = openssl
diff --git a/subprojects/packagefiles/ffmpeg-mbedtls-default-ca-certs.patch b/subprojects/packagefiles/ffmpeg-mbedtls-default-ca-certs.patch
new file mode 100644
index 0000000..dc2c45f
--- /dev/null
+++ b/subprojects/packagefiles/ffmpeg-mbedtls-default-ca-certs.patch
@@ -0,0 +1,79 @@
+--- a/libavformat/tls_mbedtls.c
++++ b/libavformat/tls_mbedtls.c
+@@ -42,6 +42,66 @@
+ #include "libavutil/avstring.h"
+ #include "libavutil/random_seed.h"
+ #include "libavutil/intreadwrite.h"
++#include "libavutil/getenv_utf8.h"
++
++/*
++ * mbedtls has no built-in default certificate location, unlike openssl which
++ * falls back to the location it was compiled with. Without this the peer
++ * certificate can only be verified when the caller passes a ca_file, so look
++ * for the certificate store of the system instead, honoring the same
++ * environment variables as openssl does.
++ */
++static const char * const default_ca_files[] = {
++    "/etc/ssl/certs/ca-certificates.crt", // debian, ubuntu, arch, alpine, gentoo
++    "/etc/pki/tls/certs/ca-bundle.crt",   // fedora, rhel
++    "/etc/ssl/ca-bundle.pem",             // opensuse
++    "/etc/ssl/cert.pem",                  // openbsd, freebsd, macos
++    "/usr/local/etc/ssl/cert.pem",        // freebsd ports
++};
++
++static const char * const default_ca_dirs[] = {
++    "/etc/ssl/certs",
++    "/etc/pki/tls/certs",
++};
++
++/* A positive return value from mbedtls means that only some of the certificates failed to parse, which is not fatal */
++static int mbedtls_load_default_ca_certs(URLContext *h, mbedtls_x509_crt *ca_cert)
++{
++    char *env_ca_file = getenv_utf8("SSL_CERT_FILE");
++    char *env_ca_dir = getenv_utf8("SSL_CERT_DIR");
++    int loaded = 0;
++
++    if (env_ca_file && mbedtls_x509_crt_parse_file(ca_cert, env_ca_file) >= 0)
++        loaded = 1;
++
++    if (!loaded && env_ca_dir && mbedtls_x509_crt_parse_path(ca_cert, env_ca_dir) >= 0)
++        loaded = 1;
++
++    freeenv_utf8(env_ca_file);
++    freeenv_utf8(env_ca_dir);
++
++    for (size_t i = 0; !loaded && i < FF_ARRAY_ELEMS(default_ca_files); i++) {
++        if (mbedtls_x509_crt_parse_file(ca_cert, default_ca_files[i]) >= 0) {
++            av_log(h, AV_LOG_VERBOSE, "loaded CA certificates from %s\n", default_ca_files[i]);
++            loaded = 1;
++        }
++    }
++
++    for (size_t i = 0; !loaded && i < FF_ARRAY_ELEMS(default_ca_dirs); i++) {
++        if (mbedtls_x509_crt_parse_path(ca_cert, default_ca_dirs[i]) >= 0) {
++            av_log(h, AV_LOG_VERBOSE, "loaded CA certificates from %s\n", default_ca_dirs[i]);
++            loaded = 1;
++        }
++    }
++
++    if (!loaded) {
++        av_log(h, AV_LOG_WARNING, "unable to find the CA certificates of the system, "
++                                  "certificate verification is going to fail\n");
++        return AVERROR(ENOENT);
++    }
++
++    return 0;
++}
+ 
+ static int mbedtls_x509_fingerprint(char *cert_buf, size_t cert_sz, char **fingerprint)
+ {
+@@ -557,6 +617,9 @@
+             av_log(h, AV_LOG_ERROR, "mbedtls_x509_crt_parse_file for CA cert returned %d\n", ret);
+             goto fail;
+         }
++    } else if (shr->verify) {
++        // Only a warning is logged when this fails, matching what the openssl backend does
++        mbedtls_load_default_ca_certs(h, &tls_ctx->ca_cert);
+     }
+ 
+     // load own certificate
diff --git a/subprojects/packagefiles/ffmpeg-mbedtls-skip-non-dtls-packets.patch b/subprojects/packagefiles/ffmpeg-mbedtls-skip-non-dtls-packets.patch
new file mode 100644
index 0000000..4f18644
--- /dev/null
+++ b/subprojects/packagefiles/ffmpeg-mbedtls-skip-non-dtls-packets.patch
@@ -0,0 +1,48 @@
+--- a/libavformat/tls_mbedtls.c
++++ b/libavformat/tls_mbedtls.c
+@@ -41,6 +41,7 @@
+ #include "libavutil/parseutils.h"
+ #include "libavutil/avstring.h"
+ #include "libavutil/random_seed.h"
++#include "libavutil/intreadwrite.h"
+ 
+ static int mbedtls_x509_fingerprint(char *cert_buf, size_t cert_sz, char **fingerprint)
+ {
+@@ -376,6 +377,27 @@
+     return handle_transport_error(h, "ffurl_write", MBEDTLS_ERR_SSL_WANT_WRITE, ret);
+ }
+ 
++/*
++ * Some webrtc servers, such as the ones based on pion, send stun packets during the
++ * dtls handshake. openssl and gnutls filter those out internally but mbedtls passes
++ * every received udp packet to its dtls state machine, which makes the handshake fail.
++ * This is the same check as the one in whip.c, which ffmpeg n8.1 doesn't share yet.
++ */
++#define DTLS_CONTENT_TYPE_CHANGE_CIPHER_SPEC 20
++#define DTLS_RECORD_LAYER_HEADER_LEN 13
++#define DTLS_VERSION_10 0xfeff
++#define DTLS_VERSION_12 0xfefd
++
++static int is_dtls_packet(const unsigned char *buf, int size)
++{
++    if (size > DTLS_RECORD_LAYER_HEADER_LEN) {
++        uint16_t version = AV_RB16(&buf[1]);
++        return buf[0] >= DTLS_CONTENT_TYPE_CHANGE_CIPHER_SPEC &&
++               (version == DTLS_VERSION_10 || version == DTLS_VERSION_12);
++    }
++    return 0;
++}
++
+ static int mbedtls_recv(void *ctx, unsigned char *buf, size_t len)
+ {
+     TLSContext *tls_ctx = (TLSContext*) ctx;
+@@ -394,6 +416,9 @@
+             }
+             av_log(tls_ctx, AV_LOG_TRACE, "Set UDP remote addr on UDP socket, now 'connected'\n");
+         }
++        /* Skip non-DTLS packets such as STUN to avoid failures. */
++        if (shr->is_dtls && !is_dtls_packet(buf, ret))
++            return MBEDTLS_ERR_SSL_WANT_READ;
+         return ret;
+     }
+     if (h->max_packet_size && len > h->max_packet_size)
diff --git a/subprojects/packagefiles/ffmpeg/meson.build b/subprojects/packagefiles/ffmpeg/meson.build
index 38df7e9..96345f2 100644
--- a/subprojects/packagefiles/ffmpeg/meson.build
+++ b/subprojects/packagefiles/ffmpeg/meson.build
@@ -8,8 +8,8 @@ install_prefix = meson.current_build_dir() / 'install'
 c_compiler = ' '.join(meson.get_compiler('c').cmd_array())
 build_library = find_program(meson.global_source_root() / 'extra' / 'build_ffmpeg.sh')
 
-# srt links against openssl and ffmpeg links against all of them, so the order matters.
-libraries = ['nv-codec-headers', 'x264', 'opus', 'openssl', 'srt', 'ffmpeg']
+# srt links against mbedtls and ffmpeg links against all of them, so the order matters.
+libraries = ['nv-codec-headers', 'x264', 'opus', 'mbedtls', 'srt', 'ffmpeg']
 
 foreach library : libraries
     source_dir = library == 'ffmpeg' ? meson.current_source_dir() : subproject(library).get_variable('source_dir')
diff --git a/subprojects/packagefiles/openssl/meson.build b/subprojects/packagefiles/mbedtls/meson.build
index d7ebb0c..802357e 100644
--- a/subprojects/packagefiles/openssl/meson.build
+++ b/subprojects/packagefiles/mbedtls/meson.build
@@ -1,3 +1,3 @@
-project('openssl')
+project('mbedtls')
 
 source_dir = meson.current_source_dir()