From f0c1ce4dc8d46917f5218d7b74e280dbfc0b52f2 Mon Sep 17 00:00:00 2001
From: Tristan Ross <tristan.ross@midstall.com>
Date: Fri, 26 Jul 2024 10:32:45 -0700
Subject: [PATCH] x265: fix compiling with clang

---
 pkgs/development/libraries/x265/default.nix   | 11 ++++--
 .../libraries/x265/fix-clang-asm.patch        | 34 +++++++++++++++++++
 2 files changed, 43 insertions(+), 2 deletions(-)
 create mode 100644 pkgs/development/libraries/x265/fix-clang-asm.patch

diff --git a/pkgs/development/libraries/x265/default.nix b/pkgs/development/libraries/x265/default.nix
index 7f44834d4867..9d485f7d6d2e 100644
--- a/pkgs/development/libraries/x265/default.nix
+++ b/pkgs/development/libraries/x265/default.nix
@@ -1,4 +1,5 @@
 { lib
+, gccStdenv
 , stdenv
 , fetchurl
 , fetchpatch
@@ -66,7 +67,11 @@ stdenv.mkDerivation rec {
     })
     # Fix detection of NEON (and armv6 build) :
     ./fix-neon-detection.patch
-  ];
+  ]
+    # CMake files require a bit of patching to support CMAKE_ASM_COMPILER.
+    # Made by @RossComputerGuy for x265 v3.5.
+    # https://mailman.videolan.org/pipermail/x265-devel/2024-July/013734.html
+    ++ lib.optional (stdenv.cc.isClang && !stdenv.targetPlatform.isDarwin) ./fix-clang-asm.patch;
 
   postPatch = ''
     substituteInPlace cmake/Version.cmake \
@@ -89,7 +94,9 @@ stdenv.mkDerivation rec {
     (mkFlag ppaSupport "ENABLE_PPA")
     (mkFlag vtuneSupport "ENABLE_VTUNE")
     (mkFlag werrorSupport "WARNINGS_AS_ERRORS")
-  ];
+  ]
+    # Clang does not support the endfunc directive so use GCC.
+    ++ lib.optional (stdenv.cc.isClang && !stdenv.targetPlatform.isDarwin) "-DCMAKE_ASM_COMPILER=${gccStdenv.cc}/bin/${gccStdenv.cc.targetPrefix}gcc";
 
   cmakeStaticLibFlags = [
     "-DHIGH_BIT_DEPTH=ON"
diff --git a/pkgs/development/libraries/x265/fix-clang-asm.patch b/pkgs/development/libraries/x265/fix-clang-asm.patch
new file mode 100644
index 000000000000..6aa5fe2f0a0e
--- /dev/null
+++ b/pkgs/development/libraries/x265/fix-clang-asm.patch
@@ -0,0 +1,34 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index a407271b4..593628e0f 100755
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -310,7 +310,7 @@ if(GCC)
+     endif()
+     check_cxx_compiler_flag(-mstackrealign CC_HAS_STACK_REALIGN) 
+     if (CC_HAS_STACK_REALIGN)
+-        add_definitions(-mstackrealign)
++        add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-mstackrealign>)
+     endif()
+     # Disable exceptions. Reduce executable size, increase compability.
+     check_cxx_compiler_flag(-fno-exceptions CC_HAS_FNO_EXCEPTIONS_FLAG)
+@@ -545,7 +545,7 @@ if((MSVC_IDE OR XCODE OR GCC) AND ENABLE_ASSEMBLY)
+             list(APPEND ASM_OBJS ${ASM}.${SUFFIX})
+             add_custom_command(
+                 OUTPUT ${ASM}.${SUFFIX}
+-                COMMAND ${CMAKE_CXX_COMPILER}
++                COMMAND ${CMAKE_ASM_COMPILER}
+                 ARGS ${ARM_ARGS} -c ${ASM_SRC} -o ${ASM}.${SUFFIX}
+                 DEPENDS ${ASM_SRC})
+         endforeach()
+diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt
+index 12b643ad5..876f3543d 100644
+--- a/common/CMakeLists.txt
++++ b/common/CMakeLists.txt
+@@ -16,6 +16,7 @@ endif(EXTRA_LIB)
+ if(ENABLE_ASSEMBLY)
+     set_source_files_properties(threading.cpp primitives.cpp pixel.cpp PROPERTIES COMPILE_FLAGS -DENABLE_ASSEMBLY=1)
+     list(APPEND VFLAGS "-DENABLE_ASSEMBLY=1")
++    enable_language(ASM)
+ endif(ENABLE_ASSEMBLY)
+ 
+ if(ENABLE_ASSEMBLY AND X86)