diff --git a/CMakeLists.txt b/CMakeLists.txt
index 049212e6..225c8343 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -129,7 +129,7 @@ include (${CMAKE_MODULE_PATH}/platform.cmake)
 # Detect OS and if Fat Runtime is available
 include (${CMAKE_MODULE_PATH}/osdetection.cmake)
 
-if(BUILD_SSE2_SIMDE)
+if(ARCH_X86_86 AND BUILD_SSE2_SIMDE AND NOT FAT_RUNTIME)
     set(SIMDE_BACKEND True)
 endif()