diff --git a/ports/speexdsp/CMakeLists.txt b/ports/speexdsp/CMakeLists.txt
index 20ae3c2905..0708faa2d0 100644
--- a/ports/speexdsp/CMakeLists.txt
+++ b/ports/speexdsp/CMakeLists.txt
@@ -1,6 +1,7 @@
 # CMAKE project for libspeexdsp
 cmake_minimum_required(VERSION 3.1)
 option(SOURCE_PATH   "Root directory.")
+option(USE_SSE       "Use SSE")
 project (libspeexdsp)
 
 file(GLOB_RECURSE LIBSPEEXDSP_SOURCES "${SOURCE_PATH}/libspeexdsp/*.c")
@@ -20,9 +21,8 @@ endif ()
 
 add_definitions(-DHAVE_CONFIG_H -D_WIN32)
 
-string (TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" LOWERCASE_SYSTEM_PROCESSOR)
-if (LOWERCASE_SYSTEM_PROCESSOR STREQUAL "x86" OR LOWERCASE_SYSTEM_PROCESSOR STREQUAL "amd64")
-    add_definitions(-D_USE_SSE)
+if (USE_SSE)
+    add_definitions(-D_USE_SSE -D_USE_SSE2)
 endif()
 
 include_directories("${SOURCE_PATH}/include"
diff --git a/ports/speexdsp/portfile.cmake b/ports/speexdsp/portfile.cmake
index 7784b73139..72bc313d7f 100644
--- a/ports/speexdsp/portfile.cmake
+++ b/ports/speexdsp/portfile.cmake
@@ -8,9 +8,15 @@ vcpkg_download_distfile(ARCHIVE_FILE
 )
 vcpkg_extract_source_archive(${ARCHIVE_FILE})
 
+set(USE_SSE OFF)
+set(USE_NEON OFF)
+if(VCPKG_TARGET_ARCHITECTURE STREQUAL "x64" OR VCPKG_TARGET_ARCHITECTURE STREQUAL "x86")
+    set(USE_SSE ON)
+endif()
+
 vcpkg_configure_cmake(
     SOURCE_PATH ${CMAKE_PATH}
-    OPTIONS -DSOURCE_PATH=${SOURCE_PATH}
+    OPTIONS -DSOURCE_PATH=${SOURCE_PATH} -DUSE_SSE=${USE_SSE}
 )
 
 vcpkg_install_cmake()