diff --git a/cmake/GtsamBuildTypes.cmake b/cmake/GtsamBuildTypes.cmake
index 99fa5d598..96a0f11f3 100644
--- a/cmake/GtsamBuildTypes.cmake
+++ b/cmake/GtsamBuildTypes.cmake
@@ -58,12 +58,10 @@ mark_as_advanced(GTSAM_CMAKE_C_FLAGS_TIMING GTSAM_CMAKE_CXX_FLAGS_TIMING GTSAM_C
                  GTSAM_CMAKE_C_FLAGS_PROFILING GTSAM_CMAKE_CXX_FLAGS_PROFILING GTSAM_CMAKE_EXE_LINKER_FLAGS_PROFILING 
                  GTSAM_CMAKE_SHARED_LINKER_FLAGS_PROFILING GTSAM_CMAKE_MODULE_LINKER_FLAGS_PROFILING)
 
-# Make CMAKE_BUILD_TYPE=None flags default to the CMAKE_BUILD_TYPE=RelWithDebInfo ones:
-set(CMAKE_C_FLAGS   ${GTSAM_CMAKE_C_FLAGS})
-set(CMAKE_CXX_FLAGS ${GTSAM_CMAKE_CXX_FLAGS})
-
 # Apply the gtsam specific build flags as normal variables. This makes it so that they only
 # apply to the gtsam part of the build if gtsam is built as a subproject
+set(CMAKE_C_FLAGS   ${GTSAM_CMAKE_C_FLAGS})
+set(CMAKE_CXX_FLAGS ${GTSAM_CMAKE_CXX_FLAGS})
 set(CMAKE_C_FLAGS_DEBUG ${GTSAM_CMAKE_C_FLAGS_DEBUG})
 set(CMAKE_CXX_FLAGS_DEBUG ${GTSAM_CMAKE_CXX_FLAGS_DEBUG})
 set(CMAKE_C_FLAGS_RELWITHDEBINFO ${GTSAM_CMAKE_C_FLAGS_RELWITHDEBINFO})