diff --git a/CMakeModules/CalamaresAddModuleSubdirectory.cmake b/CMakeModules/CalamaresAddModuleSubdirectory.cmake
index 66595d269..caf1b707e 100644
--- a/CMakeModules/CalamaresAddModuleSubdirectory.cmake
+++ b/CMakeModules/CalamaresAddModuleSubdirectory.cmake
@@ -6,9 +6,12 @@ set( MODULE_DATA_DESTINATION share/calamares/modules )
 function( calamares_add_module_subdirectory )
     set( SUBDIRECTORY ${ARGV0} )
 
+    set( MODULE_CONFIG_FILES "" )
+
     # If this subdirectory has a CMakeLists.txt, we add_subdirectory it...
     if( EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/${SUBDIRECTORY}/CMakeLists.txt" )
         add_subdirectory( ${SUBDIRECTORY} )
+        file( GLOB MODULE_CONFIG_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}/${SUBDIRECTORY} "${SUBDIRECTORY}/*.conf" )
     # ...otherwise, we look for a module.desc.
     elseif( EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/${SUBDIRECTORY}/module.desc" )
         set( MODULES_DIR ${CMAKE_INSTALL_LIBDIR}/calamares/modules )
@@ -39,12 +42,6 @@ function( calamares_add_module_subdirectory )
 #            message( "   ${Green}FILES:${ColorReset} ${MODULE_FILES}" )
             message( "   ${Green}MODULE_DESTINATION:${ColorReset} ${MODULE_DESTINATION}" )
             if( MODULE_CONFIG_FILES )
-                if ( BUILD_TESTING )
-                    foreach( _config_file ${MODULE_CONFIG_FILES} )
-                        add_test(NAME config-${SUBDIRECTORY}
-                            COMMAND test_conf ${CMAKE_CURRENT_BINARY_DIR}/${SUBDIRECTORY}/${_config_file} )
-                    endforeach()
-                endif()
                 if ( INSTALL_CONFIG )
                     message( "   ${Green}CONFIGURATION_FILES:${ColorReset} ${MODULE_CONFIG_FILES} => ${MODULE_DATA_DESTINATION}" )
                 else()
@@ -62,9 +59,23 @@ function( calamares_add_module_subdirectory )
                 RENAME calamares-${SUBDIRECTORY}.mo
             )
         endif()
-
     else()
         message( "-- ${BoldYellow}Warning:${ColorReset} tried to add module subdirectory ${BoldRed}${SUBDIRECTORY}${ColorReset} which has no CMakeLists.txt or module.desc." )
         message( "" )
     endif()
+
+    # Check any config files for basic correctness
+    if ( BUILD_TESTING AND MODULE_CONFIG_FILES )
+        set( _count 0 )
+        foreach( _config_file ${MODULE_CONFIG_FILES} )
+            set( _count_str "-${_count}" )
+            if ( _count EQUAL 0 )
+                set( _count_str "" )
+            endif()
+            add_test(
+                NAME config-${SUBDIRECTORY}${_count_str}
+                COMMAND test_conf ${CMAKE_CURRENT_BINARY_DIR}/${SUBDIRECTORY}/${_config_file} )
+            math( EXPR _count "${_count} + 1" )
+        endforeach()
+    endif()
 endfunction()