diff --git a/drivers/accel/Makefile b/drivers/accel/Makefile
index 07aa77aed1c8dd27dd63b0ed658cb923c5cde5f4..f22fd44d586b2eab2ed203819588fc494c22014b 100644
--- a/drivers/accel/Makefile
+++ b/drivers/accel/Makefile
@@ -1,4 +1,4 @@
 # SPDX-License-Identifier: GPL-2.0-only
 
-obj-y	+= habanalabs/
-obj-y	+= ivpu/
+obj-$(CONFIG_DRM_ACCEL_HABANALABS)	+= habanalabs/
+obj-$(CONFIG_DRM_ACCEL_IVPU)		+= ivpu/