diff --git a/pkgs/tools/graphics/graphviz/default.nix b/pkgs/tools/graphics/graphviz/default.nix
index 643de951e009..1172173dd326 100644
--- a/pkgs/tools/graphics/graphviz/default.nix
+++ b/pkgs/tools/graphics/graphviz/default.nix
@@ -74,9 +74,8 @@ stdenv.mkDerivation (finalAttrs: {
   configureFlags = [
     "--with-ltdl-lib=${libtool.lib}/lib"
     "--with-ltdl-include=${libtool}/include"
-  ]
-  # TODO: this should probably be !withXorg instead of false, however it causes 17k rebuilds
-  ++ optional false "--without-x";
+    (lib.withFeature withXorg "x")
+  ];
 
   enableParallelBuilding = true;
   strictDeps = true;