diff --git a/_ci-linux-alpine.sh b/_ci-linux-alpine.sh
index ae7aa45fa..1dbbb1952 100755
--- a/_ci-linux-alpine.sh
+++ b/_ci-linux-alpine.sh
@@ -21,7 +21,9 @@ if [[ "${CW_CONFIG:-}" = *'win'* ]]; then
 elif [[ "${CW_CONFIG:-}" = *'linux'* ]]; then
   apk add --no-cache checksec-rs --repository=https://dl-cdn.alpinelinux.org/alpine/edge/community/
   extra+=' compiler-rt libc++-static'  # for llvm
-  extra+=' linux-headers'  # for openssl 'secure-memory' feature
+  if [[ "${CW_CONFIG:-}" =~ (quictls|openssl) ]]; then
+    extra+=' linux-headers'  # for openssl 'secure-memory' feature
+  fi
   if [[ "${CW_CONFIG:-}" = *'gcc'* ]]; then
     extra+=' gcc'
   fi
diff --git a/_ci-linux-debian.sh b/_ci-linux-debian.sh
index 0268e2572..3610cefdd 100755
--- a/_ci-linux-debian.sh
+++ b/_ci-linux-debian.sh
@@ -78,11 +78,13 @@ elif [[ "${CW_CONFIG:-}" = *'linux'* ]]; then
     if [[ "${CW_CONFIG:-}" = *'gcc'* ]]; then
       extra+=" libgcc${CW_GCCSUFFIX}-dev"
     fi
-    # for openssl 'secure-memory' feature
-    if [ "$(uname -m)" = 'aarch64' ]; then
-      extra+=' linux-headers-arm64'
-    elif [ "$(uname -m)" = 'x86_64' ]; then
-      extra+=' linux-headers-amd64'
+    if [[ "${CW_CONFIG:-}" =~ (quictls|openssl) ]]; then
+      # for openssl 'secure-memory' feature
+      if [ "$(uname -m)" = 'aarch64' ]; then
+        extra+=' linux-headers-arm64'
+      elif [ "$(uname -m)" = 'x86_64' ]; then
+        extra+=' linux-headers-amd64'
+      fi
     fi
   else
     # FIXME: workaround for glibc-llvm-riscv64 builds: