diff --git a/m4/xiph_openssl.m4 b/m4/xiph_openssl.m4
index f62a524..dcc8202 100644
--- a/m4/xiph_openssl.m4
+++ b/m4/xiph_openssl.m4
@@ -30,6 +30,7 @@ else
 fi
 
 # Now try linking to openssl
+if test x$with_openssl != xno; then
 xt_save_CFLAGS="$CFLAGS"
 xt_save_LIBS="$LIBS"
 CFLAGS="$CFLAGS $OPENSSL_CFLAGS"
@@ -37,6 +38,7 @@ LIBS="$OPENSSL_LIBS $LIBS"
 AC_TRY_LINK([#include <openssl/ssl.h>], [void *a = SSL_new], [openssl_ok='yes'])
 CFLAGS="$xt_save_CFLAGS"
 LIBS="$xt_save_LIBS"
+fi
 
 if test "$openssl_ok" = "yes"; then
     AC_DEFINE(HAVE_OPENSSL, 1, [Define if you have libopenssl.])