diff --git a/lisp/textmodes/refill.el b/lisp/textmodes/refill.el
index 26e26d6bf6e..22aba1cbbe3 100644
--- a/lisp/textmodes/refill.el
+++ b/lisp/textmodes/refill.el
@@ -145,8 +145,7 @@ This is used to optimize refilling.")
 		  (if use-hard-newlines
 		      (fill-region beg end arg)
 		    (fill-region-as-paragraph beg end arg))))
-	  (move-overlay refill-ignorable-overlay beg (point)))))
-    (skip-line-prefix fill-pfx)))
+	  (move-overlay refill-ignorable-overlay beg (point)))))))
 
 (defun refill-fill-paragraph (arg)
   "Like `fill-paragraph' but don't delete whitespace at paragraph end."