mirror of
https://git.savannah.gnu.org/git/emacs.git
synced 2025-02-04 20:27:45 +00:00
Expanded testing of MH-E with multiple MH variants
* test/lisp/mh-e/mh-utils-tests.el: Environment variable TEST_MH_PATH controls which installed MH variant to test with. Moved the commentary about testing with different MH variants from above 'with-mh-test-env' definition to "Commentary" section at the top of the file. * test/lisp/mh-e/test-all-mh-variants.sh: New script to test all installed MH variants.
This commit is contained in:
Stephen Gildea
parent
b497add971
commit
5d408f1a24
@ -17,6 +17,34 @@
|
||||
;; You should have received a copy of the GNU General Public License
|
||||
;; along with GNU Emacs. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
;;; Commentary:
|
||||
|
||||
;; This test suite runs tests that use and depend on MH programs
|
||||
;; installed on the system.
|
||||
|
||||
;; When running such tests, MH-E can use a particular MH variant
|
||||
;; installed on the system, or it can use the mocks provided here.
|
||||
;; (Setup is done by the `with-mh-test-env' macro.)
|
||||
|
||||
;; By setting environment variable TEST_MH_PATH, you can select which of
|
||||
;; the installed MH variants to use, or ignore them all and use mocks.
|
||||
;; See also the script test-all-mh-variants.sh in this directory.
|
||||
|
||||
;; 1. To run these tests against the default MH variant installed on
|
||||
;; this system:
|
||||
;; cd ../.. && make lisp/mh-e/mh-utils-tests
|
||||
|
||||
;; 2. To run these tests against an MH variant installed in a
|
||||
;; specific directory, set TEST_MH_PATH, as in this example:
|
||||
;; cd ../.. && make lisp/mh-e/mh-utils-tests TEST_MH_PATH=/usr/local/nmh/bin
|
||||
|
||||
;; 3. To search for and run these tests against all MH variants
|
||||
;; installed on this system:
|
||||
;; ./test-all-mh-variants.sh
|
||||
|
||||
;; Setting the environment variable TEST_MH_DEBUG or the Lisp variable
|
||||
;; mh-test-utils-debug-mocks logs access to the file system during the test.
|
||||
|
||||
;;; Code:
|
||||
|
||||
(require 'ert)
|
||||
@ -56,34 +84,32 @@
|
||||
;; Folder names that are used by the following tests.
|
||||
(defvar mh-test-rel-folder "rela-folder")
|
||||
(defvar mh-test-abs-folder "/abso-folder")
|
||||
(defvar mh-test-no-such-folder "/testdir/none"
|
||||
"Name of a folder that the user does not have.")
|
||||
(defvar mh-test-no-such-folder "/testdir/none" "A folder that does not exist.")
|
||||
|
||||
(defvar mh-test-utils-variants nil
|
||||
"The value of `mh-variants' used for these tests.
|
||||
This variable allows setting `mh-variants' to a limited set for targeted
|
||||
testing. Its value can be different from the normal value when
|
||||
environment variable TEST_MH_PATH is set. By remembering the value, we
|
||||
can log the choice only once, which makes the batch log easier to read.")
|
||||
|
||||
(defvar mh-test-variant-logged-already nil
|
||||
"Whether `with-mh-test-env' has written the MH variant to the log.")
|
||||
(setq mh-test-variant-logged-already nil) ;reset if buffer is re-evaluated
|
||||
|
||||
(defvar mh-test-utils-debug-mocks nil
|
||||
(defvar mh-test-utils-debug-mocks (> (length (getenv "TEST_MH_DEBUG")) 0)
|
||||
"Whether to log detailed behavior of mock functions.")
|
||||
|
||||
(defvar mh-test-call-process-real (symbol-function 'call-process))
|
||||
(defvar mh-test-file-directory-p-real (symbol-function 'file-directory-p))
|
||||
|
||||
|
||||
;;; This macro wraps tests that touch the file system and/or run programs.
|
||||
;;; When running such tests, MH-E can use a particular MH variant
|
||||
;;; installed on the system, or it can use the mocks provided below.
|
||||
|
||||
;;; By setting PATH and mh-sys-path, you can select which of the
|
||||
;;; installed MH variants to use or ignore them all and use mocks.
|
||||
;;; The macro with-mh-test-env wraps tests that touch the file system
|
||||
;;; and/or run programs.
|
||||
|
||||
(defmacro with-mh-test-env (&rest body)
|
||||
"Evaluate BODY with a test mail environment.
|
||||
Functions that touch the file system or run MH programs are either
|
||||
mocked out or pointed at a test tree. When called from Emacs's batch
|
||||
testing infrastructure, this will use mocks and thus run on systems
|
||||
that do not have any MH variant installed. MH-E developers can
|
||||
install an MH variant and test it interactively."
|
||||
mocked out or pointed at a test tree. Uses `mh-test-utils-setup' to
|
||||
select which."
|
||||
(declare (indent defun))
|
||||
`(cl-letf ((temp-home-dir nil)
|
||||
;; make local bindings for things we will modify for test env
|
||||
@ -93,26 +119,56 @@ install an MH variant and test it interactively."
|
||||
((symbol-function 'file-directory-p))
|
||||
;; the test always gets its own sub-folders cache
|
||||
(mh-sub-folders-cache (make-hash-table :test #'equal))
|
||||
;; Allow envvar TEST_MH_PATH to control mh-variants.
|
||||
(mh-variants mh-test-utils-variants)
|
||||
;; remember the original value
|
||||
(original-mh-test-variant-logged mh-test-variant-logged-already)
|
||||
(original-mh-path mh-path)
|
||||
(original-mh-sys-path mh-sys-path)
|
||||
(original-exec-path exec-path)
|
||||
(original-mh-variant-in-use mh-variant-in-use)
|
||||
(original-mh-progs mh-progs)
|
||||
(original-mh-lib mh-lib)
|
||||
(original-mh-lib-progs mh-lib-progs)
|
||||
(original-mh-envvar (getenv "MH")))
|
||||
(unwind-protect
|
||||
(progn
|
||||
(setq temp-home-dir (mh-test-utils-setup))
|
||||
,@body)
|
||||
(unless noninteractive
|
||||
;; If interactive, forget that we logged the variant and
|
||||
;; restore any changes TEST_MH_PATH made.
|
||||
(setq mh-test-variant-logged-already original-mh-test-variant-logged
|
||||
mh-path original-mh-path
|
||||
mh-sys-path original-mh-sys-path
|
||||
exec-path original-exec-path
|
||||
mh-variant-in-use original-mh-variant-in-use
|
||||
mh-progs original-mh-progs
|
||||
mh-lib original-mh-lib
|
||||
mh-lib-progs original-mh-lib-progs))
|
||||
(if temp-home-dir (delete-directory temp-home-dir t))
|
||||
(setenv "MH" original-mh-envvar))))
|
||||
|
||||
(defun mh-test-utils-setup ()
|
||||
"Set dynamically bound variables needed by mock and/or variants.
|
||||
Call `mh-variant-set' to look through the directories named by
|
||||
envionment variable `TEST_MH_PATH' (default: `mh-path' and `mh-sys-path')
|
||||
to find the MH variant to use, if any.
|
||||
Return the name of the root of the created directory tree, if any."
|
||||
(when (getenv "TEST_MH_PATH")
|
||||
;; force mh-variants to use only TEST_MH_PATH
|
||||
(setq mh-path (split-string (getenv "TEST_MH_PATH") path-separator t)
|
||||
mh-sys-path nil
|
||||
exec-path '("/bin" "/usr/bin")))
|
||||
(unless mh-test-variant-logged-already
|
||||
(mh-variant-set mh-variant)
|
||||
(setq mh-test-utils-variants mh-variants)
|
||||
(setq mh-test-variant-logged-already t))
|
||||
;; As `call-process'' and `file-directory-p' will be redefined, the
|
||||
;; native compiler will invoke `call-process' to compile the
|
||||
;; respective trampolines. To avoid interference with the
|
||||
;; `call-process' mocking, we build these ahead of time.
|
||||
(when (native-comp-available-p)
|
||||
;; As `call-process'' and `file-directory-p' will be redefined, the
|
||||
;; native compiler will invoke `call-process' to compile the
|
||||
;; respective trampolines. To avoid interference with the
|
||||
;; `call-process' mocking, we build these ahead of time.
|
||||
(mapc #'comp-subr-trampoline-install '(call-process file-directory-p)))
|
||||
(if mh-variant-in-use
|
||||
(mh-test-utils-setup-with-variant)
|
||||
|
||||
104
test/lisp/mh-e/test-all-mh-variants.sh
Executable file
104
test/lisp/mh-e/test-all-mh-variants.sh
Executable file
@ -0,0 +1,104 @@
|
||||
#! /bin/bash
|
||||
# Run the mh-utils-tests against all MH variants found on this system.
|
||||
|
||||
# Copyright (C) 2021 Free Software Foundation, Inc.
|
||||
|
||||
# This file is part of GNU Emacs.
|
||||
|
||||
# GNU Emacs is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
|
||||
# GNU Emacs is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with GNU Emacs. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
# Commentary:
|
||||
|
||||
# By default runs all tests; test names or Emacs-style regexps may be
|
||||
# given on the command line to run just those tests.
|
||||
#
|
||||
# Option -d turns on Emacs variable mh-test-utils-debug-mocks, which
|
||||
# causes the tests to output all interactions with the file system.
|
||||
|
||||
# If you want to run the tests for only one MH variant, you don't need
|
||||
# to use this script, because "make" can do it. See the commentary at
|
||||
# the top of ./mh-utils-tests.el for the recipe.
|
||||
|
||||
debug=
|
||||
if [[ "$1" = -* ]]; then
|
||||
if [[ "$1" != -d ]]; then
|
||||
echo "Usage: $(basename "$0") [-d] [test ...]" >&2
|
||||
exit 2
|
||||
fi
|
||||
debug=t
|
||||
shift
|
||||
fi
|
||||
|
||||
shopt -s extglob
|
||||
ert_test_list=()
|
||||
for tst; do
|
||||
# Guess the type the test spec
|
||||
case $tst in
|
||||
*[\[\].*+\\]*) # Regexp: put in string quotes
|
||||
ert_test_list+=("\"$tst\"")
|
||||
;;
|
||||
*) # Lisp expression, keyword, or symbol: use as is
|
||||
ert_test_list+=("$tst")
|
||||
;;
|
||||
esac
|
||||
done
|
||||
if [[ ${#ert_test_list[@]} -eq 0 ]]; then
|
||||
# t means true for all tests, runs everything
|
||||
ert_test_list=(t)
|
||||
fi
|
||||
|
||||
# This script is 3 directories down in the Emacs source tree.
|
||||
cd "$(dirname "$0")"
|
||||
cd ../../..
|
||||
emacs=(src/emacs --batch -Q)
|
||||
|
||||
# MH-E has a good list of directories where an MH variant might be installed,
|
||||
# so we look in each of those.
|
||||
read -r -a mh_sys_path \
|
||||
< <("${emacs[@]}" -l mh-e --eval "(princ mh-sys-path)" | sed 's/[()]//g')
|
||||
|
||||
have_done_mocked_variant=false
|
||||
declare -i tests_total=0 tests_passed=0
|
||||
|
||||
for path in "${mh_sys_path[@]}"; do
|
||||
if [[ ! -x "$path/mhparam" ]]; then
|
||||
if [[ "$have_done_mocked_variant" = false ]]; then
|
||||
have_done_mocked_variant=true
|
||||
else
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
echo "Testing with PATH $path"
|
||||
((++tests_total))
|
||||
# The LD_LIBRARY_PATH setting is needed
|
||||
# to run locally installed Mailutils.
|
||||
TEST_MH_PATH=$path TEST_MH_DEBUG=$debug \
|
||||
LD_LIBRARY_PATH=/usr/local/lib HOME=/nonexistent \
|
||||
"${emacs[@]}" -l ert \
|
||||
--eval "(setq load-prefer-newer t)" \
|
||||
--eval "(load \"$PWD/test/lisp/mh-e/mh-utils-tests\" nil t)" \
|
||||
--eval "(ert-run-tests-batch-and-exit '(or ${ert_test_list[*]}))" \
|
||||
&& ((++tests_passed))
|
||||
done
|
||||
|
||||
if (( tests_total == 0 )); then
|
||||
echo "NO tests run"
|
||||
exit 1
|
||||
elif (( tests_total == tests_passed )); then
|
||||
echo "All tested variants pass: $tests_passed/$tests_total"
|
||||
else
|
||||
echo "Tested variants passing: $tests_passed/$tests_total," \
|
||||
"FAILING: $((tests_total - tests_passed))/$tests_total"
|
||||
exit 1
|
||||
fi
|
||||
Loading…
x
Reference in New Issue
Block a user