simple-test.lisp 13.9 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13
;;;; -*- mode:lisp;coding:utf-8 -*-
;;;;**************************************************************************
;;;;FILE:               simple-test.lisp
;;;;LANGUAGE:           Common-Lisp
;;;;SYSTEM:             Common-Lisp
;;;;USER-INTERFACE:     NONE
;;;;DESCRIPTION
;;;;    
;;;;    Defines a simple test tool.
;;;;    
;;;;AUTHORS
;;;;    <PJB> Pascal J. Bourguignon <[email protected]>
;;;;MODIFICATIONS
14 15
;;;;    2015-01-25 <PJB> Added format-control/arguments to
;;;;                     progress-failure and macros callint it.
16 17 18 19 20
;;;;    2010-12-14 <PJB> Created.
;;;;BUGS
;;;;LEGAL
;;;;    AGPL3
;;;;    
21
;;;;    Copyright Pascal J. Bourguignon 2010 - 2015
22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37
;;;;    
;;;;    This program is free software: you can redistribute it and/or modify
;;;;    it under the terms of the GNU Affero General Public License as published by
;;;;    the Free Software Foundation, either version 3 of the License, or
;;;;    (at your option) any later version.
;;;;    
;;;;    This program is distributed in the hope that it will be useful,
;;;;    but WITHOUT ANY WARRANTY; without even the implied warranty of
;;;;    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;;;;    GNU Affero General Public License for more details.
;;;;    
;;;;    You should have received a copy of the GNU Affero General Public License
;;;;    along with this program.  If not, see <http://www.gnu.org/licenses/>
;;;;**************************************************************************

(defpackage "COM.INFORMATIMAGO.COMMON-LISP.CESARUM.SIMPLE-TEST"
38 39
  (:use "COMMON-LISP"
        "COM.INFORMATIMAGO.COMMON-LISP.LISP-SEXP.SOURCE-FORM")
40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133
  (:export "*DEBUG-ON-ERROR*" "WITH-DEBUGGER-ON-ERROR"
           "DEFINE-TEST" "TEST" "ASSERT-TRUE" "EXPECT-CONDITION"

           "*VERBOSE-TALLY*" "*VERBOSE-PROGRESS*")
  (:documentation "
This package defines a simple test tool.

License:

    AGPL3
    
    Copyright Pascal J. Bourguignon 2010 - 2012
    
    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.
    
    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.
    
    You should have received a copy of the GNU Affero General Public License
    along with this program.
    If not, see <http://www.gnu.org/licenses/>
"))
(in-package "COM.INFORMATIMAGO.COMMON-LISP.CESARUM.SIMPLE-TEST")


(defvar *debug-on-error*          nil
  "Whether an error in a test should go to the debugger.")
(defvar *success-count*           0
  "The total number of successful tests.")
(defvar *failure-count*           0
  "The total number of failed tests.")

(defvar *verbose-tally* t
  "Whether to print the number of successful, failed and performed tests.")
(defvar *verbose-progress* nil
  "Whether to display dots or exclamation points while testing.")

(defvar *test-output* *standard-output*)

;; Private:
(defvar *last-success-p*          nil)
(defvar *current-test-name*       nil)
(defvar *current-test-parameters* nil)
(defvar *current-test-printed-p*  nil)
(defvar *report-string*           "")
(defparameter *cr*                #\return)


(defun progress-start ()
  (setf *success-count*  0
        *failure-count*  0
        *last-success-p* nil
        *report-string*  (make-array 8
                                     :element-type 'character
                                     :adjustable t
                                     :fill-pointer 0))
  (values))


(defun verbose (default)
  (and default
       (or (not *load-pathname*)
           *load-verbose*
           (and (find-package "ASDF")
                (find-symbol "*ASDF-VERBOSE*" "ASDF")
                (symbol-value (find-symbol "*ASDF-VERBOSE*" "ASDF")))
           (and (find-package "QUICKLISP")
                (find-symbol "*QUICKLOAD-VERBOSE*" "QUICKLISP-CLIENT")
                (symbol-value (find-symbol "*QUICKLOAD-VERBOSE*" "QUICKLISP-CLIENT"))))))


(defun progress-report (new-last-succcess-p)
  (setf *last-success-p* new-last-succcess-p)
  (when (verbose *verbose-progress*)
    (if *last-success-p*
        (format *test-output* "~A" (aref *report-string* (1- (length *report-string*))))
        (format *test-output* "~&~A" *report-string*))
    (finish-output *test-output*))
  (values))


(defun progress-success ()
  (incf *success-count*)
  (vector-push-extend #\. *report-string*)
  (progress-report t))


(defun current-test-identification (&optional max-length)
  (let ((*print-circle* nil))
134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179
    (if (or (null max-length) (null *current-test-parameters*))
        (format nil "(~{~S~^ ~})" (cons *current-test-name* *current-test-parameters*))
        (let* ((items (mapcar (lambda (parameter)
                                (let ((label (let ((*package* (if (and (symbolp parameter)
                                                                       (symbol-package parameter))
                                                                  (symbol-package parameter)
                                                                  *package*)))
                                               (format nil "~S" parameter))))
                                  (list (length label) label)))
                              (cons *current-test-name* *current-test-parameters*)))
               (idlength (+ 1 (length items) (reduce (function +) items :key (function first))))
               (candidates (sort (butlast (loop
                                            :for cell :on items
                                            :collect cell))
                                 (function >)
                                 :key (function caadr))))
          (loop
            :while (and candidates (< max-length idlength))
            :do (progn
                  (decf idlength (1- (caadar candidates)))
                  (setf (car (cdadar candidates)) "…")
                  (pop candidates))
            :finally (return (format nil "(~{~A~^ ~})" (mapcar (function second) items))))))))


(defun test/current-test-identification ()
  (assert (equal (let ((*current-test-name* 'hello-world)
                       (*current-test-parameters* '()))
                   (current-test-identification  nil))
                 "(hello-world)"))

  (assert (equal (let ((*current-test-name* 'hello-world)
                       (*current-test-parameters* '((1 2 3 4) "howdy doo dabadaboo" #(a b c d e f))))
                   (current-test-identification  nil))
                 "(hello-world (1 2 3 4) \"howdy doo dabadaboo\" #(a b c d e f))"))

  (assert (equal (let ((*current-test-name* 'hello-world)
                       (*current-test-parameters* '((1 2 3 4) "howdy doo dabadaboo" #(a b c d e f))))
                   (current-test-identification  1))
                 "(hello-world … … …)"))

  (assert (equal (let ((*current-test-name*  'test/non-empty-vector-with-too-little-data)
                       (*current-test-parameters* '()))
                   (current-test-identification  20))
                 "(test/non-empty-vector-with-too-little-data)"))
  :success)
180 181 182 183 184 185 186 187 188 189



(defun progress-failure-message (expression message &rest arguments)
  (incf *failure-count*)
  (vector-push-extend #\! *report-string*)
  (unless *current-test-printed-p*
    (setf  *current-test-printed-p* t)
    (format *test-output* "~&~A" (current-test-identification)))
  (format *test-output* "~&Failure:     expression: [email protected]
190
                         ~&~?~%"
191 192 193 194
          expression message arguments)
  (progress-report nil))


195 196
(defun progress-failure (compare expression expected-result result
                         &optional places format-control &rest format-arguments)
197 198 199
  (progress-failure-message expression "~&           evaluates to: [email protected]
                                        ~&           which is not  [email protected]
                                        ~& to the expected result: [email protected]
200 201 202 203 204
                                        ~{~&~23A: ~S~}[email protected][[email protected]
                                        ~&~?~]"
                            result compare expected-result places
                            format-control format-arguments))

205 206 207 208 209 210


(defun progress-tally (success-count failure-count)
  (when (verbose *verbose-tally*)
    (let ((name-max-length 40))
     (flet ((genline (name)
211
              (format nil "~VA~5D ~A,~3D ~9A~:[,~3D ~8A~;~]."
212 213
                      name-max-length name
                      (+ success-count failure-count)
214 215 216 217
                      (format nil "test~P" (+ success-count failure-count))
                      success-count (format nil "success~[es~;~:;es~]" success-count)
                      (zerop failure-count)
                      failure-count (format nil "failure~P" failure-count))))
218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234
       (format *test-output* "~&~A~%"
               (genline  (current-test-identification name-max-length)))
       (finish-output *test-output*)
       ;; (let* ((test-name (current-test-identification name-max-length))
       ;;        (data (genline ""))
       ;;        (nlen (length test-name)))
       ;;   (format *test-output* "~&~A~%" 
       ;;           (if (and (< nlen (+ name-max-length 4)) (char= #\space (aref data nlen)))
       ;;               (progn
       ;;                 (replace data test-name)
       ;;                 data)
       ;;               (genline (concatenate 'string (subseq test-name 0 43) "…"))))
       ;;   (finish-output *test-output*))
       )))
  (values))


235 236

(defmacro assert-true (expression &optional places format-control &rest format-arguments)
237 238 239 240 241 242 243 244 245 246 247 248 249
  "Evaluates a test EXPRESSION and check it returns true.
EXAMPLE:  (assert-true (= 2 (+ 1 1))))
"
  (let ((vresult   (gensym "RESULT-")))
    `(let ((,vresult   (if *debug-on-error*
                           (handler-bind
                               ((error (function invoke-debugger)))
                             ,expression)
                           (handler-case
                               ,expression
                             (error (err) (list 'error (princ-to-string err)))))))
       (if ,vresult
           (progress-success)
250 251 252
           (progress-failure 'equivalent ',expression 't ,vresult
                             (list ,@(mapcan (lambda (place) `(',place ,place)) places))
                             ,format-control ,@format-arguments)))))
253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285


(defmacro expect-condition (condition-class expression)
  "Evaluates a test EXPRESSION and check that it signals a condition of the specified CONDITION-CLASS.
EXAMPLE:  (expect-condition division-by-zero (/ 1 0))
"
  (let ((body (gensym)))
    `(flet ((,body ()
                   ,expression
                   (progress-failure-message ',expression
                                             "Didn't signal the expected ~S condition."
                                             ',condition-class)))
       (if *debug-on-error*
           (block expect
             (handler-bind
                 ((,condition-class (lambda (condition)
                                      (declare (ignore condition))
                                      (progress-success)
                                      (return-from expect)))
                  (t (function invoke-debugger)))
               (,body)))
           (handler-case
               (,body)
             (,condition-class ()
               (progress-success))
             (t (condition)
               (progress-failure-message ',expression
                                         "Signaled an unexpected ~S condition instead of ~S."
                                         condition
                                         ',condition-class)))))))



286
(defmacro test (compare expression expected &optional places format-control &rest format-arguments)
287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302
  "Evaluates a test EXPRESSION and compare the result with EXPECTED (evaluated) using the COMPARE operator.
EXAMPLE:  (test equal (list 1 2 3) '(1 2 3))
"
  (let ((vresult   (gensym "RESULT-"))
        (vexpected (gensym "EXPECTED-")))
    `(let ((,vresult   (if *debug-on-error*
                           (handler-bind
                               ((error (function invoke-debugger)))
                             ,expression)
                           (handler-case
                               ,expression
                             (error (err) (list 'error (princ-to-string err))))))
           (,vexpected ,expected))
       (if (,compare ,vresult ,vexpected)
           (progress-success)
           (progress-failure ',compare ',expression ,vexpected ,vresult
303 304 305
                             (list ,@(mapcan (lambda (place) `(',place ,place)) places))
                             ,format-control
                             ,@format-arguments)))))
306 307 308 309 310 311 312 313


(defmacro define-test (name parameters &body body)
  "Like DEFUN, but wraps the body in test reporting boilerplate."
  (let ((mandatory (loop
                     :for param :in parameters
                     :while (symbolp param)
                     :collect param)))
314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332
    (multiple-value-bind (docstrings declarations forms) (parse-body :lambda body)
      `(defun ,name ,parameters
         ,@docstrings
         ,@declarations
         (multiple-value-bind (successes failures)
             (let ((*success-count* 0)
                   (*failure-count* 0)
                   (*current-test-name*        ',name)
                   (*current-test-parameters* (list ,@mandatory))
                   (*current-test-printed-p*  nil))
               (progress-start)
               (progn ,@forms)
               (progress-tally *success-count* *failure-count*)
               (values *success-count* *failure-count*))
           (incf *success-count* successes)
           (incf *failure-count* failures)
           (if (zerop failures)
               :success
               :failure))))))
333 334 335 336 337 338

(defmacro with-debugger-on-error (&body body)
  `(let ((*debug-on-error* t))
     ,@body))

;;;; THE END ;;;;