#!/usr/local/bin/python
# testsuite.py - functional and regression tests for ExpectPy
# Copyright (C) 1998 Michael P. Reilly, All rights reserved
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.

# $Log: testsuite.py,v $
# Revision 1.10  1998/08/24 00:49:44  arcege
# Delete ed(1) HUP file, if present
#
# Revision 1.9  1998/08/22 00:33:48  arcege
# Fix the verbage for the new test filenames
#
# Revision 1.8  1998/08/19 11:57:16  arcege
# Rename test programs, allows for ordered tests
#
# Revision 1.7  1998/08/19 11:50:59  arcege
# Fix some things:
# * Hard code the ExpectPy module into the virtual test environment
# * Give the test environment a different stdin
# * Remove the test '__main__' from the modules_dict
# * Handle the new sys.exc_info function and remove old debuging statement
#
# Revision 1.6  1998/08/03 11:04:54  arcege
# Import ExpectPy inside testsuite to prevent delays inside test cases.
# Add command-line option support: -d for srcdir, -D for debug messages.
# If no test cases are specified on the command-line, find them in srcdir.
# Change output to tabular form by default.
#
# Revision 1.5  1998/07/27 12:21:04  arcege
# Copyright information
#
# Revision 1.4  1998/07/26 18:20:23  arcege
# Forgot to redirect stdin
#
# Revision 1.3  1998/07/25 21:22:10  arcege
# Allow testsuite.py to take multiple tests and handle exceptions
#
# Revision 1.2  1998/07/25 20:41:45  arcege
# Add RCS keywords
#

import __builtin__
import getopt, glob, os, string, sys, traceback, types
import ihooks

# import ExpectPy now so we:
# 1. don't have a delay during testing, and
# 2. determine errors with importing the module
sys.path.insert(0, os.pardir)
import ExpectPy

try:
  from cStringIO import StringIO
except ImportError:
  # it would be much better if cStringIO was used, StringIO will slow down
  # the tests
  from StringIO import StringIO
try:
  import timing
except ImportError:
  # make stubs for the timing module (a dict could probably be used here,
  # but I like this implementation)
  import imp
  timing = imp.new_module('timing')
  timing.start = timing.finish = lambda *args: None
  timing.seconds = timing.micro = timing.milli = lambda *args: 0
  del imp

class DHooks(ihooks.Hooks):
  def __init__(self, verbose=0):
    ihooks.Hooks.__init__(self, verbose)
    self.modules = {}
  def modules_dict(self):
    return self.modules
  def default_path(self):
    return self.modules['sys'].path

class TestEnv:
  def __init__(self):
    self.hooks = hooks = DHooks()
    self.importer = ihooks.ModuleImporter(
      ihooks.FancyModuleLoader(hooks)
    )

    self.builtins = builtins = self.add_module('__builtin__')
    for name, value in __builtin__.__dict__.items():
      if name == '_':
	value = None
      elif type(value) is types.ListType:
	value = value[:]
      elif type(value) is types.DictType:
	value = value.copy()
      setattr(builtins, name, value)

    self.sys = d_sys = self.add_module('sys')
    for name, value in sys.__dict__.items():
      if name == 'modules':
	value = hooks.modules_dict()
      elif name == '__builtins__':
	value = builtins
      elif type(value) is types.ListType:
	value = value[:]
      setattr(d_sys, name, value)
    d_sys.path.insert(0, os.pardir)      # allow importing from parent dir
    d_sys.modules['ExpectPy'] = ExpectPy

    builtins.__import__ = self.importer.import_module
    builtins.reload = self.importer.reload
    builtins.unload = self.importer.unload

  def add_module(self, name):
    return self.hooks.add_module(name)
  def new_main(self):
    self.sys.stdin = StringIO()
    self.sys.stdout = self.sys.stderr = StringIO()
    main = self.add_module('__main__')
    main.__builtins__ = self.builtins
    return main
  def delete(self, module):
    data = self.sys.stdout.getvalue()
    self.sys.stdout.close()
    self.sys.stdin = self.sys.stdout = self.sys.stderr = None
    self.importer.unload(module)
    try:
      del self.hooks.modules_dict()[module.__name__]
    except:
      pass
    return data

class Testcounter:
  stream = sys.stdout
  def __init__(self, max, debug):
    self.max, self.debug = max, debug
    self.size = len(str(max-1))             # max characters in cases numbers
    self.brklimit = int(80 / (self.size+1)) # how many iterations are on a line
    self.count = 0
  def __del__(self):
    if self.debug and (self.count % self.brklimit):
      self.stream.write("\n")
      self.stream.flush()
  def write(self, num):
    if self.debug:
      self.stream.write("%0*d" % (self.size, num))
      self.count = self.count + 1
      if self.count % self.brklimit:
	self.stream.write(" ")
      else:
	self.stream.write("\n")
      self.stream.flush()

def start_test(numcases, program):
  global debug
  testenv = TestEnv()
  cases, timesum, howlong = [], 0L, 0
  debugout = Testcounter(numcases, debug)

  for i in xrange(numcases):
    debugout.write(i)
    this_main = testenv.new_main()

    timing.start()
    try:
      try:
        execfile(program, this_main.__dict__, this_main.__dict__)
      except (IOError, ImportError), value:
	if hasattr(sys, 'exc_info'):
	  exc_t, exc_v, exc_b = sys.exc_info()
	else:
	  exc_t, exc_v, exc_b = sys.exc_type, sys.exc_value, sys.exc_traceback
	raise exc_t, exc_v, exc_b
      except:
        pass  # for now, do nothing
    finally:
      timing.finish()

    howlong = timing.milli()
    timesum = timesum + long(howlong)
    errcode = hasattr(this_main, 'status') and (this_main.status != 0)
    data = testenv.delete(this_main)
    cases.append({
      'errcode': errcode,
      'timing': howlong,
      'output': data
    })
  # in case the test used ed(1)
  if os.path.exists('ed.hup'):
    os.remove('ed.hup')
  return cases, int(timesum / numcases)

def compare_outputs(cases):
  uniq = {}
  for case in cases:
    if not uniq.has_key(case['output']):
      uniq[case['output']] = None
  return len(uniq)

def howd_we_do(cases):
  s = f = 0
  for case in cases:
    if case['errcode']:
      s = s + 1
    else:
      f = f + 1
  return s, f

def caveat():
  if ExpectPy.re_type == 're':
    print """\
PCRE has been compiled into ExpectPy, so the TclRE test should fail.
Do not be alarmed."""
  elif ExpectPy.re_type == 'tclre':
    print """\
TclRE has been compiled into ExpectPy, so the PCRE test should fail.
Do not be alarmed."""
  elif ExpectPy.re_type is None:
    print """\
No regular expression support was compiled into ExpectPy, both PCRE
and TclRE tests should fail.  Do not be alarmed."""

if __name__ == '__main__':
  debug, dir = 0, os.curdir
  try:
    opts, args = getopt.getopt(sys.argv[1:], 'Dd:')
    for opt, val in opts:
      if opt == '-D':
	debug = 1
      elif opt == '-d':
	dir = val
  except getopt.error, value:
    raise SystemError, sys.argv[0] + ': ' + value

  numcases = 100
  if os.environ.has_key("TESTCASES"):
    numcases = string.atoi(os.environ["TESTCASES"])
  if args:
    testcases = map(lambda f, d=dir: os.path.join(d, f), args)
  else:
    testcases = glob.glob(os.path.join(dir, "[0-9][0-9]*.py"))

  caveat()
  if not debug:
    print 'Test        \t Success\t     Average\tRegression'
    print 'Name        \t    rate\truntime (ms)\ttest (output)'

  testcases.sort()
  for program in testcases:
    name = os.path.basename(program)
    if debug: print 'Testing', name
    try:
      cases, ave_time = start_test(numcases, program)
      s, f = howd_we_do(cases)
      uniq = compare_outputs(cases)
      reg_test = (uniq == 1) and 'passed' or 'failed'

      if debug:
	print '''\
Success rate: %d%%\tAverage run time: %d msec.
Regression test %s ''' % (
	  int(s*100/numcases), ave_time, reg_test
	)
      else:
	print '%-12s\t%8d%%\t%12d\t%s' % (
	  name,
	  int(s*100/numcases), ave_time, reg_test
	)

    except ImportError, value:
      print "No such module:",  value
    except IOError, (errno, message):
      print "%s: %s" % (program, message)
    except:
      print "Exception raised - test aborting"
      traceback.print_exc()