haskell-te: 474e05a4a48e360da29a1662f2367ae921eeae92

     1: # Benchmark QuickSpec on the theories stored in benchmarks/
     2: 
     3: from json       import dumps as jdumps, loads as jloads
     4: from os         import chmod, getenv
     5: from parameters import timeout_secs
     6: from sexpdata   import loads as sloads, dumps as sdumps
     7: from stat       import S_IRWXU, S_IRWXG, S_IRWXO
     8: from util       import cached, generate_cache, pipe, reps, set_attributes, theories, timed_run
     9: 
    10: # def setup_cache():
    11: #     '''Run all repetitions on all theories up-front, to avoid expensive
    12: #     re-running of tools.'''
    13: 
    14: #     data = jloads(getenv('qsStandalone'))
    15: 
    16: #     def gen(theory, rep):
    17: #         '''Performs a single repetition of a single theory.'''
    18: #         thy    = data[theory]
    19: #         result = {'theory' : theory}
    20: 
    21: #         result.update(timed_run([thy['runner']], '', timeout_secs))
    22: 
    23: #         to_analyse         = result['stdout'] if result['success'] else '[]'
    24: #         analysis           = pipe([thy['analyser']], to_analyse)
    25: #         result['analysis'] = jloads(analysis['stdout'])
    26: 
    27: #         return result
    28: 
    29: #     # Run all repetitions of all theories
    30: #     result = generate_cache(data.keys(), gen)
    31: 
    32: #     # Merge in the theory-specific data once, rather than copying for each run
    33: #     for theory in data:
    34: #         result[theory].update(data[theory])
    35: 
    36: #     return result
    37: 
    38: # setup_cache.timeout = max(3600, timeout_secs * len(reps) * len(theories()))
    39: 
    40: # def track_data(cache, _):
    41: #     '''A dummy benchmark which spits out the raw data, for archiving.'''
    42: #     return cache
    43: # track_data.repeat      = 1
    44: # track_data.number      = 1
    45: # track_data.params      = (["dummy"],)
    46: # track_data.param_names = ["dummy"]
    47: 
    48: 
    49: # # Benchmarks
    50: 
    51: # def track_ground_truth(cache, theory):
    52: #     sexps = sloads('(\n{0}\n)'.format(cache[theory]['ground_truth']))
    53: #     return len(sexps)
    54: 
    55: # def track_equations(cache, rep, theory):
    56: #     return len(jloads(cached(cache, theory, rep, 'stdout')))
    57: 
    58: # def track_precision(cache, rep, theory):
    59: #     return cached(cache, theory, rep, 'analysis', 'precision')
    60: 
    61: # def track_recall(cache, rep, theory):
    62: #     return cached(cache, theory, rep, 'analysis', 'recall')
    63: 
    64: # def track_time(cache, rep, theory):
    65: #     return cached(cache, theory, rep, 'time')
    66: 
    67: # # Assign parameters to benchmarks
    68: 
    69: # set_attributes([track_ground_truth, track_equations, track_precision,
    70: #                 track_recall, track_time],
    71: #                {
    72: #                    'repeat'      : 1,
    73: #                    'number'      : 1,
    74: #                    'params'      : (reps, theories()),
    75: #                    'param_names' : ['rep', 'theory']
    76: #                })
    77: 
    78: # # The ground truth doesn't change across reps, so only track it per theory
    79: # track_ground_truth.params      = (theories())
    80: # track_ground_truth.param_names = ['theory']