# Author: Hubert Kario, (c) 2015-2019 # Released under Gnu GPL v2.0, see LICENSE file for details """Example MAC value fuzzer""" from __future__ import print_function import traceback import sys import getopt from itertools import chain from random import sample from tlsfuzzer.runner import Runner from tlsfuzzer.messages import Connect, ClientHelloGenerator, \ ClientKeyExchangeGenerator, ChangeCipherSpecGenerator, \ FinishedGenerator, ApplicationDataGenerator, \ fuzz_padding, AlertGenerator from tlsfuzzer.expect import ExpectServerHello, ExpectCertificate, \ ExpectServerHelloDone, ExpectChangeCipherSpec, ExpectFinished, \ ExpectAlert, ExpectClose, ExpectApplicationData, \ ExpectServerKeyExchange from tlsfuzzer.utils.lists import natural_sort_keys from tlsfuzzer.helpers import RSA_SIG_ALL, AutoEmptyExtension from tlslite.constants import CipherSuite, AlertLevel, AlertDescription, \ GroupName, ExtensionType from tlslite.extensions import SupportedGroupsExtension, \ SignatureAlgorithmsExtension, SignatureAlgorithmsCertExtension from tlslite.utils.compat import compatAscii2Bytes version = 6 def help_msg(): print("Usage: [-h hostname] [-p port] [[probe-name] ...]") print(" -h hostname name of the host to run the test against") print(" localhost by default") print(" -p port port number to use for connection, 4433 by default") print(" probe-name if present, will run only the probes with given") print(" names and not all of them, e.g \"sanity\"") print(" -e probe-name exclude the probe from the list of the ones run") print(" may be specified multiple times") print(" -x probe-name expect the probe to fail. When such probe passes despite being marked like this") print(" it will be reported in the test summary and the whole script will fail.") print(" May be specified multiple times.") print(" -X message expect the `message` substring in exception raised during") print(" execution of preceding expected failure probe") print(" usage: [-x probe-name] [-X exception], order is compulsory!") print(" -n num run 'num' or all(if 0) tests instead of default(all)") print(" (\"sanity\" tests are always executed)") print(" -d negotiate (EC)DHE instead of RSA key exchange") print(" --echo-headers expect the server to echo the headers (so its reply") print(" will be split over two ApplicationData records, not") print(" one for the tests with 2**14 byte payload)") print(" --help this message") def add_dhe_extensions(extensions): groups = [GroupName.secp256r1, GroupName.ffdhe2048] extensions[ExtensionType.supported_groups] = SupportedGroupsExtension()\ .create(groups) extensions[ExtensionType.signature_algorithms] = \ SignatureAlgorithmsExtension().create(RSA_SIG_ALL) extensions[ExtensionType.signature_algorithms_cert] = \ SignatureAlgorithmsCertExtension().create(RSA_SIG_ALL) def main(): """check if atypical padding is accepted by server""" host = "localhost" port = 4433 num_limit = None run_exclude = set() expected_failures = {} last_exp_tmp = None dhe = False echo = False argv = sys.argv[1:] opts, args = getopt.getopt(argv, "h:p:e:x:X:n:d", ["help", "echo-headers"]) for opt, arg in opts: if opt == '-h': host = arg elif opt == '-p': port = int(arg) elif opt == '-e': run_exclude.add(arg) elif opt == '-x': expected_failures[arg] = None last_exp_tmp = str(arg) elif opt == '-X': if not last_exp_tmp: raise ValueError("-x has to be specified before -X") expected_failures[last_exp_tmp] = str(arg) elif opt == '-n': num_limit = int(arg) elif opt == '-d': dhe = True elif opt == '--help': help_msg() sys.exit(0) elif opt == '--echo-headers': echo = True else: raise ValueError("Unknown option: {0}".format(opt)) if args: run_only = set(args) else: run_only = None conversations = {} conversation = Connect(host, port) node = conversation if dhe: ext = {} add_dhe_extensions(ext) ciphers = [CipherSuite.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, CipherSuite.TLS_DHE_RSA_WITH_AES_128_CBC_SHA, CipherSuite.TLS_EMPTY_RENEGOTIATION_INFO_SCSV] else: ext = None ciphers = [CipherSuite.TLS_RSA_WITH_AES_128_CBC_SHA, CipherSuite.TLS_EMPTY_RENEGOTIATION_INFO_SCSV] node = node.add_child(ClientHelloGenerator(ciphers, extensions=ext)) node = node.add_child(ExpectServerHello()) node = node.add_child(ExpectCertificate()) if dhe: node = node.add_child(ExpectServerKeyExchange()) node = node.add_child(ExpectServerHelloDone()) node = node.add_child(ClientKeyExchangeGenerator()) node = node.add_child(ChangeCipherSpecGenerator()) node = node.add_child(FinishedGenerator()) node = node.add_child(ExpectChangeCipherSpec()) node = node.add_child(ExpectFinished()) node = node.add_child(ApplicationDataGenerator( bytearray(b"GET / HTTP/1.0\r\n\r\n"))) node = node.add_child(ExpectApplicationData()) node = node.add_child(AlertGenerator(AlertLevel.warning, AlertDescription.close_notify)) node = node.add_child(ExpectAlert()) node.next_sibling = ExpectClose() conversations["sanity"] = conversation # check if SHA256 ciphers work conversation = Connect(host, port) node = conversation if dhe: ext = {} add_dhe_extensions(ext) ciphers = [CipherSuite.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, CipherSuite.TLS_DHE_RSA_WITH_AES_128_CBC_SHA256, CipherSuite.TLS_EMPTY_RENEGOTIATION_INFO_SCSV] else: ext = None ciphers = [CipherSuite.TLS_RSA_WITH_AES_128_CBC_SHA256, CipherSuite.TLS_EMPTY_RENEGOTIATION_INFO_SCSV] node = node.add_child(ClientHelloGenerator(ciphers, extensions=ext)) node = node.add_child(ExpectServerHello()) node = node.add_child(ExpectCertificate()) if dhe: node = node.add_child(ExpectServerKeyExchange()) node = node.add_child(ExpectServerHelloDone()) node = node.add_child(ClientKeyExchangeGenerator()) node = node.add_child(ChangeCipherSpecGenerator()) node = node.add_child(FinishedGenerator()) node = node.add_child(ExpectChangeCipherSpec()) node = node.add_child(ExpectFinished()) node = node.add_child(ApplicationDataGenerator( bytearray(b"GET / HTTP/1.0\r\n\r\n"))) node = node.add_child(ExpectApplicationData()) node = node.add_child(AlertGenerator(AlertLevel.warning, AlertDescription.close_notify)) node = node.add_child(ExpectAlert()) node.next_sibling = ExpectClose() conversations["sanity - SHA256 HMAC"] = conversation # check if SHA384 ciphers work conversation = Connect(host, port) node = conversation ext = {} add_dhe_extensions(ext) ciphers = [CipherSuite.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384, CipherSuite.TLS_EMPTY_RENEGOTIATION_INFO_SCSV] node = node.add_child(ClientHelloGenerator(ciphers, extensions=ext)) node = node.add_child(ExpectServerHello()) node = node.add_child(ExpectCertificate()) node = node.add_child(ExpectServerKeyExchange()) node = node.add_child(ExpectServerHelloDone()) node = node.add_child(ClientKeyExchangeGenerator()) node = node.add_child(ChangeCipherSpecGenerator()) node = node.add_child(FinishedGenerator()) node = node.add_child(ExpectChangeCipherSpec()) node = node.add_child(ExpectFinished()) node = node.add_child(ApplicationDataGenerator( bytearray(b"GET / HTTP/1.0\r\n\r\n"))) node = node.add_child(ExpectApplicationData()) node = node.add_child(AlertGenerator(AlertLevel.warning, AlertDescription.close_notify)) node = node.add_child(ExpectAlert()) node.next_sibling = ExpectClose() conversations["sanity - SHA384 HMAC"] = conversation # check if Encrypt Then Mac works conversation = Connect(host, port) node = conversation ext = {ExtensionType.encrypt_then_mac: AutoEmptyExtension()} if dhe: add_dhe_extensions(ext) ciphers = [CipherSuite.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, CipherSuite.TLS_DHE_RSA_WITH_AES_128_CBC_SHA, CipherSuite.TLS_EMPTY_RENEGOTIATION_INFO_SCSV] else: ciphers = [CipherSuite.TLS_RSA_WITH_AES_128_CBC_SHA, CipherSuite.TLS_EMPTY_RENEGOTIATION_INFO_SCSV] node = node.add_child(ClientHelloGenerator(ciphers, extensions=ext)) extensions = {ExtensionType.encrypt_then_mac:None, ExtensionType.renegotiation_info:None} node = node.add_child(ExpectServerHello(extensions=extensions)) node = node.add_child(ExpectCertificate()) if dhe: node = node.add_child(ExpectServerKeyExchange()) node = node.add_child(ExpectServerHelloDone()) node = node.add_child(ClientKeyExchangeGenerator()) node = node.add_child(ChangeCipherSpecGenerator()) node = node.add_child(FinishedGenerator()) node = node.add_child(ExpectChangeCipherSpec()) node = node.add_child(ExpectFinished()) node = node.add_child(ApplicationDataGenerator( bytearray(b"GET / HTTP/1.0\r\n\r\n"))) node = node.add_child(ExpectApplicationData()) node = node.add_child(AlertGenerator(AlertLevel.warning, AlertDescription.close_notify)) node = node.add_child(ExpectAlert()) node.next_sibling = ExpectClose() conversations["sanity - encrypt then MAC"] = conversation # maximum size of padding conversation = Connect(host, port) node = conversation if dhe: ext = {} add_dhe_extensions(ext) ciphers = [CipherSuite.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, CipherSuite.TLS_DHE_RSA_WITH_AES_128_CBC_SHA, CipherSuite.TLS_EMPTY_RENEGOTIATION_INFO_SCSV] else: ext = None ciphers = [CipherSuite.TLS_RSA_WITH_AES_128_CBC_SHA, CipherSuite.TLS_EMPTY_RENEGOTIATION_INFO_SCSV] node = node.add_child(ClientHelloGenerator(ciphers, extensions=ext)) node = node.add_child(ExpectServerHello()) node = node.add_child(ExpectCertificate()) if dhe: node = node.add_child(ExpectServerKeyExchange()) node = node.add_child(ExpectServerHelloDone()) node = node.add_child(ClientKeyExchangeGenerator()) node = node.add_child(ChangeCipherSpecGenerator()) node = node.add_child(FinishedGenerator()) node = node.add_child(ExpectChangeCipherSpec()) node = node.add_child(ExpectFinished()) text = b"GET / HTTP/1.0\r\nX-bad: a\r\n\r\n" hmac_tag_length = 20 block_size = 16 # make sure that padding has full blocks to work with assert (len(text) + hmac_tag_length) % block_size == 0 node = node.add_child(fuzz_padding(ApplicationDataGenerator(text), min_length=255)) node = node.add_child(ExpectApplicationData()) node = node.add_child(AlertGenerator(AlertLevel.warning, AlertDescription.close_notify)) node = node.add_child(ExpectAlert(AlertLevel.warning, AlertDescription.close_notify)) node.next_sibling = ExpectClose() node = node.add_child(ExpectClose()) conversations["256 bytes of padding"] = \ conversation # maximum size of padding with SHA256 conversation = Connect(host, port) node = conversation if dhe: ext = {} add_dhe_extensions(ext) ciphers = [CipherSuite.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, CipherSuite.TLS_DHE_RSA_WITH_AES_128_CBC_SHA256, CipherSuite.TLS_EMPTY_RENEGOTIATION_INFO_SCSV] else: ext = None ciphers = [CipherSuite.TLS_RSA_WITH_AES_128_CBC_SHA256, CipherSuite.TLS_EMPTY_RENEGOTIATION_INFO_SCSV] node = node.add_child(ClientHelloGenerator(ciphers, extensions=ext)) node = node.add_child(ExpectServerHello()) node = node.add_child(ExpectCertificate()) if dhe: node = node.add_child(ExpectServerKeyExchange()) node = node.add_child(ExpectServerHelloDone()) node = node.add_child(ClientKeyExchangeGenerator()) node = node.add_child(ChangeCipherSpecGenerator()) node = node.add_child(FinishedGenerator()) node = node.add_child(ExpectChangeCipherSpec()) node = node.add_child(ExpectFinished()) text = b"GET / HTTP/1.0\r\nX-bad: aaaaa\r\n\r\n" hmac_tag_length = 32 block_size = 16 # make sure that padding has full blocks to work with assert (len(text) + hmac_tag_length) % block_size == 0, len(text) node = node.add_child(fuzz_padding(ApplicationDataGenerator(text), min_length=255)) node = node.add_child(ExpectApplicationData()) node = node.add_child(AlertGenerator(AlertLevel.warning, AlertDescription.close_notify)) node = node.add_child(ExpectAlert(AlertLevel.warning, AlertDescription.close_notify)) node.next_sibling = ExpectClose() node = node.add_child(ExpectClose()) conversations["256 bytes of padding with SHA256"] = \ conversation # ... and SHA384 conversation = Connect(host, port) node = conversation ext = {} groups = [GroupName.secp256r1, GroupName.ffdhe2048] ext[ExtensionType.supported_groups] = SupportedGroupsExtension()\ .create(groups) ext[ExtensionType.signature_algorithms] = \ SignatureAlgorithmsExtension().create(RSA_SIG_ALL) ext[ExtensionType.signature_algorithms_cert] = \ SignatureAlgorithmsCertExtension().create(RSA_SIG_ALL) ciphers = [CipherSuite.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384, CipherSuite.TLS_EMPTY_RENEGOTIATION_INFO_SCSV] node = node.add_child(ClientHelloGenerator(ciphers, extensions=ext)) node = node.add_child(ExpectServerHello()) node = node.add_child(ExpectCertificate()) node = node.add_child(ExpectServerKeyExchange()) node = node.add_child(ExpectServerHelloDone()) node = node.add_child(ClientKeyExchangeGenerator()) node = node.add_child(ChangeCipherSpecGenerator()) node = node.add_child(FinishedGenerator()) node = node.add_child(ExpectChangeCipherSpec()) node = node.add_child(ExpectFinished()) text = b"GET / HTTP/1.0\r\nX-bad: aaaaa\r\n\r\n" hmac_tag_length = 48 block_size = 16 # make sure that padding has full blocks to work with assert (len(text) + hmac_tag_length) % block_size == 0, len(text) node = node.add_child(fuzz_padding(ApplicationDataGenerator(text), min_length=255)) node = node.add_child(ExpectApplicationData()) node = node.add_child(AlertGenerator(AlertLevel.warning, AlertDescription.close_notify)) node = node.add_child(ExpectAlert()) node.next_sibling = ExpectClose() conversations["256 bytes of padding with SHA384"] = \ conversation # longest possible padding with max size Application data conversation = Connect(host, port) node = conversation if dhe: ext = {} add_dhe_extensions(ext) ciphers = [CipherSuite.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, CipherSuite.TLS_DHE_RSA_WITH_AES_128_CBC_SHA, CipherSuite.TLS_EMPTY_RENEGOTIATION_INFO_SCSV] else: ext = None ciphers = [CipherSuite.TLS_RSA_WITH_AES_128_CBC_SHA, CipherSuite.TLS_EMPTY_RENEGOTIATION_INFO_SCSV] node = node.add_child(ClientHelloGenerator(ciphers, extensions=ext)) node = node.add_child(ExpectServerHello()) node = node.add_child(ExpectCertificate()) if dhe: node = node.add_child(ExpectServerKeyExchange()) node = node.add_child(ExpectServerHelloDone()) node = node.add_child(ClientKeyExchangeGenerator()) node = node.add_child(ChangeCipherSpecGenerator()) node = node.add_child(FinishedGenerator()) node = node.add_child(ExpectChangeCipherSpec()) node = node.add_child(ExpectFinished()) text = b"GET / HTTP/1.0\r\nX-bad: a\r\n" for i in range(3): text += b"X-ba" + compatAscii2Bytes(str(i)) + b": " + \ b"a" * (4096 - 9) + b"\r\n" text += b"X-ba3: " + b"a" * (4096 - 37) + b"\r\n" text += b"\r\n" assert len(text) == 2**14, len(text) node = node.add_child(fuzz_padding(ApplicationDataGenerator(text), min_length=252)) if echo: node = node.add_child(ExpectApplicationData(size=2**14)) node = node.add_child(ExpectApplicationData()) node = node.add_child(AlertGenerator(AlertLevel.warning, AlertDescription.close_notify)) node = node.add_child(ExpectAlert(AlertLevel.warning, AlertDescription.close_notify)) node.next_sibling = ExpectClose() node = node.add_child(ExpectClose()) conversations["2^14 bytes of AppData with 253 bytes of padding (SHA1)"] = \ conversation # longest possible padding with max size Application data (and EtM) conversation = Connect(host, port) node = conversation ext = {ExtensionType.encrypt_then_mac: AutoEmptyExtension()} if dhe: add_dhe_extensions(ext) ciphers = [CipherSuite.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, CipherSuite.TLS_DHE_RSA_WITH_AES_128_CBC_SHA, CipherSuite.TLS_EMPTY_RENEGOTIATION_INFO_SCSV] else: ciphers = [CipherSuite.TLS_RSA_WITH_AES_128_CBC_SHA, CipherSuite.TLS_EMPTY_RENEGOTIATION_INFO_SCSV] node = node.add_child(ClientHelloGenerator(ciphers, extensions=ext)) extensions = {ExtensionType.encrypt_then_mac:None, ExtensionType.renegotiation_info:None} node = node.add_child(ExpectServerHello(extensions=extensions)) node = node.add_child(ExpectCertificate()) if dhe: node = node.add_child(ExpectServerKeyExchange()) node = node.add_child(ExpectServerHelloDone()) node = node.add_child(ClientKeyExchangeGenerator()) node = node.add_child(ChangeCipherSpecGenerator()) node = node.add_child(FinishedGenerator()) node = node.add_child(ExpectChangeCipherSpec()) node = node.add_child(ExpectFinished()) text = b"GET / HTTP/1.0\r\nX-bad: a\r\n" for i in range(3): text += b"X-ba" + compatAscii2Bytes(str(i)) + b": " + \ b"a" * (4096 - 9) + b"\r\n" text += b"X-ba3: " + b"a" * (4096 - 37) + b"\r\n" text += b"\r\n" assert len(text) == 2**14, len(text) node = node.add_child(fuzz_padding(ApplicationDataGenerator(text), min_length=255)) if echo: node = node.add_child(ExpectApplicationData(size=2**14)) node = node.add_child(ExpectApplicationData()) node = node.add_child(AlertGenerator(AlertLevel.warning, AlertDescription.close_notify)) node = node.add_child(ExpectAlert(AlertLevel.warning, AlertDescription.close_notify)) node.next_sibling = ExpectClose() node = node.add_child(ExpectClose()) conversations["2^14 bytes of AppData with 256 bytes of padding (SHA1 " "+ Encrypt then MAC)"] = \ conversation # longest possible padding with max size Application data with SHA256 conversation = Connect(host, port) node = conversation if dhe: ext = {} add_dhe_extensions(ext) ciphers = [CipherSuite.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, CipherSuite.TLS_DHE_RSA_WITH_AES_128_CBC_SHA256, CipherSuite.TLS_EMPTY_RENEGOTIATION_INFO_SCSV] else: ext = None ciphers = [CipherSuite.TLS_RSA_WITH_AES_128_CBC_SHA256, CipherSuite.TLS_EMPTY_RENEGOTIATION_INFO_SCSV] node = node.add_child(ClientHelloGenerator(ciphers, extensions=ext)) node = node.add_child(ExpectServerHello()) node = node.add_child(ExpectCertificate()) if dhe: node = node.add_child(ExpectServerKeyExchange()) node = node.add_child(ExpectServerHelloDone()) node = node.add_child(ClientKeyExchangeGenerator()) node = node.add_child(ChangeCipherSpecGenerator()) node = node.add_child(FinishedGenerator()) node = node.add_child(ExpectChangeCipherSpec()) node = node.add_child(ExpectFinished()) text = b"GET / HTTP/1.0\r\nX-bad: a\r\n" for i in range(3): text += b"X-ba" + compatAscii2Bytes(str(i)) + b": " + \ b"a" * (4096 - 9) + b"\r\n" text += b"X-ba3: " + b"a" * (4096 - 37) + b"\r\n" text += b"\r\n" assert len(text) == 2**14, len(text) node = node.add_child(fuzz_padding(ApplicationDataGenerator(text), min_length=255)) if echo: node = node.add_child(ExpectApplicationData(size=2**14)) node = node.add_child(ExpectApplicationData()) node = node.add_child(AlertGenerator(AlertLevel.warning, AlertDescription.close_notify)) node = node.add_child(ExpectAlert(AlertLevel.warning, AlertDescription.close_notify)) node.next_sibling = ExpectClose() node = node.add_child(ExpectClose()) conversations["2^14 bytes of AppData with 256 bytes of padding (SHA256)"] = \ conversation conversation = Connect(host, port) node = conversation ext = {} add_dhe_extensions(ext) ciphers = [CipherSuite.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384, CipherSuite.TLS_EMPTY_RENEGOTIATION_INFO_SCSV] node = node.add_child(ClientHelloGenerator(ciphers, extensions=ext)) node = node.add_child(ExpectServerHello()) node = node.add_child(ExpectCertificate()) node = node.add_child(ExpectServerKeyExchange()) node = node.add_child(ExpectServerHelloDone()) node = node.add_child(ClientKeyExchangeGenerator()) node = node.add_child(ChangeCipherSpecGenerator()) node = node.add_child(FinishedGenerator()) node = node.add_child(ExpectChangeCipherSpec()) node = node.add_child(ExpectFinished()) text = b"GET / HTTP/1.0\r\nX-bad: a\r\n" for i in range(3): text += b"X-ba" + compatAscii2Bytes(str(i)) + b": " + \ b"a" * (4096 - 9) + b"\r\n" text += b"X-ba3: " + b"a" * (4096 - 37) + b"\r\n" text += b"\r\n" assert len(text) == 2**14, len(text) node = node.add_child(fuzz_padding(ApplicationDataGenerator(text), min_length=255)) if echo: node = node.add_child(ExpectApplicationData(size=2**14)) node = node.add_child(ExpectApplicationData()) node = node.add_child(AlertGenerator(AlertLevel.warning, AlertDescription.close_notify)) node = node.add_child(ExpectAlert()) node.next_sibling = ExpectClose() conversations["2^14 bytes of AppData with 256 bytes of padding (SHA384)"] = \ conversation # run the conversation good = 0 bad = 0 xfail = 0 xpass = 0 failed = [] xpassed = [] if not num_limit: num_limit = len(conversations) # make sure that sanity test is run first and last # to verify that server was running and kept running throughout sanity_tests = [('sanity', conversations['sanity'])] if run_only: if num_limit > len(run_only): num_limit = len(run_only) regular_tests = [(k, v) for k, v in conversations.items() if k in run_only] else: regular_tests = [(k, v) for k, v in conversations.items() if (k != 'sanity') and k not in run_exclude] sampled_tests = sample(regular_tests, min(num_limit, len(regular_tests))) ordered_tests = chain(sanity_tests, sampled_tests, sanity_tests) for c_name, c_test in ordered_tests: if run_only and c_name not in run_only or c_name in run_exclude: continue print("{0} ...".format(c_name)) runner = Runner(c_test) res = True exception = None try: runner.run() except Exception as exp: exception = exp print("Error while processing") print(traceback.format_exc()) res = False if c_name in expected_failures: if res: xpass += 1 xpassed.append(c_name) print("XPASS-expected failure but test passed\n") else: if expected_failures[c_name] is not None and \ expected_failures[c_name] not in str(exception): bad += 1 failed.append(c_name) print("Expected error message: {0}\n" .format(expected_failures[c_name])) else: xfail += 1 print("OK-expected failure\n") else: if res: good += 1 print("OK\n") else: bad += 1 failed.append(c_name) print("Test if server can handle records with large (but valid) padding") print("Tests both with small records and records that carry maximum amount") print("of user data.\n") print("Test end") print(20 * '=') print("version: {0}".format(version)) print(20 * '=') print("TOTAL: {0}".format(len(sampled_tests) + 2*len(sanity_tests))) print("SKIP: {0}".format(len(run_exclude.intersection(conversations.keys())))) print("PASS: {0}".format(good)) print("XFAIL: {0}".format(xfail)) print("FAIL: {0}".format(bad)) print("XPASS: {0}".format(xpass)) print(20 * '=') sort = sorted(xpassed ,key=natural_sort_keys) if len(sort): print("XPASSED:\n\t{0}".format('\n\t'.join(repr(i) for i in sort))) sort = sorted(failed, key=natural_sort_keys) if len(sort): print("FAILED:\n\t{0}".format('\n\t'.join(repr(i) for i in sort))) if bad or xpass: sys.exit(1) if __name__ == "__main__": main()