import unittest # timeout.py import timeout # Requires gradescope_utils from gradescope_utils.autograder_utils.decorators import number, weight, visibility, hide_errors import subprocess from time import sleep from re import sub # utils.py from utils import * import os # Main unit test class class TestDiff(unittest.TestCase): # Array of all the expected file names files = ['PA3.c', 'printMenu.c', 'printArray.c', 'convertCase.c', 'wordSearch.c', 'defs.h', 'makefile'] # Set up unittest environment def setUp(self): self.maxDiff = None self.longMessage = False self.addTypeEqualityFunc(str, self.customCompare) # Define custom TypeEquality function that calls function from utils.py def customCompare(self, first, second, msg=None): customAssertMultiLineEqual(self, first, second, msg) # Associated test number within Gradescope @number("1") # Test visibility @visibility("visible") # Associated point value within Gradescope @weight(0) def test_01checkFiles(self): """Ensure all required files are present""" checkFiles(self.files) sleep(1) # Associated test number within Gradescope @number("2") # Test visibility @visibility("visible") # Associated point value within Gradescope @weight(5) def test_02CompileMain(self): # Title used by Gradescope """Clean compile (using makefile)""" checkSourceFiles(self, self.files) # Create a subprocess to run the student's Makefile to ensure it compiles test = subprocess.Popen(["make"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) try: stdout, stderr = test.communicate(timeout=10) except (subprocess.TimeoutExpired): os.popen(removeLastExecutableCommand) kill_fail(test, self, compileTimeoutErrorMessage) # Try to decode stderr try: stderr = stderr.strip().decode('utf-8') test.kill() self.assertTrue(stderr == "", msg=("See compiler output:\n" + stderr)) # Catch exception for decode error except (UnicodeDecodeError): kill_fail(test, self, compileDecodeErrorMessage) test.terminate() # Associated test number within Gradescope @number("3") # Test visibility @visibility("visible") # Individual test case timeout (in seconds) @timeout.timeout(10, exception_message=wrap(programTimeoutErrorMessage, 65), use_signals=False) # Associated point value within Gradescope @weight(2.5) def test_03EnforceValidMenuInput(self): # Title used by Gradescope """Program enforces valid menu input""" checkExecutables(self, os.popen(findLastExecutableCommand).read().split()) # Create a subprocess to run the student's code to obtain an output test = subprocess.Popen(["make -s run < input/enforceValidMenuInput.txt"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = test.communicate() try: checkRuntimeErrors(test, self, stdout, stderr) # Try to decode stdout try: stdout = checkForUninitializedChars(stdout.strip().decode('utf-8')) test.kill() # Open reference output and decode reference = open('reference/enforceValidMenuInput.txt', 'rb').read().strip().decode('utf-8') # Remove empty lines from both output and reference stdout = removeEmptyLines(stdout) reference = removeEmptyLines(reference) # Check the contents of stdout against reference self.customCompare(stdout, reference, msg='Program output does not match expected output') # Catch exception for decode error except (UnicodeDecodeError): kill_fail(test, self, decodeErrorMessage) # Catch exception for uninitialized characters except (UninitializedCharError): kill_fail(test, self, uninitializedCharacterMessage) # Catch runtime error exceptions except (RuntimeAbort, RuntimeSegFault, RuntimeFPE, RuntimeBusError, RuntimeIllegalInstruction, MakefileError): pass test.terminate() # Associated test number within Gradescope @number("4") # Test visibility @visibility("visible") # Individual test case timeout (in seconds) @timeout.timeout(10, exception_message=wrap(programTimeoutErrorMessage, 65), use_signals=False) # Associated point value within Gradescope @weight(5) def test_04PrintPoem(self): # Title used by Gradescope """Program prints poem""" checkExecutables(self, os.popen(findLastExecutableCommand).read().split()) # Create a subprocess to run the student's code to obtain an output test = subprocess.Popen(["make -s run < input/printPoem.txt"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = test.communicate() try: checkRuntimeErrors(test, self, stdout, stderr) # Try to decode stdout try: stdout = checkForUninitializedChars(stdout.strip().decode('utf-8')) test.kill() # Open reference output and decode reference = open('reference/printPoem.txt', 'rb').read().strip().decode('utf-8') # Remove empty lines from both output and reference stdout = removeEmptyLines(stdout) reference = removeEmptyLines(reference) # Check the contents of stdout against reference self.customCompare(stdout, reference, msg='Program output does not match expected output') # Catch exception for decode error except (UnicodeDecodeError): kill_fail(test, self, decodeErrorMessage) # Catch exception for uninitialized characters except (UninitializedCharError): kill_fail(test, self, uninitializedCharacterMessage) # Catch runtime error exceptions except (RuntimeAbort, RuntimeSegFault, RuntimeFPE, RuntimeBusError, RuntimeIllegalInstruction, MakefileError): pass test.terminate() # Associated test number within Gradescope @number("5") # Test visibility @visibility("visible") # Individual test case timeout (in seconds) @timeout.timeout(10, exception_message=wrap(programTimeoutErrorMessage, 65), use_signals=False) # Associated point value within Gradescope @weight(2.5) def test_05NumLines(self): # Title used by Gradescope """Program prints number of lines in poem""" checkExecutables(self, os.popen(findLastExecutableCommand).read().split()) # Create a subprocess to run the student's code to obtain an output test = subprocess.Popen(["make -s run < input/numLines.txt"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = test.communicate() try: checkRuntimeErrors(test, self, stdout, stderr) # Try to decode stdout try: stdout = checkForUninitializedChars(stdout.strip().decode('utf-8')) test.kill() # Open reference output and decode reference = open('reference/numLines.txt', 'rb').read().strip().decode('utf-8') # Remove empty lines from both output and reference stdout = removeEmptyLines(stdout) reference = removeEmptyLines(reference) # Check the contents of stdout against reference self.customCompare(stdout, reference, msg='Program output does not match expected output') # Catch exception for decode error except (UnicodeDecodeError): kill_fail(test, self, decodeErrorMessage) # Catch exception for uninitialized characters except (UninitializedCharError): kill_fail(test, self, uninitializedCharacterMessage) # Catch runtime error exceptions except (RuntimeAbort, RuntimeSegFault, RuntimeFPE, RuntimeBusError, RuntimeIllegalInstruction, MakefileError): pass test.terminate() # Associated test number within Gradescope @number("6") # Test visibility @visibility("visible") # Individual test case timeout (in seconds) @timeout.timeout(10, exception_message=wrap(programTimeoutErrorMessage, 65), use_signals=False) # Associated point value within Gradescope @weight(5) def test_06ConvertCase(self): # Title used by Gradescope """Program prints poem with converted case""" checkExecutables(self, os.popen(findLastExecutableCommand).read().split()) # Create a subprocess to run the student's code to obtain an output test = subprocess.Popen(["make -s run < input/convertCase.txt"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = test.communicate() try: checkRuntimeErrors(test, self, stdout, stderr) # Try to decode stdout try: stdout = checkForUninitializedChars(stdout.strip().decode('utf-8')) test.kill() # Open reference output and decode reference = open('reference/convertCase.txt', 'rb').read().strip().decode('utf-8') # Remove empty lines from both output and reference stdout = removeEmptyLines(stdout) reference = removeEmptyLines(reference) # Check the contents of stdout against reference self.customCompare(stdout, reference, msg='Program output does not match expected output') # Catch exception for decode error except (UnicodeDecodeError): kill_fail(test, self, decodeErrorMessage) # Catch exception for uninitialized characters except (UninitializedCharError): kill_fail(test, self, uninitializedCharacterMessage) # Catch runtime error exceptions except (RuntimeAbort, RuntimeSegFault, RuntimeFPE, RuntimeBusError, RuntimeIllegalInstruction, MakefileError): pass test.terminate() # Associated test number within Gradescope @number("7") # Test visibility @visibility("visible") # Individual test case timeout (in seconds) @timeout.timeout(10, exception_message=wrap(programTimeoutErrorMessage, 65), use_signals=False) # Associated point value within Gradescope @weight(15) def test_07WordSearchLowercase(self): # Title used by Gradescope """Program prints number of occurances of words in all lowercase""" checkExecutables(self, os.popen(findLastExecutableCommand).read().split()) # Create a subprocess to run the student's code to obtain an output test = subprocess.Popen(["make -s run < input/wordSearchLowercase.txt"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = test.communicate() try: checkRuntimeErrors(test, self, stdout, stderr) # Try to decode stdout try: stdout = checkForUninitializedChars(stdout.strip().decode('utf-8')) test.kill() # Open reference output and decode reference = open('reference/wordSearchLowercase.txt', 'rb').read().strip().decode('utf-8') # Remove empty lines from both output and reference stdout = removeEmptyLines(stdout) reference = removeEmptyLines(reference) # Check the contents of stdout against reference self.customCompare(stdout, reference, msg='Program output does not match expected output') # Catch exception for decode error except (UnicodeDecodeError): kill_fail(test, self, decodeErrorMessage) # Catch exception for uninitialized characters except (UninitializedCharError): kill_fail(test, self, uninitializedCharacterMessage) # Catch runtime error exceptions except (RuntimeAbort, RuntimeSegFault, RuntimeFPE, RuntimeBusError, RuntimeIllegalInstruction, MakefileError): pass test.terminate() # Associated test number within Gradescope @number("8") # Test visibility @visibility("visible") # Individual test case timeout (in seconds) @timeout.timeout(10, exception_message=wrap(programTimeoutErrorMessage, 65), use_signals=False) # Associated point value within Gradescope @weight(15) def test_08WordSearchMixedCase(self): # Title used by Gradescope """Program prints number of occurances of words with mixed case""" checkExecutables(self, os.popen(findLastExecutableCommand).read().split()) # Create a subprocess to run the student's code to obtain an output test = subprocess.Popen(["make -s run < input/wordSearchMixedCase.txt"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = test.communicate() try: checkRuntimeErrors(test, self, stdout, stderr) # Try to decode stdout try: stdout = checkForUninitializedChars(stdout.strip().decode('utf-8')) test.kill() # Open reference output and decode reference = open('reference/wordSearchMixedCase.txt', 'rb').read().strip().decode('utf-8') # Remove empty lines from both output and reference stdout = removeEmptyLines(stdout) reference = removeEmptyLines(reference) # Check the contents of stdout against reference self.customCompare(stdout, reference, msg='Program output does not match expected output') # Catch exception for decode error except (UnicodeDecodeError): kill_fail(test, self, decodeErrorMessage) # Catch exception for uninitialized characters except (UninitializedCharError): kill_fail(test, self, uninitializedCharacterMessage) # Catch runtime error exceptions except (RuntimeAbort, RuntimeSegFault, RuntimeFPE, RuntimeBusError, RuntimeIllegalInstruction, MakefileError): pass test.terminate() # Associated test number within Gradescope @number("9") # Test visibility @visibility("visible") # Individual test case timeout (in seconds) @timeout.timeout(10, exception_message=wrap(programTimeoutErrorMessage, 65), use_signals=False) # Associated point value within Gradescope @weight(2.5) def test_09QuitImmediately(self): # Title used by Gradescope """Program quits immediately""" checkExecutables(self, os.popen(findLastExecutableCommand).read().split()) # Create a subprocess to run the student's code to obtain an output test = subprocess.Popen(["make -s run < input/quitImmediately.txt"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = test.communicate() try: checkRuntimeErrors(test, self, stdout, stderr) # Try to decode stdout try: stdout = checkForUninitializedChars(stdout.strip().decode('utf-8')) test.kill() # Open reference output and decode reference = open('reference/quitImmediately.txt', 'rb').read().strip().decode('utf-8') # Remove empty lines from both output and reference stdout = removeEmptyLines(stdout) reference = removeEmptyLines(reference) # Check the contents of stdout against reference self.customCompare(stdout, reference, msg='Program output does not match expected output') # Catch exception for decode error except (UnicodeDecodeError): kill_fail(test, self, decodeErrorMessage) # Catch exception for uninitialized characters except (UninitializedCharError): kill_fail(test, self, uninitializedCharacterMessage) # Catch runtime error exceptions except (RuntimeAbort, RuntimeSegFault, RuntimeFPE, RuntimeBusError, RuntimeIllegalInstruction, MakefileError): pass test.terminate() # Associated test number within Gradescope @number("10") # Test visibility @visibility("visible") # Individual test case timeout (in seconds) @timeout.timeout(10, exception_message=wrap(programTimeoutErrorMessage, 65), use_signals=False) # Associated point value within Gradescope @weight(5) def test_10Lab7DocumentExample(self): # Title used by Gradescope """Example run from Lab 7 document""" checkExecutables(self, os.popen(findLastExecutableCommand).read().split()) # Create a subprocess to run the student's code to obtain an output test = subprocess.Popen(["make -s run < input/lab7DocumentExample.txt"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = test.communicate() try: checkRuntimeErrors(test, self, stdout, stderr) # Try to decode stdout try: stdout = checkForUninitializedChars(stdout.strip().decode('utf-8')) test.kill() # Open reference output and decode reference = open('reference/lab7DocumentExample.txt', 'rb').read().strip().decode('utf-8') # Remove empty lines from both output and reference stdout = removeEmptyLines(stdout) reference = removeEmptyLines(reference) # Check the contents of stdout against reference self.customCompare(stdout, reference, msg='Program output does not match expected output') # Catch exception for decode error except (UnicodeDecodeError): kill_fail(test, self, decodeErrorMessage) # Catch exception for uninitialized characters except (UninitializedCharError): kill_fail(test, self, uninitializedCharacterMessage) # Catch runtime error exceptions except (RuntimeAbort, RuntimeSegFault, RuntimeFPE, RuntimeBusError, RuntimeIllegalInstruction, MakefileError): pass test.terminate() # Associated test number within Gradescope @number("11") # Test visibility @visibility("visible") # Individual test case timeout (in seconds) @timeout.timeout(10, exception_message=wrap(programTimeoutErrorMessage, 65), use_signals=False) # Associated point value within Gradescope @weight(12.5) def test_11PA3DocumentExample(self): # Title used by Gradescope """Example run from PA3 document""" checkExecutables(self, os.popen(findLastExecutableCommand).read().split()) # Create a subprocess to run the student's code to obtain an output test = subprocess.Popen(["make -s run < input/PA3DocumentExample.txt"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = test.communicate() try: checkRuntimeErrors(test, self, stdout, stderr) # Try to decode stdout try: stdout = checkForUninitializedChars(stdout.strip().decode('utf-8')) test.kill() # Open reference output and decode reference = open('reference/PA3DocumentExample.txt', 'rb').read().strip().decode('utf-8') # Remove empty lines from both output and reference stdout = removeEmptyLines(stdout) reference = removeEmptyLines(reference) # Check the contents of stdout against reference self.customCompare(stdout, reference, msg='Program output does not match expected output') # Catch exception for decode error except (UnicodeDecodeError): kill_fail(test, self, decodeErrorMessage) # Catch exception for uninitialized characters except (UninitializedCharError): kill_fail(test, self, uninitializedCharacterMessage) # Catch runtime error exceptions except (RuntimeAbort, RuntimeSegFault, RuntimeFPE, RuntimeBusError, RuntimeIllegalInstruction, MakefileError): pass test.terminate() # Associated test number within Gradescope @number("12") # Test visibility @visibility("visible") # Individual test case timeout (in seconds) @timeout.timeout(10, exception_message=wrap(programTimeoutErrorMessage, 65), use_signals=False) # Associated point value within Gradescope @weight(2.5) def test_12MakeCleanTarget(self): # Title used by Gradescope """make clean target works correctly""" checkExecutables(self, os.popen(findLastExecutableCommand).read().split()) file = os.popen(findLastExecutableCommand).read() # Create a subprocess to run the student's code to obtain an output test = subprocess.Popen(["make clean"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = test.communicate() test.kill() if Path(file).is_file() or test.returncode: self.assertTrue(False, wrap("\"" + file + "\" was not removed by \"make clean\".", 65)) else: self.assertTrue(True) test.terminate() # Associated test number within Gradescope @number("13") # Test visibility @visibility("after_due_date") # Associated point value within Gradescope @weight(0) def test_13CompilePrintMenuTest(self): # Title used by Gradescope """Compile unit test for printMenu""" checkSourceFiles(self, self.files) # Create a subprocess to run the student's Makefile to ensure it compiles test = subprocess.Popen(["gcc -Wall printMenuDriver.c printMenu.c -o printMenuTest.out"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = test.communicate() # Try to decode stderr try: stderr = stderr.strip().decode('utf-8') test.kill() self.assertTrue(stderr == "", msg=("See compiler output:\n" + stderr)) # Catch exception for decode error except (UnicodeDecodeError): kill_fail(test, self, compileDecodeErrorMessage) test.terminate() # Associated test number within Gradescope @number("14") # Test visibility @visibility("visible") # Individual test case timeout (in seconds) @timeout.timeout(10, exception_message=wrap(programTimeoutErrorMessage, 65), use_signals=False) # Associated point value within Gradescope @weight(7.5) # Custom error message override (so as not to show expected unit test output) @hide_errors(wrap("Test Failed: printMenu function did not return integers in the order they were inputted. Make sure your printMenu function returns the integer that a user would enter for a menu option.", 65)) def test_14PrintMenuUnitTest(self): # Title used by Gradescope """printMenu() function returns integers correctly""" checkExecutables(self, ['printMenuTest.out']) # Create a subprocess to run the student's code to obtain an output test = subprocess.Popen(["./printMenuTest.out < input/printMenuUnitTest.txt"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = test.communicate() try: checkRuntimeErrors(test, self, stdout, stderr) # Try to decode stdout try: stderr = checkForUninitializedChars(stderr.strip().decode('utf-8')) test.kill() # Open reference output and decode reference = open('reference/printMenuUnitTest.txt', 'rb').read().strip().decode('utf-8') # Remove empty lines from both output and reference stderr = removeEmptyLines(stderr) reference = removeEmptyLines(reference) # Check the contents of stdout against reference self.customCompare(stderr, reference, msg='Unit test failed') # Catch exception for decode error except (UnicodeDecodeError): kill_fail(test, self, decodeErrorMessage) # Catch exception for uninitialized characters except (UninitializedCharError): kill_fail(test, self, uninitializedCharacterMessage) # Catch runtime error exceptions except (RuntimeAbort, RuntimeSegFault, RuntimeFPE, RuntimeBusError, RuntimeIllegalInstruction, MakefileError): pass test.terminate()