ZCPU Automated Testing console command & framework (#18)

* Basic test suite complete

* IOBus execution test + Fixed IOBus/Membus readcell

* No internal ram/rom external test + fixes

* Simple file read, color print, include interception

* Ifdef test, Access CPUName + reformatting

* Smarter ConCMD, Run tests by name(with wildcards)

* Reorganize cpu_load.lua

* Don't append a VM to your VM + Init VM before test

* Suggestions pass 1 + Make command SP Only

* Corrected error function, now prints all varargs

* Spelling fixes for test framework

* Reset warning count, don't print test list on runall

* Spelling error corrected, now runs in singleplayer

* Test compilation now runs on dedicated servers + instant compilation

* Revert CPULib.print back to print

* Switches uses of ' out for " for strings for consistency

* Space operators / minor formatting changes

* Create color constants & table.pack(...) to {...}

* Collapse if statements, missed spots from other commits

* More commented code removal

* Return a local table from tests instead of global.

* Tests revised

Tests can have a virtual filesystem(still falls back to actually using the folder if you don't use it)

Easier to use "Deploy" function, for compiling and flashing straight to a device

Revises all tests to use assert inside of a pcalled context

lua/autorun/cpu_load.lua

@@ -17,6 +17,10 @@ AddCSLuaFile("wire/zvm/zvm_features.lua")
 AddCSLuaFile("wire/zvm/zvm_opcodes.lua")
 AddCSLuaFile("wire/zvm/zvm_data.lua")
 
+if SERVER then
+	include("wire/zvm/zvm_tests.lua")
+end
+
 AddCSLuaFile("wire/cpulib.lua")
 include("wire/cpulib.lua")
 

lua/wire/cpulib.lua

@@ -4,7 +4,7 @@
 local INVALID_BREAKPOINT_IP = 2e7
 
 CPULib = CPULib or {}
-if CLIENT then
+if CLIENT or TESTING then
   -- Sourcecode available as compiled binary
   CPULib.Source = ""
   -- Compiled binary

lua/wire/zvm/tests/example.lua

@@ -0,0 +1,19 @@
+local Test = {}
+
+function Test.Run(CPU,TestSuite)
+	TestSuite:Deploy(CPU,"x: INC R0 JMP x",Test.CompileError)
+	CPU.Clk = 1
+	-- Run the VM for 4096 cycles
+	for i = 0, 4096 do
+		CPU:RunStep()
+	end
+
+	-- On false, will cause test to fail with message
+	assert(CPU.R0 == 4096,"R0 is not 4096! R0 is " .. tostring(CPU.R0))
+end
+
+function Test.CompileError(msg)
+	assert(false,"compile time error: " .. msg)
+end
+
+return Test

lua/wire/zvm/tests/execute_from_iobus.lua

@@ -0,0 +1,35 @@
+local Test = {}
+
+function Test.Run(CPU,TestSuite)
+	TestSuite.Compile("MOV R0,6 ADD R0,R0 MUL R0,2", "internal", nil, Test.CompileError)
+	-- end result of the above code should be R0 = 24
+	local buff = TestSuite.GetCompileBuffer()
+	local IOBus = TestSuite.CreateVirtualIOBus(#buff + 1) -- create an IOBus large enough to hold this code
+	-- reverse the compiled code, the CPU will read them in reverse if it's in the IOBus
+	-- because CS will be negative, and IP only increments
+
+	-- ipairs won't index 0 and the cpu compile buffer uses 0
+	for i = 0, #buff do
+		IOBus.InPorts[#buff-i] = buff[i]
+	end
+
+	-- JMPF jumps to 0 IP, CS = (code length+1)*-1 because first index of IOBus is "cell -1" of extern read/write
+	local generatedcode = "CMP R0,0 JNER -3 JMPF 0," .. (#buff + 1) * -1
+
+	TestSuite:Deploy(CPU, generatedcode, Test.CompileError)
+	TestSuite.Initialize(CPU, nil, IOBus) -- reinitialize the CPU with the IOBus
+	CPU.Clk = 1
+	for i = 0, 32 do
+		CPU:RunStep()
+	end
+
+	-- On false, will cause test to fail with message
+	assert(CPU.R0 == 24, "R0 != 24, R0 = " .. tostring(CPU.R0))
+end
+
+
+function Test.CompileError(msg)
+	assert(false, "compile time error " .. msg)
+end
+
+return Test

lua/wire/zvm/tests/file_example.lua

@@ -0,0 +1,30 @@
+local Test = {}
+
+-- If Test.Files is present, the compiler and suite will read files from it instead of the test directory
+Test.Files = {
+	["file_example.txt"] = [[
+x:
+INC R0
+JMP x
+	]]
+}
+
+function Test.Run(CPU,TestSuite)
+	-- Use the suite to load the file from our provided files table
+	local code = TestSuite:LoadFile("file_example.txt")
+	TestSuite:Deploy(CPU,code,Test.CompileError)
+	CPU.Clk = 1
+	-- Run the VM for 4096 cycles
+	for i = 0, 4096 do
+		CPU:RunStep()
+	end
+
+	-- On false, will cause test to fail with message
+	assert(CPU.R0 == 4096, "R0 is not 4096! R0 is " .. tostring(CPU.R0))
+end
+
+function Test.CompileError(msg)
+	assert(false, "compile time error: " .. msg)
+end
+
+return Test

lua/wire/zvm/tests/ifdefs.lua

@@ -0,0 +1,115 @@
+local Test = {}
+
+Test.Files = {
+	["ifdefs.txt"] = [[
+/*defs will be added programatically*/
+
+#ifdef x
+#define z
+#pragma cpuname Test X
+    ALLOC 1
+#ifdef y
+#ifdef x
+#pragma cpuname Test X and Y
+#endif
+    ALLOC 2
+#endif
+//not x
+#else 
+    ALLOC 4
+#endif
+
+#ifdef y
+#ifndef x
+#pragma cpuname Test Y
+#endif
+ALLOC 8
+#endif
+/*
+#ifdef y
+    ALLOC 8
+#endif
+*/
+//above comment intentional for making sure ifdef handler doesn't skip into the middle of a comment
+]]
+}
+
+--not x and y = 14, name "Test Y"
+-- x and y = 11, name "Test Y"
+-- x and not y = 1, name "Test Y"
+
+-- culling update
+-- not x and y = 12, name "Test Y"
+-- x and y = 11, name "Test X and Y"
+-- x and not y = 1, name "Test X"
+
+Test.ExpectedVariations1 = {"X", "Y", "X and Y", "Y", "Y", "Y"} -- CPU Name vars
+Test.ExpectedVariations2 = {1, 12, 11, 1, 14, 11}
+Test.ResultVariations1 = {}
+Test.ResultVariations2 = {}
+
+Test.Variations1 = {"true", "false"}
+Test.Variations2 = {"#define x\n", "#define y\n", "#define x\n#define y\n"}
+Test.Variation1Index = 1
+Test.Variation2Index = 1
+
+function Test.Run(CPU,TestSuite)
+	Test.TestSuite = TestSuite
+	Test.Src = TestSuite:LoadFile("ifdefs.txt")
+	Test.CompileNext()
+end
+
+function Test.CompileNext()
+	local cursrc
+	if Test.Variation1Index <= #Test.Variations1 then
+		cursrc = "#pragma set NewIfDefs " .. Test.Variations1[Test.Variation1Index] .. "\n"
+	else
+		return Test.CompareResults()
+	end
+	if Test.Variation2Index <= #Test.Variations2 then
+		cursrc = cursrc .. Test.Variations2[Test.Variation2Index] .. "\n" .. Test.Src
+		Test.TestSuite.Compile(cursrc,nil,Test.LogResults,Test.CompileError)
+	else
+		Test.Variation1Index = Test.Variation1Index + 1
+		Test.Variation2Index = 1
+		Test.CompileNext()
+	end
+end
+
+function Test.LogResults()
+	Test.ResultVariations1[Test.Variation2Index + #Test.Variations2*(Test.Variation1Index-1)] = Test.TestSuite.GetCPUName() or "ERROR"
+	Test.ResultVariations2[Test.Variation2Index + #Test.Variations2*(Test.Variation1Index-1)] = #Test.TestSuite.GetCompileBuffer() + 1 or "ERROR"
+	Test.Variation2Index = Test.Variation2Index + 1
+	Test.CompileNext()
+end
+
+function Test.CompareResults()
+	local fail, results1, results2 = false, {}, {}
+	for ind, i in ipairs(Test.ExpectedVariations1) do
+		if Test.ResultVariations1[ind] == "Test " .. i then
+				results1[ind] = true
+			else
+				fail = true
+				results1[ind] = false
+		end
+	end
+	for ind, i in ipairs(Test.ExpectedVariations2) do
+		if Test.ResultVariations2[ind] == i then
+			results2[ind] = true
+		else
+			fail = true
+			results2[ind] = false
+		end
+	end
+	if fail then
+		PrintTable({Test.ResultVariations1, results1, Test.ResultVariations2, results2})
+		assert(false, "Unexpected Test Results!")
+	end
+	return
+end
+
+function Test.CompileError(msg)
+	assert(false, "compile time error: " .. msg)
+end
+
+return Test

lua/wire/zvm/tests/includes.lua

@@ -0,0 +1,29 @@
+local Test = {}
+
+Test.Files = {
+	["includes_1.txt"] = [[
+#include <includes_2.txt>
+ADD R0,1
+]],
+
+	["includes_2.txt"] = [[
+MOV R0,1
+]]
+}
+
+function Test.Run(CPU,TestSuite)
+	local src = TestSuite:LoadFile("includes_1.txt")
+	TestSuite:Deploy(CPU, src, Test.CompileError)
+	CPU.Clk = 1
+	for i = 0, 16 do
+		CPU:RunStep()
+	end
+
+	assert(CPU.R0 == 2, "R0 is not 2! R0 is " .. tostring(CPU.R0))
+end
+
+function Test.CompileError(msg)
+	assert(false, "compile time error: " .. msg)
+end
+
+return Test

lua/wire/zvm/tests/intentional_compile_error.lua

@@ -0,0 +1,11 @@
+local Test = {}
+
+function Test.Run(CPU,TestSuite)
+	return TestSuite.Compile("MOV R0,", nil, Test.CompileSucceed)
+end
+
+function Test.CompileSucceed()
+	assert(false, "Compiler should have errored!")
+end
+
+return Test

lua/wire/zvm/tests/intentional_failed_test.lua

@@ -0,0 +1,19 @@
+local Test = {}
+
+function Test.Run(CPU,TestSuite)
+	TestSuite:Deploy(CPU,"x: INC R0 JMP x",Test.CompileError)
+	CPU.Clk = 1
+	-- Run the VM for 4096 cycles
+	for i = 0, 4096 do
+		CPU:RunStep()
+	end
+
+	-- On false, will cause test to fail with message
+	assert(CPU.R0 == 4095, "[INTENTIONAL ERROR] R0 is not 4095! R0 is " .. tostring(CPU.R0))
+end
+
+function Test.CompileError(msg)
+	assert(false, "compile time error: " .. msg)
+end
+
+return Test

lua/wire/zvm/tests/no_internal_mem.lua

@@ -0,0 +1,24 @@
+local Test = {}
+
+
+function Test.Run(CPU,TestSuite)
+	TestSuite.Compile("MOV R0,1", "internal", nil, Test.CompileError)
+	local buff = TestSuite.GetCompileBuffer()
+	local bus = TestSuite.CreateVirtualMemBus(#buff) -- get external ram device large enough to hold program
+	TestSuite.FlashData(bus, buff) -- upload compiled to membus
+	CPU.RAMSize = 0
+	CPU.ROMSize = 0
+	TestSuite.Initialize(CPU, bus, nil) -- reinitialize the CPU with the membus
+	CPU.Clk = 1
+	for i = 0, 16 do
+		CPU:RunStep()
+	end
+
+	assert(CPU.R0 == 1, "CPU with no ram/rom failed to execute code from bus! R0 = " .. CPU.R0)
+end
+
+function Test.CompileError(msg)
+	assert(false, "compile time error: " .. msg)
+end
+
+return Test

lua/wire/zvm/tests/virtualiobus.lua

@@ -0,0 +1,24 @@
+local Test = {}
+
+function Test.Run(CPU,TestSuite)
+	TestSuite:Deploy(CPU, "MOV PORT0,1 MOV R0,PORT0",  Test.CompileError)
+	local IOBus = TestSuite.CreateVirtualIOBus(4) -- get external IO device of size 4
+	TestSuite.Initialize(CPU, nil, IOBus) -- reinitialize the CPU with the IOBus
+
+	IOBus.InPorts[0] = 24
+	CPU.Clk = 1
+	for i = 0, 16 do
+		CPU:RunStep()
+	end
+
+	-- False = no error, True = error
+	assert(IOBus:ReadCell(0) == 24, "IOBus InPort 0 != 24! Possibly CPU's fault? InPort 0 = " .. IOBus:ReadCell(0))
+	assert(IOBus.OutPorts[0] == 1, "CPU failed to write to output port! Port0 = " .. tostring(IOBus.OutPorts[0]))
+	assert(CPU.R0 == 24, "CPU failed to read input port! R0 = " .. CPU.R0)
+end
+
+function Test.CompileError(msg)
+	assert(false, "compile time error: " .. msg)
+end
+
+return Test

lua/wire/zvm/tests/virtualmembus.lua

@@ -0,0 +1,20 @@
+local Test = {}
+
+function Test.Run(CPU,TestSuite)
+	TestSuite:Deploy(CPU,"CPUGET R0,43 MOV [R0],1 MOV R1,[R0]",Test.CompileError)
+	local bus = TestSuite.CreateVirtualMemBus(4) -- get external ram device of size 4
+	TestSuite.Initialize(CPU, bus, nil) -- reinitialize the CPU with the membus
+	CPU.Clk = 1
+	for i = 0, 16 do
+		CPU:RunStep()
+	end
+
+	assert(bus:ReadCell(0) == 1, "CPU failed to write to bus! " .. tostring(bus:ReadCell(0)))
+	assert(CPU.R1 == 1, "CPU failed to read the bus! R1 was " .. CPU.R1)
+end
+
+function Test.CompileError(msg)
+	assert(false, "compile time error: " .. msg)
+end
+
+return Test