Merge pull request #2248 from avancinirodrigo/master

Allowing latin characters

packages/base/lua/Directory.lua

@@ -21,6 +21,12 @@
 -- of this software and its documentation.
 --
 -------------------------------------------------------------------------------------------
+local function checkInvalidChars(fullpath)
+	local invalidChars = string.gsub(fullpath, "[^\34\42\60\62\63\124]", "")
+	if #invalidChars ~= 0 then
+		customError("Directory path '"..fullpath.."' contains invalid character '"..invalidChars.."'.")
+	end
+end
 
 Directory_ = {
 	type_ = "Directory",
@@ -247,11 +253,7 @@ function Directory(data)
 	data.fullpath = data.name
 	data.name = nil
 
-	local invalidCharIdx = data.fullpath:find("[*<>?|\"]")
-	if invalidCharIdx then
-		local invalidChar = data.fullpath:sub(invalidCharIdx, invalidCharIdx)
-		customError("Directory path '"..data.fullpath.."' cannot contain character '"..invalidChar.."'.")
-	end
+	checkInvalidChars(data.fullpath)
 
 	if not (data.fullpath:match("\\") or data.fullpath:match("/")) then
 		if data.fullpath == "." then data.fullpath = "" end
@@ -261,9 +263,8 @@ function Directory(data)
 
 	data.fullpath = _Gtme.makePathCompatibleToAllOS(data.fullpath)
 
-	local invalidChar = string.gsub(data.fullpath, "[^\33-\44\59-\64\91-\94\96\123-\125\127-\255]", "")
-	if #invalidChar ~= 0 then
-		customError("Directory path '"..data.fullpath.."' contains invalid character '"..invalidChar.."'.")
+	if sessionInfo().system == "windows" then
+		data.fullpath = replaceLatinCharacters(data.fullpath) --SKIP
 	end
 
 	if data.fullpath:sub(-1) == "/" then
@@ -280,7 +281,11 @@ function Directory(data)
 		local cmd = runCommand("mktemp -d \""..data.fullpath.."\"")[1]
 		table.insert(_Gtme.tmpdirectory__, data)
 
-		data.fullpath = cmd
+		if sessionInfo().system == "windows" then
+			data.fullpath = replaceLatinCharacters(cmd) --SKIP
+		else
+			data.fullpath = cmd --SKIP
+		end
 	end
 
 	if isFile(data.fullpath) then

packages/base/lua/File.lua

@@ -74,6 +74,13 @@ local function parseLine(line, sep, cline)
 	return res
 end
 
+local function checkInvalidChars(filename)
+	local invalidChars = string.gsub(filename, "[^\34\42\47\58\60\62\63\92\124]", "")
+	if #invalidChars ~= 0 then
+		customError("File name '"..filename.."' contains invalid character '"..invalidChars.."'.")
+	end
+end
+
 File_ = {
 	type_ = "File",
 	--- Return a table with the file attributes corresponding to filepath (or nil followed by an error
@@ -202,6 +209,7 @@ File_ = {
 		local s = sessionInfo().separator
 
 		local result = os.execute("rm -f \""..self.filename.."\" 2> "..directory..s.."a.txt")
+
 		directory:delete()
 
 		if result ~= true then
@@ -555,15 +563,10 @@ function File(data)
 		customError("Directory '"..dir.."' does not exist.")
 	end
 
-	local invalidCharIdx = data.filename:find("[&*<>?|\"]")
-	if invalidCharIdx then
-		customError("Filename '"..data.filename.."' cannot contain character '"..data.filename:sub(invalidCharIdx, invalidCharIdx).."'.")
-	end
+	checkInvalidChars(data:name())
 
-	local fileName = data:name()
-	local invalidChar = string.gsub(fileName, "[^\33-\44\58-\64\91-\94\96\123\125\127-\255]", "")
-	if #invalidChar ~= 0 then
-		customError("File name '"..fileName.."' contains invalid character '"..invalidChar.."'.")
+	if sessionInfo().system == "windows" then
+		data.filename = replaceLatinCharacters(data.filename) --SKIP
 	end
 
 	if isDirectory(data.filename) then

packages/base/lua/Utils.lua

@@ -1609,3 +1609,64 @@ function vardump(o, indent)
 	return recursiveVardump(o, indent, {})
 end
 
+local LatinCharacterMapper = {
+	[128] = "\xC0",	[129] = "\xC1",	[130] = "\xC2",
+	[131] = "\xC3",	[132] = "\xC4",	[133] = "\xC5",
+	[134] = "\xC6",	[135] = "\xC7",	[136] = "\xC8",
+	[137] = "\xC9",	[138] = "\xCA",	[139] = "\xCB",
+	[140] = "\xCC",	[141] = "\xCD",	[142] = "\xCE",
+	[143] = "\xCF",	[144] = "\xD0",	[145] = "\xD1",
+	[146] = "\xD2",	[147] = "\xD3",	[148] = "\xD4",
+	[149] = "\xD5",	[150] = "\xD6",	[151] = "\xD7",
+	[152] = "\xD8",	[153] = "\xD9",	[154] = "\xDA",
+	[155] = "\xDB",	[156] = "\xDC",	[157] = "\xDD",
+	[158] = "\xDE",	[159] = "\xDF",	[160] = "\xE0",
+	[161] = "\xE1",	[162] = "\xE2",	[163] = "\xE3",
+	[164] = "\xE4",	[165] = "\xE5",	[166] = "\xE6",
+	[167] = "\xE7",	[168] = "\xE8",	[169] = "\xE9",
+	[170] = "\xEA",	[171] = "\xEB",	[172] = "\xEC",
+	[173] = "\xED",	[174] = "\xEE",	[175] = "\xEF",
+	[176] = "\xF0",	[177] = "\xF1",	[178] = "\xF2",
+	[179] = "\xF3",	[180] = "\xF4",	[181] = "\xF5",
+	[182] = "\xF6",	[183] = "\xF7",	[184] = "\xF8",
+	[185] = "\xF9",	[186] = "\xFA",	[187] = "\xFB",
+	[188] = "\xFC",	[189] = "\xFD",	[190] = "\xFE",
+	[191] = "\xFF"
+}
+
+local function hasLatinCharacters(str)
+	local latinChars = string.gsub(str, "[^\192-\255]", "")
+	return #latinChars > 0
+end
+
+--- This function converts latin characters to hexadecimal code by the characters bytes.
+-- It is necessary to Windows OS works correctly with latin characters.
+-- Also, it is necessary to set Lua locale as the same as the system locale.
+-- See os.setlocale() for more details.
+-- The list of unicode characters can be found in
+-- https://en.wikipedia.org/wiki/List_of_Unicode_characters.
+-- @arg str A string.
+-- @usage --DONTRUN
+-- print(replaceLatinCharacters(águia))
+-- -- \xE1guia
+function replaceLatinCharacters(str)
+	if not hasLatinCharacters(str) then
+		return str
+	end
+
+	local rep = ""
+
+	for c in string.gmatch(str, ".") do
+		local b = string.byte(c, 1, -1)
+		if b ~= 195 then
+			if LatinCharacterMapper[b] then
+				rep = rep..LatinCharacterMapper[b]
+			else
+				rep = rep..c
+			end
+		end
+	end
+
+	return rep
+end
+

packages/base/tests/functional/alternative/Directory.lua

@@ -40,7 +40,7 @@ return{
 			Directory("abc\"")
 		end
 
-		unitTest:assertError(error_func, "Directory path 'abc\"' cannot contain character '\"'.")
+		unitTest:assertError(error_func, "Directory path 'abc\"' contains invalid character '\"'.")
 
 		error_func = function()
 			Directory{}
@@ -58,18 +58,6 @@ return{
 			Directory(packageInfo("base").path.."data/agents.csv")
 		end
 		unitTest:assertError(error_func, "'agents.csv' is a file, and not a directory.", 0, true)
-
-		local invalidCharError = function()
-			Directory("C:/Usuários")
-		end
-
-		unitTest:assertError(invalidCharError, "Directory path 'C:/Usuários' contains invalid character 'á'.")
-
-		invalidCharError = function()
-			Directory("C:/Usuários/Ação")
-		end
-
-		unitTest:assertError(invalidCharError, "Directory path 'C:/Usuários/Ação' contains invalid character 'áçã'.")
 	end,
 	attributes = function(unitTest)
 		local dir = Directory("/my/path/my_dir")

packages/base/tests/functional/alternative/File.lua

@@ -48,33 +48,25 @@ return{
 
 		unitTest:assertError(error_func, "Directory '/my/path/' does not exist.")
 
-		local path = packageInfo("base").data
-
-		local filename = path.."file*"
+		local filename = "file*"
 		error_func = function()
 			File(filename)
 		end
 
-		unitTest:assertError(error_func, "Filename '"..filename.."' cannot contain character '*'.")
+		unitTest:assertError(error_func, "File name '"..filename.."' contains invalid character '*'.")
 
-		filename = path.."file\""
+		filename = "file\""
 		error_func = function()
 			File(filename)
 		end
 
-		unitTest:assertError(error_func, "Filename '"..filename.."' cannot contain character '\"'.")
+		unitTest:assertError(error_func, "File name '"..filename.."' contains invalid character '\"'.")
 
 		error_func = function()
 			File(tostring(packageInfo("base").path))
 		end
 
 		unitTest:assertError(error_func, "'base' is a directory, and not a file.", 0, true)
-
-		local invalidChar = function()
-			File("ação.txt")
-		end
-
-		unitTest:assertError(invalidChar, "File name 'ação.txt' contains invalid character 'çã'.")
 	end,
 	attributes = function(unitTest)
 		local file = filePath("agents.csv", "base")

packages/base/tests/functional/basics/Directory.lua

@@ -106,6 +106,22 @@ return{
 		unitTest:assert(blankSpaceDir:create())
 		unitTest:assert(blankSpaceDir:exists())
 		unitTest:assert(blankSpaceDir:delete())
+
+		local validCharacterInDirName = function()
+			local d = Directory("dir++")
+			unitTest:assert(d:create())
+			unitTest:assert(d:delete())
+		end
+
+		unitTest:assert(validCharacterInDirName)
+
+		local latinCharacter = function()
+			local d = Directory("Ação")
+			unitTest:assert(d:create())
+			unitTest:assert(d:delete())
+		end
+
+		unitTest:assert(latinCharacter)
 	end,
 	delete = function(unitTest)
 		local dir = Directory("test_dir_delete")

packages/base/tests/functional/basics/File.lua

@@ -26,6 +26,13 @@ return{
 	File = function(unitTest)
 		local file = File("abc.txt")
 		unitTest:assertType(file, "File")
+
+		file = File("ação.txt")
+		if sessionInfo().system == "windows" then
+			unitTest:assert(tostring(file) ~= currentDir().."ação.txt") -- SKIP
+		end
+		local f2 = File("ação.txt")
+		unitTest:assertEquals(tostring(file), tostring(f2))
 	end,
 	__concat = function(unitTest)
 		local f = File("abcd1234")

packages/base/tests/functional/basics/Package.lua

@@ -45,7 +45,7 @@ return{
 		unitTest:assertType(cs, "CellularSpace")
 
 		-- The assert below checks the number of functions in package 'base'.
-		unitTest:assertEquals(getn(base), 180)
+		unitTest:assertEquals(getn(base), 181)
 	end,
 	import = function(unitTest)
 		forEachCell = nil

packages/base/tests/functional/basics/Utils.lua

@@ -1161,6 +1161,13 @@ return{
 		end)
 
 		unitTest:assertEquals(count, 84)
+	end,
+	replaceLatinCharacters = function(unitTest)
+		local str = "action"
+		unitTest:assertEquals(replaceLatinCharacters(str), "action")
+
+		str = "ação"
+		unitTest:assertEquals(replaceLatinCharacters(str), "a\xE7\xE3o")
 	end
 }