Merge branch 'fixup_ansi_win_api_and_fs' into 1.12

.travis.yml

@@ -2,11 +2,19 @@ language: cpp
 compiler:
     - clang
     - gcc
-install:
+before_install:
     - sudo add-apt-repository -y "deb http://archive.ubuntu.com/ubuntu/ saucy main universe"
     - sudo apt-get update -qq
-    - sudo apt-get install -qq libboost-filesystem-dev libboost-iostreams-dev libboost-program-options-dev libboost-regex-dev libboost-system-dev libboost-test-dev libcairo2-dev libfribidi-dev libpango1.0-dev libsdl-image1.2-dev libsdl-mixer1.2-dev libsdl-net1.2-dev libsdl-ttf2.0-dev
-script: scons cxxtool=$CXX strict=True wesnoth wesnothd campaignd test
+    - sudo apt-get install -qq libsdl-mixer1.2-dev 
+    - sudo add-apt-repository -y --remove "deb http://archive.ubuntu.com/ubuntu/ saucy main universe"
+    - if [ "$CXX" = "g++" ]; then sudo add-apt-repository -y ppa:ubuntu-toolchain-r/test ; fi
+    - sudo apt-get update -qq
+    - if [ "$CXX" = "g++" ]; then sudo apt-get -qq install g++-4.8; fi
+    - if [ "$CXX" = "g++" ]; then sudo add-apt-repository -y --remove ppa:ubuntu-toolchain-r/test ; fi
+    - if [ "$CXX" = "g++" ]; then export CXX="g++-4.8"; fi
+install:
+    - sudo apt-get install -qq libboost-filesystem-dev libboost-iostreams-dev libboost-program-options-dev libboost-regex-dev libboost-system-dev libboost-test-dev libcairo2-dev libfribidi-dev libpango1.0-dev libsdl-image1.2-dev libsdl-net1.2-dev libsdl-ttf2.0-dev
+script: scons cxxtool=$CXX strict=True extra_flags_release="-O0" wesnoth wesnothd campaignd test
 after_script:
     - "export DISPLAY=:99.0"
     - "sh -e /etc/init.d/xvfb start"

SConstruct

@@ -331,7 +331,8 @@ if env["prereqs"]:
         conf.CheckSDL(require_version = '1.2.7') and \
         conf.CheckSDL('SDL_net') & \
         conf.CheckBoost("system") & \
-		((not env["boostfilesystem"]) or (conf.CheckBoost("filesystem", require_version = "1.44.0"))) \
+		((not env["boostfilesystem"]) or (conf.CheckBoost("filesystem", require_version = "1.44.0"))) & \
+        ((not env["PLATFORM"] == "win32") or (not env["boostfilesystem"]) or (conf.CheckBoost("locale"))) \
             or Warning("Base prerequisites are not met.")
 
     env = conf.Finish()

src/SConscript

@@ -33,6 +33,7 @@ libwesnoth_core_sources = Split("""
     serialization/schema_validator.cpp
     serialization/string_utils.cpp
     serialization/tokenizer.cpp
+    serialization/unicode.cpp
     serialization/validator.cpp
     tools/schema/tag.cpp
     """)

src/clipboard.cpp

@@ -17,6 +17,7 @@
 #include "global.hpp"
 
 #include "clipboard.hpp"
+#include "serialization/unicode.hpp"
 #include <algorithm>
 
 #include <SDL_version.h>
@@ -432,43 +433,47 @@ void copy_to_clipboard(const std::string& text, const bool)
 		++last;
 	}
 
-	const HGLOBAL hglb = GlobalAlloc(GMEM_MOVEABLE, (str.size() + 1) * sizeof(TCHAR));
+	utf16::string ustring = unicode_cast<utf16::string>(str);
+	std::wstring wstr(ustring.begin(), ustring.end());
+
+	const HGLOBAL hglb = GlobalAlloc(GMEM_MOVEABLE, (wstr.size() + 1) * sizeof(wchar_t));
 	if(hglb == NULL) {
 		CloseClipboard();
 		return;
 	}
-	char* const buffer = reinterpret_cast<char* const>(GlobalLock(hglb));
-	strcpy(buffer, str.c_str());
+	wchar_t* const buffer = reinterpret_cast<wchar_t* const>(GlobalLock(hglb));
+	wcscpy(buffer, wstr.c_str());
 	GlobalUnlock(hglb);
-	SetClipboardData(CF_TEXT, hglb);
+	SetClipboardData(CF_UNICODETEXT, hglb);
 	CloseClipboard();
 }
 
 std::string copy_from_clipboard(const bool)
 {
-	if(!IsClipboardFormatAvailable(CF_TEXT))
+	if(!IsClipboardFormatAvailable(CF_UNICODETEXT))
 		return "";
 	if(!OpenClipboard(NULL))
 		return "";
 
-	HGLOBAL hglb = GetClipboardData(CF_TEXT);
+	HGLOBAL hglb = GetClipboardData(CF_UNICODETEXT);
 	if(hglb == NULL) {
 		CloseClipboard();
 		return "";
 	}
-	char const * buffer = reinterpret_cast<char*>(GlobalLock(hglb));
+	wchar_t const * buffer = reinterpret_cast<wchar_t*>(GlobalLock(hglb));
 	if(buffer == NULL) {
 		CloseClipboard();
 		return "";
 	}
 
+	std::wstring str(buffer);
 	// Convert newlines
-	std::string str(buffer);
-	str.erase(std::remove(str.begin(),str.end(),'\r'),str.end());
-
+	str.erase(std::remove(str.begin(),str.end(), L'\r'), str.end());
+	utf16::string ustring(str.begin(), str.end());
 	GlobalUnlock(hglb);
 	CloseClipboard();
-	return str;
+
+	return unicode_cast<std::string>(ustring);
 }
 
 #endif

src/commandline_options.cpp

@@ -53,7 +53,7 @@ bad_commandline_tuple::bad_commandline_tuple(const std::string& str,
 {
 }
 
-commandline_options::commandline_options ( int argc, char** argv ) :
+commandline_options::commandline_options (const std::vector<std::string>& args) :
 	bpp(),
 	bunzip2(),
 	bzip2(),
@@ -129,8 +129,8 @@ commandline_options::commandline_options ( int argc, char** argv ) :
 	version(false),
 	windowed(false),
 	with_replay(false),
-	argc_(argc),
-	argv_(argv),
+	args_(args.begin() + 1 , args.end()),
+	args0_(*args.begin()),
 	all_(),
 	visible_(),
 	hidden_()
@@ -253,7 +253,7 @@ commandline_options::commandline_options ( int argc, char** argv ) :
 
 	po::variables_map vm;
 	const int parsing_style = po::command_line_style::default_style ^ po::command_line_style::allow_guessing;
-	po::store(po::command_line_parser(argc_,argv_).options(all_).positional(positional).style(parsing_style).run(),vm);
+	po::store(po::command_line_parser(args_).options(all_).positional(positional).style(parsing_style).run(),vm);
 
 	if (vm.count("ai-config"))
 		multiplayer_ai_config = parse_to_uint_string_tuples_(vm["ai-config"].as<std::vector<std::string> >());
@@ -505,7 +505,7 @@ std::vector<boost::tuple<unsigned int,std::string,std::string> > commandline_opt
 
 std::ostream& operator<<(std::ostream &os, const commandline_options& cmdline_opts)
 {
-	os << "Usage: " << cmdline_opts.argv_[0] << " [<options>] [<data-directory>]\n";
+	os << "Usage: " << cmdline_opts.args0_ << " [<options>] [<data-directory>]\n";
 	os << cmdline_opts.visible_;
 	return os;
 }

src/commandline_options.hpp

@@ -41,7 +41,7 @@ class commandline_options
 friend std::ostream& operator<<(std::ostream &os, const commandline_options& cmdline_opts);
 
 public:
-	commandline_options(int argc, char **argv);
+	commandline_options(const std::vector<std::string>& args);
 
 	/// BitsPerPixel specified by --bpp option.
 	boost::optional<int> bpp;
@@ -199,8 +199,8 @@ private:
 	std::vector<boost::tuple<unsigned int,std::string> > parse_to_uint_string_tuples_(const std::vector<std::string> &strings, char separator = ':');
 	/// A helper function splitting vector of strings of format unsigned int:string:string to vector of tuples (unsigned int,string,string)
 	std::vector<boost::tuple<unsigned int,std::string,std::string> > parse_to_uint_string_string_tuples_(const std::vector<std::string> &strings, char separator = ':');
-	int argc_;
-	char **argv_;
+	std::vector<std::string> args_;
+	std::string args0_;
 	boost::program_options::options_description all_;
 	boost::program_options::options_description visible_;
 	boost::program_options::options_description hidden_;

src/filesystem.hpp

@@ -30,7 +30,6 @@
 
 class config;
 
-//TODO: will this compile for msvc <= 2010 which has a bug when it comes to forward declaring return values?
 struct SDL_RWops;
 
 namespace filesystem {

src/filesystem_boost.cpp

@@ -21,14 +21,18 @@
 #include "global.hpp"
 
 #include "filesystem.hpp"
+#include "serialization/unicode.hpp"
 
 #include <boost/filesystem.hpp>
 #include <boost/filesystem/fstream.hpp>
 #include <boost/foreach.hpp>
 #include <boost/system/windows_error.hpp>
+#include <boost/iostreams/device/file_descriptor.hpp>
+#include <boost/iostreams/stream.hpp>
 #include <set>
 
 #ifdef _WIN32
+#include <boost/locale.hpp>
 #include <windows.h>
 #endif /* !_WIN32 */
 
@@ -55,6 +59,134 @@ namespace {
 	const std::string initialcfg_filename = "_initial.cfg";
 }
 
+namespace {
+	//only used by windows but put outside the ifdef to let it check by ci build.
+	class customcodecvt : public std::codecvt<wchar_t /*intern*/, char /*extern*/, std::mbstate_t>
+	{
+	private:
+		//private static helper things
+		template<typename char_t_to>
+		struct customcodecvt_do_conversion_writer
+		{
+			char_t_to*& to_next;
+			char_t_to* to_end;
+
+			bool can_push(size_t count)
+			{
+				return static_cast<size_t>(to_end - to_next) > count;
+			}
+
+			void push(char_t_to val)
+			{
+				assert(to_next != to_end);
+				*to_next++  = val;
+			}
+		};
+
+		template<typename char_t_from , typename char_t_to>
+		static void customcodecvt_do_conversion( std::mbstate_t& /*state*/,
+			const char_t_from* from,
+			const char_t_from* from_end,
+			const char_t_from*& from_next,
+			char_t_to* to,
+			char_t_to* to_end,
+			char_t_to*& to_next )
+		{
+			typedef typename ucs4_convert_impl::convert_impl<char_t_from>::type impl_type_from;
+			typedef typename ucs4_convert_impl::convert_impl<char_t_to>::type impl_type_to;
+
+			from_next = from;
+			to_next = to;
+			customcodecvt_do_conversion_writer<char_t_to> writer = { to_next, to_end };
+			while(from_next != from_end)
+			{
+				impl_type_to::write(writer, impl_type_from::read(from_next, from_end));
+			}
+		}
+
+	public:
+
+		//Not used by boost filesystem
+		int do_encoding() const throw() { return 0; }
+		//Not used by boost filesystem
+		bool do_always_noconv() const throw() { return false; }
+		int do_length( std::mbstate_t& /*state*/,
+			const char* /*from*/,
+			const char* /*from_end*/,
+			std::size_t /*max*/ ) const
+		{
+			//Not used by boost filesystem
+			throw "Not supported";
+		}
+
+		std::codecvt_base::result unshift( std::mbstate_t& /*state*/,
+			char* /*to*/,
+			char* /*to_end*/,
+			char*& /*to_next*/) const
+		{
+			//Not used by boost filesystem
+			throw "Not supported";
+		}
+
+		//there are still some methods which could be implemented but arent because boost filesystem won't use them.
+		std::codecvt_base::result do_in( std::mbstate_t& state,
+			const char* from,
+			const char* from_end,
+			const char*& from_next,
+			wchar_t* to,
+			wchar_t* to_end,
+			wchar_t*& to_next ) const
+		{
+			try
+			{
+				customcodecvt_do_conversion<char, wchar_t>(state, from, from_end, from_next, to, to_end, to_next);
+			}
+			catch(...)
+			{
+				ERR_FS << "Invalid UTF-8 string'" << std::string(from, from_end) << "' " << std::endl;
+				return std::codecvt_base::error;
+			}
+			return std::codecvt_base::ok;
+		}
+
+		std::codecvt_base::result do_out( std::mbstate_t& state,
+			const wchar_t* from,
+			const wchar_t* from_end,
+			const wchar_t*& from_next,
+			char* to,
+			char* to_end,
+			char*& to_next ) const
+		{
+			try
+			{
+				customcodecvt_do_conversion<wchar_t, char>(state, from, from_end, from_next, to, to_end, to_next);
+			}
+			catch(...)
+			{
+				ERR_FS << "Invalid UTF-16 string" << std::endl;
+				return std::codecvt_base::error;	
+			}
+			return std::codecvt_base::ok;
+		}
+	};
+	
+#ifdef _WIN32
+	class static_runner {
+	public:
+		static_runner() {
+			// Boost uses the current locale to generate a UTF-8 one
+			std::locale utf8_loc = boost::locale::generator().generate("");
+			// use a custom locale becasue we want to use out log.hpp functions in case of an invalid string.
+			utf8_loc = std::locale(utf8_loc, new customcodecvt());
+			boost::filesystem::path::imbue(utf8_loc);
+		}
+	};
+
+	static static_runner static_bfs_path_imbuer;
+#endif
+}
+
+
 namespace filesystem {
 
 static void push_if_exists(std::vector<std::string> *vec, const path &file, bool full) {
@@ -579,6 +711,29 @@ std::string read_file(const std::string &fname)
 	ss << is->rdbuf();
 	return ss.str();
 }
+
+#if BOOST_VERSION < 1048000
+//boost iostream < 1.48 expects boost filesystem v2 paths. This is an adapter
+struct iostream_path
+{
+	template<typename stringtype>
+	iostream_path(const stringtype& s)
+		: path_(s)
+	{
+
+	}
+	typedef bfs::path::string_type external_string_type;
+	external_string_type external_file_string() const
+	{
+		return path_.native();
+	}
+	bfs::path path_;
+};
+
+#else
+typedef bfs::path iostream_path;
+#endif
+
 std::istream *istream_file(const std::string &fname)
 {
 	LOG_FS << "Streaming " << fname << " for reading.\n";
@@ -589,20 +744,36 @@ std::istream *istream_file(const std::string &fname)
 		return s;
 	}
 
-	bfs::ifstream *s = new bfs::ifstream(path(fname),std::ios_base::binary);
-	if (s->is_open()) {
+	//mingw doesn't  support std::basic_ifstream::basic_ifstream(const wchar_t* fname)
+	//that why boost::filesystem::fstream.hpp doesnt work with mingw.
+	try
+	{
+		boost::iostreams::file_descriptor_source fd(iostream_path(fname), std::ios_base::binary);
+		//TODO: has this still use ?
+		if (!fd.is_open()) {
+			ERR_FS << "Could not open '" << fname << "' for reading.\n";
+		}
+		return new boost::iostreams::stream<boost::iostreams::file_descriptor_source>(fd, 4096, 0);
+	}
+	catch(const std::exception ex)
+	{
+		ERR_FS << "Could not open '" << fname << "' for reading.\n";
+
+		bfs::ifstream *s = new bfs::ifstream();
+		s->clear(std::ios_base::failbit);
 		return s;
 	}
-
-	ERR_FS << "Could not open '" << fname << "' for reading.\n";
-
-	return s;
-
 }
+
 std::ostream *ostream_file(std::string const &fname)
 {
 	LOG_FS << "streaming " << fname << " for writing.\n";
+#if 1
+	boost::iostreams::file_descriptor_sink fd(iostream_path(fname), std::ios_base::binary);
+	return new boost::iostreams::stream<boost::iostreams::file_descriptor_sink>(fd, 4096, 0);
+#else
 	return new bfs::ofstream(path(fname), std::ios_base::binary);
+#endif
 }
 // Throws io_exception if an error occurs
 void write_file(const std::string& fname, const std::string& data)

src/filesystem_common.cpp

@@ -245,6 +245,7 @@ static int SDLCALL ifs_seek(struct SDL_RWops *context, int offset, int whence) {
 		break;
 	default:
 		assert(false);
+		throw "assertion ignored";
 	}
 	std::istream *ifs = static_cast<std::istream*>(context->hidden.unknown.data1);
 	const std::ios_base::iostate saved_state = ifs->rdstate();

src/font.cpp

@@ -29,6 +29,7 @@
 #include "serialization/parser.hpp"
 #include "serialization/preprocessor.hpp"
 #include "serialization/string_utils.hpp"
+#include "serialization/unicode.hpp"
 
 #include <boost/foreach.hpp>
 
@@ -363,9 +364,14 @@ void manager::init() const
 		std::vector<std::string> files;
 		if(filesystem::is_directory(path))
 			filesystem::get_files_in_dir(path, &files, NULL, filesystem::ENTIRE_FILE_PATH);
-		BOOST_FOREACH(const std::string& file, files)
+		BOOST_FOREACH(const std::string& file, files) {
 			if(file.substr(file.length() - 4) == ".ttf" || file.substr(file.length() - 4) == ".ttc")
-				AddFontResourceA(file.c_str());
+			{
+				utf16::string ufile = unicode_cast<utf16::string>(file);
+				std::wstring wfile(ufile.begin(), ufile.end());
+				AddFontResourceW(wfile.c_str());
+			}
+		}
 	}
 #endif
 }
@@ -381,9 +387,14 @@ void manager::deinit() const
 		std::vector<std::string> files;
 		if(filesystem::is_directory(path))
 			filesystem::get_files_in_dir(path, &files, NULL, filesystem::ENTIRE_FILE_PATH);
-		BOOST_FOREACH(const std::string& file, files)
+		BOOST_FOREACH(const std::string& file, files) {
 			if(file.substr(file.length() - 4) == ".ttf" || file.substr(file.length() - 4) == ".ttc")
-				RemoveFontResourceA(file.c_str());
+			{
+				utf16::string ufile = unicode_cast<utf16::string>(file);
+				std::wstring wfile(ufile.begin(), ufile.end());
+				RemoveFontResourceW(wfile.c_str());
+			}
+		}
 	}
 #endif
 }

src/game.cpp

@@ -21,6 +21,11 @@
 #include "game_config_manager.hpp"
 #include "game_controller.hpp"
 #include "gui/dialogs/title_screen.hpp"
+
+#ifdef _WIN32
+#include <windows.h>
+#endif
+
 #ifdef DEBUG_WINDOW_LAYOUT_GRAPHS
 #include "gui/widgets/debug.hpp"
 #endif
@@ -32,6 +37,8 @@
 #include "replay.hpp"
 #include "serialization/binary_or_text.hpp"
 #include "serialization/parser.hpp"
+#include "serialization/unicode_cast.hpp"
+
 #include "serialization/validator.hpp"
 #include "statistics.hpp"
 #include "version.hpp"
@@ -436,19 +443,19 @@ static void warn_early_init_failure()
  * Setups the game environment and enters
  * the titlescreen or game loops.
  */
-static int do_gameloop(int argc, char** argv)
+static int do_gameloop(const std::vector<std::string>& args)
 {
 	srand(time(NULL));
 
-	commandline_options cmdline_opts = commandline_options(argc,argv);
-	game_config::wesnoth_program_dir = filesystem::directory_name(argv[0]);
+	commandline_options cmdline_opts = commandline_options(args);
+	game_config::wesnoth_program_dir = filesystem::directory_name(args[0]);
 	int finished = process_command_args(cmdline_opts);
 	if(finished != -1) {
 		return finished;
 	}
 
 	boost::scoped_ptr<game_controller> game(
-		new game_controller(cmdline_opts,argv[0]));
+		new game_controller(cmdline_opts,args[0].c_str()));
 	const int start_ticks = SDL_GetTicks();
 
 	init_locale();
@@ -669,6 +676,57 @@ static int do_gameloop(int argc, char** argv)
 		}
 	}
 }
+#ifdef _WIN32
+static bool parse_commandline_argument(const char*& next, const char* end, std::string& res)
+{
+	//strip leading shitespace
+	while(next != end && *next == ' ')
+		++next;
+	if(next == end)
+		return false;
+
+	bool is_excaped = false;
+
+	for(;next != end; ++next)
+	{
+		if(*next == ' ' && !is_excaped) {
+			break;
+		}
+		else if(*next == '"' && !is_excaped) {
+			is_excaped = true;
+			continue;
+		}
+		else if(*next == '"' && is_excaped && next + 1 != end && *(next + 1) == '"') {
+			res.push_back('"');
+			++next;
+			continue;		
+		}
+		else if(*next == '"' && is_excaped ) {
+			is_excaped = false;
+			continue;	
+		}
+		else {
+			res.push_back(*next);
+		}
+	}
+	return true;
+}
+
+static std::vector<std::string> parse_commandline_arguments(std::string input)
+{
+	const char* start = &input[0];
+	const char* end = start + input.size();
+	std::string buffer;
+	std::vector<std::string> res;
+	
+	while(parse_commandline_argument(start, end, buffer))
+	{
+		res.push_back(std::string());
+		res.back().swap(buffer);
+	}
+	return res;
+}
+#endif
 
 
 #ifdef __native_client__
@@ -700,7 +758,19 @@ int main(int argc, char** argv)
 		execv(argv[0], argv);
 	}
 #endif
-
+#ifdef _WIN32
+	(void)argc;
+	(void)argv;
+	//windows argv is ansi encoded by default
+	std::vector<std::string> args = parse_commandline_arguments(unicode_cast<std::string>(std::wstring(GetCommandLineW())));
+#else
+	std::vector<std::string> args;
+	for(int i = 0; i < argc; ++i)
+	{
+		args.push_back(std::string(argv[i]));
+	}
+#endif
+	assert(!args.empty());
 	if(SDL_Init(SDL_INIT_TIMER) < 0) {
 		fprintf(stderr, "Couldn't initialize SDL: %s\n", SDL_GetError());
 		return(1);
@@ -735,7 +805,7 @@ int main(int argc, char** argv)
 			}
 		}
 
-		const int res = do_gameloop(argc,argv);
+		const int res = do_gameloop(args);
 		safe_exit(res);
 	} catch(boost::program_options::error& e) {
 		std::cerr << "Error in command line: " << e.what() << '\n';

src/hotkey/hotkey_item.cpp

@@ -20,6 +20,7 @@
 
 #include "gettext.hpp"
 #include "serialization/string_utils.hpp"
+#include "serialization/unicode_cast.hpp"
 #include "sdl_utils.hpp"
 
 #include "boost/foreach.hpp"
@@ -459,7 +460,7 @@ void hotkey_item::save(config& item) const
 	if (get_hat()       >= 0) item["hat"]      = get_hat();
 	if (get_value()     >= 0) item["value"]    = get_value();
 	if (get_keycode()   >= 0) item["key"]      = SDL_GetKeyName(SDLKey(get_keycode()));
-	if (get_character() >= 0) item["key"]      = utils::ucs4char_to_string(get_character());
+	if (get_character() >= 0) item["key"]      = unicode_cast<utf8::string>(static_cast<ucs4::char_t>(get_character()));
 	if (get_mouse()     >= 0) item["mouse"]    = get_mouse();
 	if (get_button()    >= 0) item["button"]   = get_button();
 

src/scripting/lua.cpp

@@ -1018,6 +1018,47 @@ static int intf_have_file(lua_State *L)
 	return 1;
 }
 
+class lua_filestream
+{
+public:
+	lua_filestream(const std::string& fname)
+		: pistream_(filesystem::istream_file(fname))
+	{
+		
+	}
+
+	static const char * lua_read_data(lua_State * /*L*/, void *data, size_t *size)
+	{
+		lua_filestream* lfs = static_cast<lua_filestream*>(data);
+		
+		//int startpos = lfs->pistream_->tellg();
+		lfs->pistream_->read(lfs->buff_, LUAL_BUFFERSIZE);
+		//int newpos = lfs->pistream_->tellg();
+		*size = lfs->pistream_->gcount();
+#if 0
+		ERR_LUA << "read bytes from " << startpos << " to " << newpos << " in total " *size << " from steam\n";
+		ERR_LUA << "streamstate beeing " 
+			<< " goodbit:" << lfs->pistream_->good()
+			<< " endoffile:" << lfs->pistream_->eof() 
+			<< " badbit:" <<  lfs->pistream_->bad()
+			<< " failbit:" << lfs->pistream_->fail() << "\n";
+#endif
+		return lfs->buff_;
+	}
+
+	static int lua_loadfile(lua_State *L, const std::string& fname)
+	{
+		lua_filestream lfs(fname);
+		//lua uses '@' to know that this is a file (as opposed to a something as opposed to something loaded via loadstring )
+		std::string chunkname = '@' + fname;
+		LOG_LUA << "starting to read from " << fname << "\n";
+		return  lua_load(L, &lua_filestream::lua_read_data, &lfs, chunkname.c_str(), NULL);
+	}
+private:
+	char buff_[LUAL_BUFFERSIZE];
+	boost::scoped_ptr<std::istream> pistream_;
+};
+
 /**
  * Loads and executes a Lua file.
  * - Arg 1: string containing the file name.
@@ -1031,8 +1072,22 @@ static int intf_dofile(lua_State *L)
 		return luaL_argerror(L, 1, "file not found");
 
 	lua_settop(L, 0);
+	
+#if 1 
+	try
+	{
+		if(lua_filestream::lua_loadfile(L, p))
+			return lua_error(L);
+	}
+	catch(const std::exception & ex)
+	{
+		luaL_argerror(L, 1, ex.what());
+	}
+#else
+	//oldcode to be deleted if newcode works
 	if (luaL_loadfile(L, p.c_str()))
 		return lua_error(L);
+#endif
 
 	lua_call(L, 0, LUA_MULTRET);
 	return lua_gettop(L);
@@ -1064,6 +1119,22 @@ static int intf_require(lua_State *L)
 		return luaL_argerror(L, 1, "file not found");
 
 	// Compile the file.
+	
+#if 1 
+	try
+	{
+		if(lua_filestream::lua_loadfile(L, p))
+			throw game::error(lua_tostring(L, -1));
+	}
+	catch(const std::exception & ex)
+	{
+		chat_message("Lua error", ex.what());
+		ERR_LUA << ex.what() << '\n';
+		return 0;
+	}
+#else
+	//oldcode to be deleted if newcode works
+
 	int res = luaL_loadfile(L, p.c_str());
 	if (res)
 	{
@@ -1072,6 +1143,7 @@ static int intf_require(lua_State *L)
 		ERR_LUA << m << '\n';
 		return 0;
 	}
+#endif
 
 	// Execute it.
 	if (!luaW_pcall(L, 0, 1)) return 0;

src/serialization/ucs4_convert_impl.hpp

@@ -0,0 +1,233 @@
+/*
+   Copyright (C) 2003 - 2014 by David White <dave@whitevine.net>
+   Part of the Battle for Wesnoth Project http://www.wesnoth.org/
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2 of the License, or
+   (at your option) any later version.
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY.
+
+   See the COPYING file for more details.
+*/
+
+#ifndef SERIALIZATION_UCS4_CONVERT_IMPL_HPP_INCLUDED
+#define SERIALIZATION_UCS4_CONVERT_IMPL_HPP_INCLUDED
+
+#include "unicode_types.hpp"
+#include "util.hpp"
+#include <cassert>
+
+namespace ucs4_convert_impl
+{
+	struct utf8_impl
+	{
+		static const char* get_name()  { return "utf8"; };
+		static size_t byte_size_from_ucs4_codepoint(ucs4::char_t ch)
+		{
+			if(ch < (1u << 7))
+				return 1;
+			else if(ch < (1u << 11))
+				return 2;
+			else if(ch < (1u << 16))
+				return 3;
+			else if(ch < (1u << 21))
+				return 4;
+			else if(ch < (1u << 26))
+				return 5;
+			else if(ch < (1u << 31))
+				return 6;
+			else
+				throw utf8::invalid_utf8_exception(); // Invalid UCS-4
+		}
+
+		static int byte_size_from_utf8_first(utf8::char_t ch)
+		{
+			if (!(ch & 0x80)) {
+				return 1;  // US-ASCII character, 1 byte
+			}
+			/* first bit set: character not in US-ASCII, multiple bytes
+			* number of set bits at the beginning = bytes per character
+			* e.g. 11110xxx indicates a 4-byte character */
+			int count = count_leading_ones(ch);
+			if (count == 1 || count > 6) {		// count > 4 after RFC 3629
+				throw utf8::invalid_utf8_exception(); // Stop on invalid characters
+			}
+			return count;
+		};
+
+		/**
+			@param out an object to write utf8::char_t. required operations are:
+				1) push(utf8::char_t) to write a single character
+				2) can_push(size_t n) to check whether there is still enough space
+					for n characters.
+			@param ch the ucs4 chracter to write to the stream.
+		*/
+		template<typename writer>
+		static inline void write(writer out, ucs4::char_t ch)
+		{
+			size_t count = byte_size_from_ucs4_codepoint(ch);
+			assert(out.can_push(count));
+			if(count == 1) {
+				out.push(static_cast<utf8::char_t>(ch));
+			} else {
+				for(int j = static_cast<int>(count) - 1; j >= 0; --j) {
+					unsigned char c = (ch >> (6 * j)) & 0x3f;
+					c |= 0x80;
+					if(j == static_cast<int>(count) - 1) {
+						c |= 0xff << (8 - count);
+					}
+					out.push(c);
+				}
+			}
+		}	
+		/**
+			reads an ucs4 character from an utf8 stream
+			@param input an iterator pointing to the first character of a utf8 sequence to read
+			@param end an iterator poinint to the end of teh utf8 sequence to read.
+		*/
+		template<typename iitor_t>
+		static inline ucs4::char_t read(iitor_t& input, const iitor_t& end)
+		{
+			assert(input != end);
+			size_t size = byte_size_from_utf8_first(*input);
+
+			ucs4::char_t current_char = static_cast<unsigned char>(*input);
+
+			// Convert the first character
+			if(size != 1) {
+				current_char &= 0xFF >> (size + 1);
+			}
+
+			// Convert the continuation bytes
+			// i == number of '++input'
+			++input;
+			for(size_t i = 1; i < size; ++i, ++input) {
+				// If the string ends occurs within an UTF8-sequence, this is bad.
+				if (input == end)
+					throw utf8::invalid_utf8_exception();
+
+				if ((*input & 0xC0) != 0x80)
+					throw utf8::invalid_utf8_exception();
+
+				current_char = (current_char << 6) | (static_cast<unsigned char>(*input) & 0x3F);
+			}
+			//i == size => input was increased size times.
+
+			// Check for non-shortest-form encoding
+			// This has been forbidden in Unicode 3.1 for security reasons
+			if (size > byte_size_from_ucs4_codepoint(current_char))
+				throw utf8::invalid_utf8_exception();
+			return current_char;
+		}
+	};
+	
+	struct utf16_impl
+	{
+		static const char* get_name()  { return "utf16"; };
+		template<typename writer>
+		static inline void write(writer out, ucs4::char_t ch)
+		{
+			const ucs4::char_t bit17 = 0x10000;
+
+			if(ch < bit17)
+			{
+				assert(out.can_push(1));
+				out.push(static_cast<utf16::char_t>(ch));
+			}
+			else 
+			{
+				assert(out.can_push(2));
+				const ucs4::char_t char20 = ch - bit17;
+				assert(char20 < (1 << 20));
+				const ucs4::char_t lead = 0xD800 + (char20 >> 10);
+				const ucs4::char_t trail = 0xDC00 + (char20 & 0x3FF);
+				assert(lead < bit17);
+				assert(trail < bit17);
+				out.push(static_cast<utf16::char_t>(lead));
+				out.push(static_cast<utf16::char_t>(trail));
+			}
+		}
+
+		template<typename iitor_t>
+		static inline ucs4::char_t read(iitor_t& input, const iitor_t& end)
+		{
+			const ucs4::char_t last10 = 0x3FF;
+			const ucs4::char_t type_filter = 0xFC00;
+			const ucs4::char_t type_lead = 0xD800;
+			const ucs4::char_t type_trail = 0xDC00;
+
+			assert(input != end);
+			ucs4::char_t current_char = static_cast<utf16::char_t>(*input);
+			++input;
+			ucs4::char_t type = current_char & type_filter;
+			if(type == type_trail)
+			{
+				//found trail without head
+				throw utf8::invalid_utf8_exception();
+			}
+			else if(type == type_lead)
+			{
+				if(input == end)
+				{
+					//If the string ends occurs within an UTF16-sequence, this is bad.
+					throw utf8::invalid_utf8_exception();
+				}
+				if((*input & type_filter) != type_trail)
+				{
+					throw utf8::invalid_utf8_exception();
+				}
+				current_char &= last10;
+				current_char <<= 10;
+				current_char += (*input & last10);
+				current_char += 0x10000;
+				++input;
+			}
+			return current_char;
+		}
+	};
+	
+	struct utf32_impl
+	{
+		static const char* get_name()  { return "UCS4"; };
+		template<typename writer>
+		static inline void write(writer out, ucs4::char_t ch)
+		{
+			assert(out.can_push(1));
+			out.push(ch);
+		}
+
+		template<typename iitor_t>
+		static inline ucs4::char_t read(iitor_t& input, const iitor_t& end)
+		{
+			assert(input != end);
+			ucs4::char_t current_char = *input;
+			++input;
+			return current_char;
+		}
+	};
+
+	template<typename T_CHAR>
+	struct convert_impl {};
+
+	template<>
+	struct convert_impl<utf8::char_t>
+	{
+		typedef utf8_impl type;
+	};
+	
+	template<>
+	struct convert_impl<utf16::char_t>
+	{
+		typedef utf16_impl type;
+	};
+	
+	template<>
+	struct convert_impl<ucs4::char_t>
+	{
+		typedef utf32_impl type;
+	};
+}
+
+#endif

src/serialization/ucs4_iterator_base.hpp

@@ -0,0 +1,99 @@
+#ifndef UCS4_ITERATOR_BASE_HPP_INCLUDED
+#define UCS4_ITERATOR_BASE_HPP_INCLUDED
+
+#include <stdint.h>  //uint32_t
+#include <iterator>  //input_iterator_tag
+#include <utility>   //pair
+#include <cstddef>   //ptrdiff_t
+#include <cassert>   //assert
+
+#include "unicode_types.hpp"
+
+namespace ucs4
+{
+	template<typename string_type, typename update_implementation>
+	class iterator_base
+	{
+	public:
+		typedef std::input_iterator_tag iterator_category;
+		typedef ucs4::char_t value_type;
+		typedef ptrdiff_t difference_type;
+		typedef ucs4::char_t* pointer;
+		typedef ucs4::char_t& reference;
+
+		iterator_base(const string_type& str)
+			: current_char(0)
+			, string_end(str.end())
+			, current_substr(std::make_pair(str.begin(), str.begin()))
+		{
+			update();
+		}
+
+		iterator_base(typename string_type::const_iterator const &begin, typename string_type::const_iterator const &end)
+			: current_char(0)
+			, string_end(end)
+			, current_substr(std::make_pair(begin, begin))
+		{
+			update();
+		}
+
+		static iterator_base begin(const string_type& str)
+		{
+			return iterator_base(str.begin(), str.end());
+		}
+
+		static iterator_base end(const string_type& str)
+		{
+			return iterator_base(str.end(), str.end());
+		}
+
+		bool operator==(const iterator_base& a) const
+		{
+			return current_substr.first == a.current_substr.first;
+		}
+
+		bool operator!=(const iterator_base& a) const
+		{ 
+			return ! (*this == a); 
+		}
+
+		iterator_base& operator++()
+		{
+			current_substr.first = current_substr.second;
+			update();
+			return *this;
+		}
+
+		ucs4::char_t operator*() const
+		{
+			return current_char;
+		}
+
+		bool next_is_end() const
+		{
+			if(current_substr.second == string_end)
+				return true;
+			return false;
+		}
+
+		const std::pair<typename string_type::const_iterator, typename string_type::const_iterator>& substr() const
+		{
+			return current_substr;
+		}
+	private:
+		void update()
+		{
+			assert(current_substr.first == current_substr.second);
+			if(current_substr.first == string_end)
+				return;
+			current_char = update_implementation::read(current_substr.second, string_end);
+		}
+
+		ucs4::char_t current_char;
+		typename string_type::const_iterator string_end;
+		std::pair<typename string_type::const_iterator, typename string_type::const_iterator> current_substr;
+	};
+
+}
+
+#endif

src/serialization/unicode.cpp

@@ -0,0 +1,134 @@
+/*
+   Copyright (C) 2003 by David White <dave@whitevine.net>
+   Copyright (C) 2005 by Guillaume Melquiond <guillaume.melquiond@gmail.com>
+   Copyright (C) 2005 - 2014 by Philippe Plantier <ayin@anathas.org>
+   Part of the Battle for Wesnoth Project http://www.wesnoth.org/
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2 of the License, or
+   (at your option) any later version.
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY.
+
+   See the COPYING file for more details.
+*/
+
+/**
+ * @file
+ * Unicode support functions.
+ */
+
+#include "global.hpp"
+#include "ucs4_convert_impl.hpp"
+#include "unicode_cast.hpp"
+#include "serialization/unicode.hpp"
+
+#include "log.hpp"
+#include "util.hpp"
+
+#include <cassert>
+#include <limits>
+#include <boost/foreach.hpp>
+
+static lg::log_domain log_engine("engine");
+#define ERR_GENERAL LOG_STREAM(err, lg::general)
+
+namespace utf8 {
+
+static int byte_size_from_utf8_first(const unsigned char ch)
+{
+	if (!(ch & 0x80)) {
+		return 1;  // US-ASCII character, 1 byte
+	}
+	/* first bit set: character not in US-ASCII, multiple bytes
+	 * number of set bits at the beginning = bytes per character
+	 * e.g. 11110xxx indicates a 4-byte character */
+	int count = count_leading_ones(ch);
+	if (count == 1 || count > 6) {		// count > 4 after RFC 3629
+		throw invalid_utf8_exception(); // Stop on invalid characters
+	}
+	return count;
+}
+
+utf8::string lowercase(const utf8::string& s)
+{
+	if(!s.empty()) {
+		utf8::iterator itor(s);
+		utf8::string res;
+
+		for(;itor != utf8::iterator::end(s); ++itor) {
+			ucs4::char_t uchar = *itor;
+			// If wchar_t is less than 32 bits wide, we cannot apply towlower() to all codepoints
+			if(uchar <= static_cast<ucs4::char_t>(std::numeric_limits<wchar_t>::max()))
+				uchar = towlower(static_cast<wchar_t>(uchar));
+			res += unicode_cast<utf8::string>(uchar);
+		}
+
+		res.append(itor.substr().second, s.end());
+		return res;
+	}
+	return s;
+}
+
+size_t index(const utf8::string& str, const size_t index)
+{
+	// chr counts characters, i is the codepoint index
+	// remark: several functions rely on the fallback to str.length()
+	unsigned int i = 0, len = str.size();
+	try {
+		for (unsigned int chr=0; chr<index && i<len; ++chr) {
+			i += byte_size_from_utf8_first(str[i]);
+		}
+	} catch(invalid_utf8_exception&) {
+		ERR_GENERAL << "Invalid UTF-8 string." << std::endl;
+	}
+	return i;
+}
+
+size_t size(const utf8::string& str)
+{
+	unsigned int chr, i = 0, len = str.size();
+	try {
+		for (chr=0; i<len; ++chr) {
+			i += byte_size_from_utf8_first(str[i]);
+		}
+	} catch(invalid_utf8_exception&) {
+		ERR_GENERAL << "Invalid UTF-8 string." << std::endl;
+	}
+	return chr;
+}
+
+utf8::string& insert(utf8::string& str, const size_t pos, const utf8::string& insert)
+{
+	return str.insert(index(str, pos), insert);
+}
+
+utf8::string& erase(utf8::string& str, const size_t start, const size_t len)
+{
+	if (start > size(str)) return str;
+	unsigned pos = index(str, start);
+
+	if (len == std::string::npos) {
+		// without second argument, std::string::erase truncates
+		return str.erase(pos);
+	} else {
+		return str.erase(pos, index(str,start+len) - pos);
+	}
+}
+
+utf8::string& truncate(utf8::string& str, const size_t size)
+{
+	return erase(str, size);
+}
+
+void truncate_as_ucs4(utf8::string &str, const size_t size)
+{
+	ucs4::string u4_str = unicode_cast<ucs4::string>(str);
+	if(u4_str.size() > size) {
+		u4_str.resize(size);
+		str = unicode_cast<utf8::string>(u4_str);
+	}
+}
+
+} // end namespace utf8

src/serialization/unicode.hpp

@@ -0,0 +1,88 @@
+/*
+   Copyright (C) 2003 by David White <dave@whitevine.net>
+   Copyright (C) 2005 - 2014 by Guillaume Melquiond <guillaume.melquiond@gmail.com>
+   Part of the Battle for Wesnoth Project http://www.wesnoth.org/
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2 of the License, or
+   (at your option) any later version.
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY.
+
+   See the COPYING file for more details.
+*/
+
+#ifndef SERIALIZATION_UNICODE_HPP_INCLUDED
+#define SERIALIZATION_UNICODE_HPP_INCLUDED
+
+#include "ucs4_iterator_base.hpp"
+#include "unicode_types.hpp"
+#include "ucs4_convert_impl.hpp"
+#include "unicode_cast.hpp"
+
+#include <boost/static_assert.hpp>
+#include <string>
+#include <vector>
+
+/**
+ * For win32 API.
+ * On windows, wchar_t is defined as Uint16
+ * Wide strings are expected to be UTF-16
+ */
+namespace utf16 {
+	typedef ucs4::iterator_base<utf16::string, ucs4_convert_impl::convert_impl<char_t>::type> iterator;
+}
+
+namespace utf8 {
+
+	/**
+	* Functions for converting Unicode wide-char strings to UTF-8 encoded strings,
+	* back and forth.
+	*/
+	typedef ucs4::iterator_base<std::string, ucs4_convert_impl::convert_impl<char_t>::type> iterator;
+
+	/** Returns a lowercased version of the string. */
+	utf8::string lowercase(const utf8::string&);
+
+	/**
+	 * codepoint index corresponing to the ...th character in an UTF-8 encoded string
+	 * if there are less than index characters, return str.length()
+	 */
+	size_t index(const utf8::string& str, const size_t index);
+
+	/** length in characters of an UTF-8 encoded string */
+	size_t size(const utf8::string& str);
+
+	/** insert at position pos into an UTF-8 encoded string */
+	utf8::string& insert(utf8::string& str, const size_t pos, const utf8::string& insert);
+
+	/**
+	 * erase len characters at position start from an UTF-8 encoded string
+	 * this implementation doesn't check for valid UTF-8, don't use for user input
+	 */
+	utf8::string& erase(utf8::string& str, const size_t start, const size_t len = std::string::npos);
+
+	/**
+	* truncate an UTF-8 encoded string after size characters
+	* this implementation doesn't check for valid UTF-8, don't use for user input
+	*/
+	utf8::string& truncate(utf8::string& str, const size_t size);
+
+	/**
+	 * Truncate a UTF-8 encoded string.
+	 *
+	 * If the string has more than @p size UTF-8 characters it will be truncated
+	 * to this size.
+	 * The output is guaranteed to be valid UTF-8.
+	 *
+	 * @param[in, out] str The parameter's usage is:
+	 *                     - Input: String encoded in UTF-8.
+	 *                     - Output: String encoded UTF-8 that contains at most @p size
+	 *                       codepoints.
+	 * @param size         The size to truncate at.
+	 */
+	void truncate_as_ucs4(utf8::string& str, const size_t size);
+} // end namespace utf8
+
+#endif

src/serialization/unicode_cast.hpp

@@ -0,0 +1,118 @@
+/*
+   Copyright (C) 2003 - 2014 by David White <dave@whitevine.net>
+   Part of the Battle for Wesnoth Project http://www.wesnoth.org/
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2 of the License, or
+   (at your option) any later version.
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY.
+
+   See the COPYING file for more details.
+*/
+#ifndef SERIALIZATION_UNICODE_CAST_HPP_INCLUDED
+#define SERIALIZATION_UNICODE_CAST_HPP_INCLUDED
+
+#include "ucs4_convert_impl.hpp"
+#include <iostream>   //for std::cerr
+#include <iterator>
+#include <boost/utility/enable_if.hpp>
+#include <boost/type_traits/is_arithmetic.hpp>
+
+namespace ucs4_convert_impl
+{
+	//transforms an outputiterator to a writer for ucs4_convert_impl functions.
+	template<typename oitor_t>
+	struct iteratorwriter
+	{
+		oitor_t& out_;
+		iteratorwriter(oitor_t& out) : out_(out) {};
+
+		bool can_push(size_t /*count*/)
+		{
+			return true;
+		}
+		template<typename value_type>
+		void push(value_type val)
+		{
+			*out_++  = val;
+		}
+	};
+	template<typename Tret, typename Tcheck>
+	struct enableif 
+	{ 
+		typedef Tcheck ignore; 
+		typedef Tret type; 
+	};
+}
+
+/**
+	@param TD  
+		output, a collection type.
+	@param TS 
+		input, a collection type.
+	@return an instance of TD
+*/
+template<typename TD , typename TS>
+typename ucs4_convert_impl::enableif<TD, typename TS::value_type>::type unicode_cast(const TS& source)
+//TD unicode_cast(const TS& source)
+{
+	using namespace ucs4_convert_impl;
+	typedef typename convert_impl<typename TD::value_type>::type t_impl_writer;
+	typedef typename convert_impl<typename TS::value_type>::type t_impl_reader;
+	typedef typename std::back_insert_iterator<TD> t_outputitor;
+	typedef typename TS::const_iterator t_inputitor;
+
+	TD res;
+	try
+	{	
+		t_outputitor inserter(res);
+		iteratorwriter<t_outputitor> dst(inserter);
+		t_inputitor i1 = source.begin();
+		t_inputitor i2 = source.end();
+
+		while(i1 != i2) {
+			t_impl_writer::write (dst, t_impl_reader::read(i1, i2));
+		}
+	}
+	catch(utf8::invalid_utf8_exception&) 
+	{
+		///TODO: use a ERR_.. stream but i dont know whether i can so to in header easily.
+		std::cerr << "Failed to convert a string from " << t_impl_reader::get_name() << " to " << t_impl_writer::get_name() << "\n";
+		return res;
+	}
+	return res;
+}
+
+/**
+	@param TD  
+		output, a collection type.
+	@param TI 
+		input, a single character.
+	@return an instance of TD
+*/
+template<typename TD>
+TD unicode_cast(ucs4::char_t onechar)
+{
+	using namespace ucs4_convert_impl;
+	typedef typename convert_impl<typename TD::value_type>::type t_impl_writer;
+	typedef convert_impl<ucs4::char_t>::type t_impl_reader;
+	typedef typename std::back_insert_iterator<TD> t_outputitor;
+	
+	TD res;
+	try
+	{
+		t_outputitor inserter(res);
+		iteratorwriter<t_outputitor> dst(inserter);
+		t_impl_writer::write (dst, onechar);
+	}
+	catch(utf8::invalid_utf8_exception&) 
+	{
+		///TODO: use a ERR_.. stream but i dont know whether i can so to in header easily.
+		std::cerr << "Failed to convert a string from " << t_impl_reader::get_name() << " to " << t_impl_writer::get_name() << "\n";
+		return res;
+	}
+	return res;
+}
+#endif

src/serialization/unicode_types.hpp

@@ -0,0 +1,46 @@
+/*
+   Copyright (C) 2003 - 2014 by David White <dave@whitevine.net>
+   Part of the Battle for Wesnoth Project http://www.wesnoth.org/
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2 of the License, or
+   (at your option) any later version.
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY.
+
+   See the COPYING file for more details.
+*/
+
+#ifndef SERIALIZATION_UNICODE_TYPES_HPP_INCLUDED
+#define SERIALIZATION_UNICODE_TYPES_HPP_INCLUDED
+
+#include <string>
+#include <vector>
+#include <exception>
+#include <stdint.h>
+
+namespace ucs4 {
+	typedef uint32_t char_t;
+	typedef std::vector<char_t> string;
+}
+
+namespace utf8 {
+	typedef char char_t;
+	typedef std::string string;
+	
+	/** also used for invalid utf16 or ucs4 strings */
+	class invalid_utf8_exception : public std::exception {};
+}
+
+/**
+ * For win32 API.
+ * On windows, wchar_t is defined as Uint16
+ * Wide strings are expected to be UTF-16
+ */
+namespace utf16 {
+	typedef wchar_t char_t;
+	typedef std::vector<char_t> string;
+}
+
+#endif

src/tests/test_commandline_options.cpp

@@ -16,14 +16,15 @@
 #include "commandline_options.hpp"
 
 #include <boost/test/unit_test.hpp>
+#include <boost/assign.hpp>
 
 BOOST_AUTO_TEST_SUITE( cmdline_opts )
 
 BOOST_AUTO_TEST_CASE (test_empty_options)
 {
-	const char *argv[] = {"wesnoth"};
-	const int argc = sizeof(argv)/sizeof(const char *);
-	commandline_options co(argc,const_cast<char**>(argv));
+	
+	std::vector<std::string> args = boost::assign::list_of("wesnoth");
+	commandline_options co(args);
 
 	BOOST_CHECK(!co.bpp);
 	BOOST_CHECK(!co.campaign);
@@ -96,19 +97,16 @@ BOOST_AUTO_TEST_CASE (test_empty_options)
 
 BOOST_AUTO_TEST_CASE (test_default_options)
 {
-	const char *argv[] =
-	{
-		"wesnoth",
-		"--campaign",
-		"--editor",
-		"--logdomains",
-		"--preprocess-output-macros",
-		"--server",
-		"--test"
-	};
-	const int argc = sizeof(argv)/sizeof(const char *);
-	commandline_options co(argc,const_cast<char**>(argv));
+	std::vector<std::string> args = boost::assign::list_of
+		("wesnoth")
+		("--campaign")
+		("--editor")
+		("--logdomains")
+		("--preprocess-output-macros")
+		("--server")
+		("--test");
 
+	commandline_options co(args);
 	BOOST_CHECK(!co.bpp);
 	BOOST_CHECK(co.campaign && co.campaign->empty());
 	BOOST_CHECK(!co.campaign_difficulty);
@@ -180,83 +178,81 @@ BOOST_AUTO_TEST_CASE (test_default_options)
 
 BOOST_AUTO_TEST_CASE (test_full_options)
 {
-	const char *argv[] =
-	{
-		"wesnoth",
-		"--ai-config=1:aifoo",
-		"--ai-config=2:aibar",
-		"--algorithm=3:algfoo",
-		"--algorithm=4:algbar",
-		"--bpp=32",
-		"--campaign=campfoo",
-		"--campaign-difficulty=16",
-		"--campaign-scenario=scenfoo",
-		"--clock",
-		"--controller=5:confoo",
-		"--controller=6:conbar",
-		"--data-dir=datadirfoo",
-		"--data-path",
-		"--debug",
+	std::vector<std::string> args = boost::assign::list_of
+		("wesnoth")
+		("--ai-config=1:aifoo")
+		("--ai-config=2:aibar")
+		("--algorithm=3:algfoo")
+		("--algorithm=4:algbar")
+		("--bpp=32")
+		("--campaign=campfoo")
+		("--campaign-difficulty=16")
+		("--campaign-scenario=scenfoo")
+		("--clock")
+		("--controller=5:confoo")
+		("--controller=6:conbar")
+		("--data-dir=datadirfoo")
+		("--data-path")
+		("--debug")
 #ifdef DEBUG_WINDOW_LAYOUT_GRAPHS
-		"--debug-dot-domain=ddfoo",
-		"--debug-dot-level=dlfoo",
+		("--debug-dot-domain=ddfoo")
+		("--debug-dot-level=dlfoo")
 #endif
-		"--editor=editfoo",
-		"--era=erafoo",
-		"--exit-at-end",
-		"--fps",
-		"--fullscreen",
-		"--gunzip=gunzipfoo.gz",
-		"--gzip=gzipfoo",
-		"--help",
-		"--ignore-map-settings",
-		"--label=labelfoo",
-		"--load=loadfoo",
-		"--log-error=errfoo,errbar/*",
-		"--log-warning=warnfoo,warnfoo/bar",
-		"--log-info=infofoo",
-		"--log-debug=dbgfoo,dbgbar,dbg/foo/bar/baz",
-		"--logdomains=filterfoo",
-		"--max-fps=100",
-		"--multiplayer",
-		"--new-widgets",
-		"--nocache",
-		"--nodelay",
-		"--nomusic",
-		"--nosound",
-		"--nogui",
-		"--parm=7:parmfoo:valfoo",
-		"--parm=8:parmbar:valbar",
-		"--path",
-		"--preprocess", "preppathfoo", "preptargfoo",
-		"--preprocess-defines=DEFFOO,DEFBAR",
-		"--preprocess-input-macros=inmfoo",
-		"--preprocess-output-macros=outmfoo",
-		"--proxy",
-		"--proxy-address=addressfoo",
-		"--proxy-password=passfoo",
-		"--proxy-port=portfoo",
-		"--proxy-user=userfoo",
-		"--resolution=800x600",
-		"--rng-seed=1234",
-		"--scenario=scenfoo",
-		"--screenshot", "mapfoo", "outssfoo",
-		"--side=9:sidefoo",
-		"--side=10:sidebar",
-		"--server=servfoo",
-		"--test=testfoo",
-		"--turns=42",
-		"--userconfig-dir=userconfigdirfoo",
-		"--userconfig-path",
-		"--userdata-dir=userdatadirfoo",
-		"--userdata-path",
-		"--validcache",
-		"--version",
-		"--windowed",
-		"--with-replay"
-	};
-	const int argc = sizeof(argv)/sizeof(const char *);
-	commandline_options co(argc,const_cast<char**>(argv));
+		("--editor=editfoo")
+		("--era=erafoo")
+		("--exit-at-end")
+		("--fps")
+		("--fullscreen")
+		("--gunzip=gunzipfoo.gz")
+		("--gzip=gzipfoo")
+		("--help")
+		("--ignore-map-settings")
+		("--label=labelfoo")
+		("--load=loadfoo")
+		("--log-error=errfoo,errbar/*")
+		("--log-warning=warnfoo,warnfoo/bar")
+		("--log-info=infofoo")
+		("--log-debug=dbgfoo,dbgbar,dbg/foo/bar/baz")
+		("--logdomains=filterfoo")
+		("--max-fps=100")
+		("--multiplayer")
+		("--new-widgets")
+		("--nocache")
+		("--nodelay")
+		("--nomusic")
+		("--nosound")
+		("--nogui")
+		("--parm=7:parmfoo:valfoo")
+		("--parm=8:parmbar:valbar")
+		("--path")
+		("--preprocess") ("preppathfoo") ("preptargfoo")
+		("--preprocess-defines=DEFFOO,DEFBAR")
+		("--preprocess-input-macros=inmfoo")
+		("--preprocess-output-macros=outmfoo")
+		("--proxy")
+		("--proxy-address=addressfoo")
+		("--proxy-password=passfoo")
+		("--proxy-port=portfoo")
+		("--proxy-user=userfoo")
+		("--resolution=800x600")
+		("--rng-seed=1234")
+		("--scenario=scenfoo")
+		("--screenshot") ("mapfoo") ("outssfoo")
+		("--side=9:sidefoo")
+		("--side=10:sidebar")
+		("--server=servfoo")
+		("--test=testfoo")
+		("--turns=42")
+		("--userconfig-dir=userconfigdirfoo")
+		("--userconfig-path")
+		("--userdata-dir=userdatadirfoo")
+		("--userdata-path")
+		("--validcache")
+		("--version")
+		("--windowed")
+		("--with-replay");
+
+	commandline_options co(args);
 
 	BOOST_CHECK(co.bpp && *co.bpp == 32);
 	BOOST_CHECK(co.campaign && *co.campaign == "campfoo");
@@ -350,13 +346,11 @@ BOOST_AUTO_TEST_CASE (test_full_options)
 
 BOOST_AUTO_TEST_CASE (test_positional_options)
 {
-	const char *argv[] =
-	{
-		"wesnoth",
-		"datadirfoo"
-	};
-	const int argc = sizeof(argv)/sizeof(const char *);
-	commandline_options co(argc,const_cast<char**>(argv));
+	std::vector<std::string> args = boost::assign::list_of
+		("wesnoth")
+		("datadirfoo");
+
+	commandline_options co(args);
 
 	BOOST_CHECK(!co.bpp);
 	BOOST_CHECK(!co.campaign);

src/tests/test_mp_connect.cpp

@@ -24,6 +24,7 @@
 
 #include <boost/foreach.hpp>
 #include <boost/scoped_ptr.hpp>
+#include <boost/assign.hpp>
 
 
 /* Definitions */
@@ -62,8 +63,8 @@ boost::scoped_ptr<game_state> state;
 struct mp_connect_fixture {
 	mp_connect_fixture() :
 		video(),
-		dummy_argv(),
-		cmdline_opts(1, dummy_argv),
+		dummy_args(boost::assign::list_of("wesnoth").convert_to_container<std::vector<std::string> >()),
+		cmdline_opts(dummy_args),
 		hotkey_manager(),
 		config_manager()
 	{
@@ -91,7 +92,7 @@ struct mp_connect_fixture {
 	{
 	}
 	CVideo video;
-	char** dummy_argv;
+	std::vector<std::string> dummy_args;
 	commandline_options cmdline_opts;
 	hotkey::manager hotkey_manager;
 	boost::scoped_ptr<game_config_manager> config_manager;

src/tests/test_util.cpp

@@ -19,6 +19,8 @@
 #include "serialization/string_utils.hpp"
 #include "util.hpp"
 
+#include <boost/cstdint.hpp>
+
 BOOST_AUTO_TEST_SUITE( util )
 
 BOOST_AUTO_TEST_CASE( test_lexical_cast )
@@ -80,6 +82,19 @@ BOOST_AUTO_TEST_CASE( test_lowercase )
 	BOOST_CHECK_EQUAL ( utils::lowercase("FOO") , "foo" );
 }
 
+BOOST_AUTO_TEST_CASE( test_count_leading_ones )
+{
+	BOOST_CHECK( count_leading_ones(0) == 0 );
+	BOOST_CHECK( count_leading_ones(1) == 0 );
+	BOOST_CHECK( count_leading_ones((boost::uint8_t) 0xFF) == 8 );
+	BOOST_CHECK( count_leading_ones((boost::uint16_t) 0xFFFF) == 16 );
+	BOOST_CHECK( count_leading_ones((boost::uint32_t) 0xFFFFFFFF) == 32 );
+	BOOST_CHECK( count_leading_ones((boost::uint64_t) 0xFFFFFFFFFFFFFFFF)
+		== 64 );
+	BOOST_CHECK( count_leading_ones((boost::uint8_t) 0xF8) == 5 );
+	BOOST_CHECK( count_leading_ones((boost::uint16_t) 54321) == 2 );
+}
+
 /* vim: set ts=4 sw=4: */
 
 BOOST_AUTO_TEST_SUITE_END()

src/util.hpp

@@ -22,6 +22,8 @@
 
 #include "global.hpp"
 #include <cmath>
+#include <cstddef>
+#include <limits>
 #include <math.h> // cmath may not provide round()
 #include <vector>
 #include <sstream>
@@ -209,6 +211,220 @@ bool in_ranges(const Cmp c, const std::vector<std::pair<Cmp, Cmp> >&ranges) {
 inline bool chars_equal_insensitive(char a, char b) { return tolower(a) == tolower(b); }
 inline bool chars_less_insensitive(char a, char b) { return tolower(a) < tolower(b); }
 
+/**
+ * Returns the size, in bits, of an instance of type `T`, providing a
+ * convenient and self-documenting name for the underlying expression:
+ *
+ *     sizeof(T) * std::numeric_limits<unsigned char>::digits
+ *
+ * @tparam T The return value is the size, in bits, of an instance of this
+ * type.
+ *
+ * @returns the size, in bits, of an instance of type `T`.
+ */
+template<typename T>
+inline std::size_t bit_width() {
+	return sizeof(T) * std::numeric_limits<unsigned char>::digits;
+}
+
+/**
+ * Returns the size, in bits, of `x`, providing a convenient and
+ * self-documenting name for the underlying expression:
+ *
+ *     sizeof(x) * std::numeric_limits<unsigned char>::digits
+ *
+ * @tparam T The type of `x`.
+ *
+ * @param x The return value is the size, in bits, of this object.
+ *
+ * @returns the size, in bits, of an instance of type `T`.
+ */
+template<typename T>
+inline std::size_t bit_width(const T& x) {
+	//msvc 2010 gives an unused parameter warning otherwise
+	(void)x;
+	return sizeof(x) * std::numeric_limits<unsigned char>::digits;
+}
+
+/**
+ * Returns the quantity of `1` bits in `n` — i.e., `n`’s population count.
+ *
+ * Algorithm adapted from:
+ * <https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetKernighan>
+ *
+ * This algorithm was chosen for relative simplicity, not for speed.
+ *
+ * @tparam N The type of `n`. This should be a fundamental integer type no
+ * greater than `UINT_MAX` bits in width; if it is not, the return value is
+ * undefined.
+ *
+ * @param n An integer upon which to operate.
+ *
+ * @returns the quantity of `1` bits in `n`, if `N` is a fundamental integer
+ * type.
+ */
+template<typename N>
+inline unsigned int count_ones(N n) {
+	unsigned int r = 0;
+	while (n) {
+		n &= n-1;
+		++r;
+	}
+	return r;
+}
+
+// Support functions for `count_leading_zeros`.
+#if defined(__GNUC__) || defined(__clang__)
+inline unsigned int count_leading_zeros_impl(
+		unsigned char n, std::size_t w) {
+	// Returns the result of the compiler built-in function, adjusted for
+	// the difference between the width, in bits, of the built-in
+	// function’s parameter’s type (which is `unsigned int`, at the
+	// smallest) and the width, in bits, of the input to this function, as
+	// specified at the call-site in `count_leading_zeros`.
+	return static_cast<unsigned int>(__builtin_clz(n))
+		- static_cast<unsigned int>(
+			bit_width<unsigned int>() - w);
+}
+inline unsigned int count_leading_zeros_impl(
+		unsigned short int n, std::size_t w) {
+	return static_cast<unsigned int>(__builtin_clz(n))
+		- static_cast<unsigned int>(
+			bit_width<unsigned int>() - w);
+}
+inline unsigned int count_leading_zeros_impl(
+		unsigned int n, std::size_t w) {
+	return static_cast<unsigned int>(__builtin_clz(n))
+		- static_cast<unsigned int>(
+			bit_width<unsigned int>() - w);
+}
+inline unsigned int count_leading_zeros_impl(
+		unsigned long int n, std::size_t w) {
+	return static_cast<unsigned int>(__builtin_clzl(n))
+		- static_cast<unsigned int>(
+			bit_width<unsigned long int>() - w);
+}
+inline unsigned int count_leading_zeros_impl(
+		unsigned long long int n, std::size_t w) {
+	return static_cast<unsigned int>(__builtin_clzll(n))
+		- static_cast<unsigned int>(
+			bit_width<unsigned long long int>() - w);
+}
+inline unsigned int count_leading_zeros_impl(
+		char n, std::size_t w) {
+	return count_leading_zeros_impl(
+		static_cast<unsigned char>(n), w);
+}
+inline unsigned int count_leading_zeros_impl(
+		signed char n, std::size_t w) {
+	return count_leading_zeros_impl(
+		static_cast<unsigned char>(n), w);
+}
+inline unsigned int count_leading_zeros_impl(
+		signed short int n, std::size_t w) {
+	return count_leading_zeros_impl(
+		static_cast<unsigned short int>(n), w);
+}
+inline unsigned int count_leading_zeros_impl(
+		signed int n, std::size_t w) {
+	return count_leading_zeros_impl(
+		static_cast<unsigned int>(n), w);
+}
+inline unsigned int count_leading_zeros_impl(
+		signed long int n, std::size_t w) {
+	return count_leading_zeros_impl(
+		static_cast<unsigned long int>(n), w);
+}
+inline unsigned int count_leading_zeros_impl(
+		signed long long int n, std::size_t w) {
+	return count_leading_zeros_impl(
+		static_cast<unsigned long long int>(n), w);
+}
+#else
+template<typename N>
+inline unsigned int count_leading_zeros_impl(N n, std::size_t w) {
+	// Algorithm adapted from:
+	// <http://aggregate.org/MAGIC/#Leading%20Zero%20Count>
+	for (unsigned int shift = 1; shift < w; shift *= 2) {
+		n |= (n >> shift);
+	}
+	return static_cast<unsigned int>(w) - count_ones(n);
+}
+#endif
+
+/**
+ * Returns the quantity of leading `0` bits in `n` — i.e., the quantity of
+ * bits in `n`, minus the 1-based bit index of the most significant `1` bit in
+ * `n`, or minus 0 if `n` is 0.
+ *
+ * @tparam N The type of `n`. This should be a fundamental integer type that
+ *  (a) is not wider than `unsigned long long int` (the GCC
+ *   count-leading-zeros built-in functions are defined for `unsigned int`,
+ *   `unsigned long int`, and `unsigned long long int`), and
+ *  (b) is no greater than `INT_MAX` bits in width (the GCC built-in functions
+ *   return instances of type `int`);
+ * if `N` does not satisfy these constraints, the return value is undefined.
+ *
+ * @param n An integer upon which to operate.
+ *
+ * @returns the quantity of leading `0` bits in `n`, if `N` satisfies the
+ * above constraints.
+ *
+ * @see count_leading_ones()
+ */
+template<typename N>
+inline unsigned int count_leading_zeros(N n) {
+#if defined(__GNUC__) || defined(__clang__)
+	// GCC’s `__builtin_clz` returns an undefined value when called with 0
+	// as argument.
+	// [<http://gcc.gnu.org/onlinedocs/gcc/Other-Builtins.html>]
+	if (n == 0) {
+		// Return the quantity of zero bits in `n` rather than
+		// returning that undefined value.
+		return static_cast<unsigned int>(bit_width(n));
+	}
+#endif
+	// Dispatch, on the static type of `n`, to one of the
+	// `count_leading_zeros_impl` functions.
+	return count_leading_zeros_impl(n, bit_width(n));
+	// The second argument to `count_leading_zeros_impl` specifies the
+	// width, in bits, of `n`.
+	//
+	// This is necessary because `n` may be widened (or, alas, shrunk),
+	// and thus the information of `n`’s true width may be lost.
+	//
+	// At least, this *was* necessary before there were so many overloads
+	// of `count_leading_zeros_impl`, but I’ve kept it anyway as an extra
+	// precautionary measure, that will (I hope) be optimized out.
+	//
+	// To be clear, `n` would only be shrunk in cases noted above as
+	// having an undefined result.
+}
+
+/**
+ * Returns the quantity of leading `1` bits in `n` — i.e., the quantity of
+ * bits in `n`, minus the 1-based bit index of the most significant `0` bit in
+ * `n`, or minus 0 if `n` contains no `0` bits.
+ *
+ * @tparam N The type of `n`. This should be a fundamental integer type that
+ *  (a) is not wider than `unsigned long long int`, and
+ *  (b) is no greater than `INT_MAX` bits in width;
+ * if `N` does not satisfy these constraints, the return value is undefined.
+ *
+ * @param n An integer upon which to operate.
+ *
+ * @returns the quantity of leading `1` bits in `n`, if `N` satisfies the
+ * above constraints.
+ *
+ * @see count_leading_zeros()
+ */
+template<typename N>
+inline unsigned int count_leading_ones(N n) {
+	// Explicitly specify the type for which to instantiate
+	// `count_leading_zeros` in case `~n` is of a different type.
+	return count_leading_zeros<N>(~n);
+}
+
 #ifdef __GNUC__
 #define LIKELY(a)    __builtin_expect((a),1) // Tells GCC to optimize code so that if is likely to happen
 #define UNLIKELY(a)  __builtin_expect((a),0) // Tells GCC to optimize code so that if is unlikely to happen