Fixed support for UTF-8 languages on text-boxes.

Added wstring_to_string and string_to_wstring routines in
language.cpp, which convert strings according to the current charset()

src/language.cpp

@@ -213,3 +213,144 @@ std::string get_locale()
 	std::cerr << "locale could not be determined; defaulting to locale 'en'\n";
 	return "en";
 }
+
+class invalid_utf8_exception : public std::exception {
+};
+
+namespace 
+{
+std::string wstring_to_utf8(const std::wstring &src)
+{
+        unsigned wchar_t ch;
+	std::wstring::const_iterator i;
+	int j;
+	Uint32 bitmask;
+	std::string ret;
+
+	try {
+
+		for(i = src.begin(); i != src.end(); ++i) {
+			int count;
+			ch = *i;
+			
+			/* Determine the bytes required */
+			count = 1;
+			if(ch >= 0x80)
+				count++;
+
+			bitmask = 0x800;
+			for(j = 0; j < 5; ++j) {
+				if(ch >= bitmask)
+					count++;
+				bitmask <<= 5;
+			}
+			
+			if(count > 6)
+				throw invalid_utf8_exception();
+
+			if(count == 1) {
+				ret.push_back(ch);
+			} else {
+				for(j = count-1; j >= 0; --j) {
+					unsigned char c = (ch >> (6*j)) & 0x3f;
+					if(j == count-1)
+						c |= 0xff << (8 - count);
+					ret.push_back(c) ;
+				}
+			}
+
+		}		
+	}
+	catch(invalid_utf8_exception e) {
+		std::cerr << "Invalid wide character string\n";
+		return ret;
+	}
+}
+
+std::wstring utf8_to_wstring(const std::string &src)
+{
+	std::wstring ret;	
+        unsigned wchar_t ch;
+	std::string::const_iterator i;
+	
+	try {
+		for(i = src.begin(); i != src.end(); ++i ) {
+			ch = (unsigned char)*i;
+			int count;
+
+			if ((ch & 0x80) == 0)
+				count = 1;
+			else if ((ch & 0xE0) == 0xC0)
+				count = 2;
+			else if ((ch & 0xF0) == 0xE0)
+				count = 3;
+			else if ((ch & 0xF8) == 0xF0)
+				count = 4;
+			else if ((ch & 0xFC) == 0xF8)
+				count = 5;
+			else if ((ch & 0xFE) == 0xFC)
+				count = 6;
+			else
+				throw invalid_utf8_exception(); /* stop on invalid characters */
+		
+			if(i + count - 1 == src.end())
+				throw invalid_utf8_exception();
+
+			/* Convert the first character */
+			if (count != 1) {
+				ch &= 0xFF >> (count + 1);
+			}
+			
+			/* Convert the continuation bytes */
+			for(std::string::const_iterator j = i+1; j != i+count; ++j) {
+				if((*j & 0xC0) != 0x80)
+					throw invalid_utf8_exception();
+				
+				ch = (ch << 6) | (*j & 0x3F);
+			}
+			i += (count - 1);
+			
+			ret.push_back(ch);
+		}
+	}
+	catch(invalid_utf8_exception e) {
+		std::cerr << "Invalid UTF-8 string: \"" << src << "\"\n";
+		return ret;
+	}
+
+	return ret;
+}
+
+}
+
+std::string wstring_to_string(const std::wstring &src)
+{
+	if(charset() == CHARSET_UTF8) {
+		return wstring_to_utf8(src);
+	}
+	
+	std::string ret;
+	for(std::wstring::const_iterator itor = src.begin(); itor != src.end(); ++itor) {
+		if(*itor <= 0xff) {
+			ret.push_back(*itor);
+		} else {
+			ret.push_back('?');
+		}
+	}
+	return ret;
+}
+
+std::wstring string_to_wstring(const std::string &src)
+{
+	if(charset() == CHARSET_UTF8) {
+		return utf8_to_wstring(src);
+	}
+
+	std::wstring ret;
+	for(std::string::const_iterator itor = src.begin(); itor != src.end(); ++itor) {
+		ret.push_back(*itor);
+	}
+	return ret;
+}
+
+

src/language.hpp

@@ -61,6 +61,12 @@ const std::string& get_language();
 //function which attempts to query and return the locale on the system
 std::string get_locale();
 
+//functions for converting Unicode wide-char strings to UTF-8 encoded
+//strings, back and forth
+std::string wstring_to_string(const std::wstring &);
+std::wstring string_to_wstring(const std::string &);
+
+
 //two character sets are supported: LATIN1 and UTF-8. This is
 //set in the translation by using encoding=(LATIN1|UTF-8)
 //the character set used affects the font rendering function called

src/widgets/textbox.cpp

@@ -15,6 +15,7 @@
 #include "../show_dialog.hpp"
 #include "../video.hpp"
 #include "../util.hpp"
+#include "../language.hpp"
 #include "SDL.h"
 
 #include <algorithm>
@@ -25,8 +26,8 @@ namespace gui {
 const int font_size = 16;
 
 textbox::textbox(display& d, int width, const std::string& text)
-           : widget(d), text_(text), text_pos_(0),
-             cursor_(text.size()), selstart_(-1), selend_(-1), grabmouse_(false),
+           : widget(d), text_(string_to_wstring(text)), text_pos_(0),
+             cursor_(text_.size()), selstart_(-1), selend_(-1), grabmouse_(false),
 	     show_cursor_(true), text_image_(NULL)
 {
 	static const SDL_Rect area = d.screen_area();
@@ -36,14 +37,15 @@ textbox::textbox(display& d, int width, const std::string& text)
 	update_text_cache(true);
 }
 
-const std::string& textbox::text() const
+const std::string textbox::text() const
 {
-	return text_;
+	const std::string &ret = wstring_to_string(text_);
+	return ret;
 }
 
 void textbox::set_text(std::string text)
 {
-	text_ = text;
+	text_ = string_to_wstring(text);
 	cursor_ = text_.size();
 	set_dirty(true);
 	update_text_cache(true);
@@ -51,7 +53,7 @@ void textbox::set_text(std::string text)
 
 void textbox::clear()
 {
-	text_ = "";
+	text_ = L"";
 	cursor_ = 0;
 	cursor_pos_ = 0;
 	text_pos_ = 0;
@@ -134,8 +136,12 @@ void textbox::update_text_cache(bool changed)
 		// width of each substring, but this is a flawed assumption which won't work with
 		// some more complex scripts (that is, RTL languages). This part of the work should
 		// actually be done by the font-rendering system.
-		for(int i = 0; i < text_.size(); ++i) {
-			const std::string visible_string = text_.substr(0, i+1);
+		std::string visible_string;
+		
+		for(std::wstring::const_iterator itor = text_.begin(); itor != text_.end(); itor++) {
+			std::wstring s;
+			s.push_back(*itor);
+			visible_string.append(wstring_to_string(s));
 			const int w = font::line_width(visible_string, font_size);		
 		
 			char_pos_.push_back(w);
@@ -143,10 +149,11 @@ void textbox::update_text_cache(bool changed)
 
 		text_size_.x = 0;
 		text_size_.y = 0;
-		text_size_.w = font::line_width(text_, font_size);
+		const std::string s = wstring_to_string(text_);
+		text_size_.w = font::line_width(s, font_size);
 		text_size_.h = location().h;
 
-		text_image_.assign(font::get_rendered_text(text_, font_size, font::NORMAL_COLOUR));		
+		text_image_.assign(font::get_rendered_text(s, font_size, font::NORMAL_COLOUR));		
 	}
 
 	int cursor_x = char_pos_[cursor_];
@@ -169,7 +176,7 @@ void textbox::erase_selection()
 	if(!is_selection())
 		return;
 	
-	std::string::iterator itor = text_.begin() + minimum(selstart_, selend_);
+	std::wstring::iterator itor = text_.begin() + minimum(selstart_, selend_);
 	text_.erase(itor, itor + abs(selend_ - selstart_));
 	cursor_ = minimum(selstart_, selend_);
 	selstart_ = selend_ = -1;
@@ -276,11 +283,10 @@ void textbox::handle_event(const SDL_Event& event)
 		}
 	}
 
-	// const char character = static_cast<char>(key.unicode);
-	int character = key.unicode;
+	wchar_t character = key.unicode;
 
 	if(character != 0)
-		std::cerr << "Char: " << character << "\n";
+		std::cerr << "Char: " << character << ", c = " << c << "\n";
 	
 	if(/*isgraph(character) || character == ' '*/ character >= 32 && character != 127) {
 		changed = true;
@@ -294,7 +300,7 @@ void textbox::handle_event(const SDL_Event& event)
 	if(is_selection() && (selend_ != cursor_))
 		selstart_ = selend_ = -1;
 
-	update_text_cache(true);
+	update_text_cache(changed);
 	set_dirty(true);
 	draw();
 }

src/widgets/textbox.hpp

@@ -29,7 +29,7 @@ class textbox : public widget
 public:
 	textbox(display& d, int width, const std::string& text="");
 
-	const std::string& text() const;
+	const std::string text() const;
 	void set_text(std::string text);
 	void clear();
 	void process();
@@ -40,7 +40,7 @@ protected:
 	using widget::dirty;
 
 private:
-	std::string text_;
+	std::wstring text_;
 	
 	// mutable unsigned int firstOnScreen_;
 	int cursor_;