0ct0pu5/wesnoth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Replaced split_utf8_string by a new "utf8_iterator" class

Browse source 
Moved all utf8 encoding / decoding-related code to
serialization/string_utils.hpp
This commit is contained in:
Philippe Plantier 2005-03-03 21:52:40 +00:00
parent d3bfad16f1
commit 1949375acf
19 changed files with 293 additions and 238 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
src/builder.cpp
View file

@ -90,11 +90,16 @@ void terrain_builder::tile::rebuild_cache(const std::string &tod) const
ordered_ri_list::const_iterator itor;
for(itor = horizontal_images.begin(); itor != horizontal_images.end(); ++itor) {
add_image_to_cache(tod, itor);
if (itor->first <= 0)
add_image_to_cache(tod, itor);
}
for(itor = vertical_images.begin(); itor != vertical_images.end(); ++itor) {
add_image_to_cache(tod, itor);
}
for(itor = horizontal_images.begin(); itor != horizontal_images.end(); ++itor) {
if (itor->first > 0)
add_image_to_cache(tod, itor);
}
}
void terrain_builder::tile::clear()

1
src/filechooser.cpp
View file

@ -2,6 +2,7 @@
#include "show_dialog.hpp"
#include "widgets/file_chooser.hpp"
#include "gettext.hpp"
#include <vector>
#include <string>

71
src/font.cpp
View file

@ -83,53 +83,34 @@ std::vector<text_chunk> split_text(const std::string& utf8_text) {
std::vector<text_chunk> chunks;
try {
size_t i = 0;
while(i < utf8_text.size()) {
wchar_t ch = (unsigned char)utf8_text[i];
const int num_bytes = byte_size_from_utf8_first(ch);
if(i + num_bytes > utf8_text.size()) {
throw invalid_utf8_exception();
}
if(num_bytes != 1) {
ch &= 0xFF >> (num_bytes + 1);
}
for(size_t j = i + 1; j != i + num_bytes; ++j) {
const unsigned char ch2 = utf8_text[j];
if((ch2 & 0xC0) != 0x80) {
throw invalid_utf8_exception();
}
ch = (ch << 6) | (ch2 & 0x3F);
}
//size_t i = 0;
for(utils::utf8_iterator ch(utf8_text); ch != utils::utf8_iterator::end(utf8_text); ++ch) {
if(first) {
if(ch < font_map.size() && font_map[ch] >= 0) {
current_font = font_map[ch];
if(*ch < font_map.size() && font_map[*ch] >= 0) {
current_font = font_map[*ch];
} else {
current_font = 0;
}
first = false;
}
if(ch >= font_map.size() || font_map[ch] < 0) {
current_chunk.append(utf8_text, i, num_bytes);
} else if(font_map[ch] == current_font) {
current_chunk.append(utf8_text, i, num_bytes);
if(*ch >= font_map.size() || font_map[*ch] < 0) {
current_chunk.append(ch.substr().first, ch.substr().second);
} else if(font_map[*ch] == current_font) {
current_chunk.append(ch.substr().first, ch.substr().second);
} else {
chunks.push_back(text_chunk(current_font, current_chunk));
current_chunk.clear();
current_chunk.append(utf8_text, i, num_bytes);
current_font = font_map[ch];
current_chunk.append(ch.substr().first, ch.substr().second);
current_font = font_map[*ch];
}
i += num_bytes;
}
if (!current_chunk.empty()) {
chunks.push_back(text_chunk(current_font, current_chunk));
}
}
catch(invalid_utf8_exception e) {
catch(utils::invalid_utf8_exception e) {
WRN_FT << "Invalid UTF-8 string: \"" << utf8_text << "\"\n";
}
return chunks;
@ -801,16 +782,17 @@ std::string word_wrap_text(const std::string& unwrapped_text, int font_size, int
if (line_width(cur_word, font_size) > (max_width /*/ 2*/)) {
// The last word is too big to fit in a nice way, split it on a char basis
std::vector<std::string> split_word = split_utf8_string(cur_word);
utils::utf8_iterator i(cur_word);
for (std::vector<std::string>::iterator i = split_word.begin(); i != split_word.end(); ++i) {
if (line_width(cur_line + *i, font_size) > max_width) {
for (; i != utils::utf8_iterator::end(cur_word); ++i) {
std::string tmp = cur_line;
tmp.append(i.substr().first, i.substr().second);
if (line_width(tmp, font_size) > max_width) {
wrapped_text += cur_line + '\n';
cur_line = *i;
} else {
cur_line += *i;
cur_line = "";
}
cur_line.append(i.substr().first, i.substr().second);
}
} else {
@ -847,15 +829,20 @@ std::string make_text_ellipsis(const std::string &text, int font_size, int max_w
if(line_width(ellipsis, font_size) > max_width)
return "";
std::vector<std::string> characters = split_utf8_string(text);
std::string current_substring = "";
std::string current_substring;
for(std::vector<std::string>::const_iterator itor = characters.begin(); itor != characters.end(); ++itor) {
if (line_width(current_substring + *itor + ellipsis, font_size ) > max_width) {
utils::utf8_iterator itor(text);
for(; itor != utils::utf8_iterator::end(text); ++itor) {
std::string tmp = current_substring;
tmp.append(itor.substr().first, itor.substr().second);
tmp += ellipsis;
if (line_width(tmp, font_size) > max_width) {
return current_substring + ellipsis;
}
current_substring += *itor;
current_substring.append(itor.substr().first, itor.substr().second);
}
return text; // Should not happen

34
src/intro.cpp
View file

@ -215,12 +215,10 @@ bool show_intro_part(display& screen, const config& part,
}
const std::string& story = part["story"];
const std::vector<std::string> story_chars = split_utf8_string(story);
utils::utf8_iterator itor(story);
std::cerr << story << std::endl;
std::vector<std::string>::const_iterator j = story_chars.begin();
bool skip = false, last_key = true;
int xpos = textx, ypos = texty;
@ -228,20 +226,19 @@ bool show_intro_part(display& screen, const config& part,
//the maximum position that text can reach before wrapping
const int max_xpos = next_button.location().x - 10;
size_t height = 0;
//std::string buf;
for(;;) {
if(j != story_chars.end()) {
//unsigned char c = *j;
if(*j == " ") {
if(itor != utils::utf8_iterator::end(story)) {
if(*itor == ' ') {
//we're at a space, so find the next space or end-of-text,
//to find out if the next word will fit, or if it has to be wrapped
std::vector<std::string>::const_iterator end_word = std::find(j+1,story_chars.end()," ");
utils::utf8_iterator start_word = itor;
++start_word;
utils::utf8_iterator end_word = std::find(start_word, utils::utf8_iterator::end(story), ' ');
std::string word;
for(std::vector<std::string>::const_iterator k = j+1;
k != end_word; ++k) {
word += *k;
for(; start_word != end_word; ++start_word) {
word.append(start_word.substr().first, start_word.substr().second);
}
const SDL_Rect rect = font::draw_text(NULL,screen.screen_area(),
font::SIZE_PLUS,font::NORMAL_COLOUR,
@ -250,15 +247,18 @@ bool show_intro_part(display& screen, const config& part,
if(xpos + rect.w >= max_xpos) {
xpos = textx;
ypos += height;
++j;
++itor;
continue;
}
}
// output the character
// FIXME: this is broken: it does not take kerning into account.
std::string tmp;
tmp.append(itor.substr().first, itor.substr().second);
const SDL_Rect rect = font::draw_text(&screen,
screen.screen_area(),font::SIZE_PLUS,
font::NORMAL_COLOUR,*j,xpos,ypos,
font::NORMAL_COLOUR,tmp,xpos,ypos,
false);
if(rect.h > height)
@ -266,8 +266,8 @@ bool show_intro_part(display& screen, const config& part,
xpos += rect.w;
update_rect(rect);
++j;
if(j == story_chars.end())
++itor;
if(itor == utils::utf8_iterator::end(story))
skip = true;
}
@ -275,7 +275,7 @@ bool show_intro_part(display& screen, const config& part,
const bool keydown = key[SDLK_SPACE] || key[SDLK_RETURN];
if(keydown && !last_key || next_button.pressed()) {
if(skip == true || j == story_chars.end()) {
if(skip == true || itor == utils::utf8_iterator::end(story)) {
break;
} else {
skip = true;
@ -292,7 +292,7 @@ bool show_intro_part(display& screen, const config& part,
events::raise_draw_event();
screen.video().flip();
if(!skip || j == story_chars.end())
if(!skip || itor == utils::utf8_iterator::end(story))
SDL_Delay(20);
}

153
src/language.cpp
View file

@ -209,156 +209,3 @@ const language_def& get_locale()
return known_languages[0];
}
namespace
{
std::string wstring_to_utf8(const wide_string &src)
{
wchar_t ch;
wide_string::const_iterator i;
int j;
Uint32 bitmask;
std::string ret;
try {
for(i = src.begin(); i != src.end(); ++i) {
int count;
ch = *i;
/* Determine the bytes required */
count = 1;
if(ch >= 0x80)
count++;
bitmask = 0x800;
for(j = 0; j < 5; ++j) {
if(ch >= bitmask)
count++;
bitmask <<= 5;
}
if(count > 6)
throw invalid_utf8_exception();
if(count == 1) {
push_back(ret,ch);
} else {
for(j = count-1; j >= 0; --j) {
unsigned char c = (ch >> (6*j)) & 0x3f;
c |= 0x80;
if(j == count-1)
c |= 0xff << (8 - count);
push_back(ret,c);
}
}
}
return ret;
}
catch(invalid_utf8_exception e) {
std::cerr << "Invalid wide character string\n";
return ret;
}
}
wide_string utf8_to_wstring(const std::string &src)
{
wide_string ret;
try {
for(size_t i = 0, l = src.size(); i != l;) {
wchar_t ch = (unsigned char)src[i];
const int count = byte_size_from_utf8_first(ch);
if (i + count > l)
throw invalid_utf8_exception();
/* Convert the first character */
if (count != 1) {
ch &= 0xFF >> (count + 1);
}
/* Convert the continuation bytes */
for(size_t j = i + 1; j != i + count; ++j) {
unsigned char ch2 = src[j];
if ((ch2 & 0xC0) != 0x80)
throw invalid_utf8_exception();
ch = (ch << 6) | (ch2 & 0x3F);
}
i += count;
push_back(ret,ch);
}
}
catch(invalid_utf8_exception e) {
std::cerr << "Invalid UTF-8 string: \"" << src << "\"\n";
return ret;
}
return ret;
}
}
int byte_size_from_utf8_first(unsigned char ch)
{
int count;
if ((ch & 0x80) == 0)
count = 1;
else if ((ch & 0xE0) == 0xC0)
count = 2;
else if ((ch & 0xF0) == 0xE0)
count = 3;
else if ((ch & 0xF8) == 0xF0)
count = 4;
else if ((ch & 0xFC) == 0xF8)
count = 5;
else if ((ch & 0xFE) == 0xFC)
count = 6;
else
throw invalid_utf8_exception(); /* stop on invalid characters */
return count;
}
std::vector<std::string> split_utf8_string(const std::string &src)
{
std::vector<std::string> ret;
try {
for(size_t i = 0; i < src.size(); /* nop */) {
const int size = byte_size_from_utf8_first(src[i]);
if(i + size > src.size())
throw invalid_utf8_exception();
ret.push_back(src.substr(i, size));
i += size;
}
}
catch(invalid_utf8_exception e) {
std::cerr << "Invalid UTF-8 string: \"" << src << "\"\n";
return ret;
}
return ret;
}
std::string wstring_to_string(const wide_string &src)
{
return wstring_to_utf8(src);
}
std::string wchar_to_string(const wchar_t c) {
wide_string s;
s.push_back(c);
return wstring_to_utf8(s);
}
wide_string string_to_wstring(const std::string &src)
{
return utf8_to_wstring(src);
}

14
src/language.hpp
View file

@ -19,9 +19,7 @@
#include <map>
#include <string>
#include <vector>
//the type we use to represent Unicode strings.
typedef std::vector<wchar_t> wide_string;
#include <iterator>
//this module controls internationalization.
@ -65,14 +63,4 @@ const language_def& get_language();
//function which attempts to query and return the locale on the system
const language_def& get_locale();
//functions for converting Unicode wide-char strings to UTF-8 encoded
//strings, back and forth
class invalid_utf8_exception : public std::exception {
};
int byte_size_from_utf8_first(unsigned char ch);
std::vector<std::string> split_utf8_string(const std::string &src);
std::string wstring_to_string(const wide_string &);
wide_string string_to_wstring(const std::string &);
std::string wchar_to_string(const wchar_t);
#endif

1
src/leader_list.cpp
View file

@ -11,6 +11,7 @@
See the COPYING file for more details.
*/
#include "gettext.hpp"
#include "global.hpp"
#include "leader_list.hpp"
#include "wml_separators.hpp"

1
src/multiplayer.cpp
View file

@ -23,6 +23,7 @@
#include "log.hpp"
#include "playlevel.hpp"
#include "network.hpp"
#include "gettext.hpp"
#define LOG_NW lg::info(lg::network)

1
src/multiplayer_connect.cpp
View file

@ -19,6 +19,7 @@
#include "preferences.hpp"
#include "show_dialog.hpp"
#include "wassert.hpp"
#include "gettext.hpp"
#include "serialization/string_utils.hpp"
#define LOG_NW lg::info(lg::network)

1
src/multiplayer_create.cpp
View file

@ -16,6 +16,7 @@
#include "multiplayer_create.hpp"
#include "filesystem.hpp"
#include "preferences.hpp"
#include "gettext.hpp"
namespace {
const SDL_Rect null_rect = {0, 0, 0, 0};

1
src/multiplayer_lobby.cpp
View file

@ -17,6 +17,7 @@
#include "font.hpp"
#include "wassert.hpp"
#include "wml_separators.hpp"
#include "gettext.hpp"
namespace mp {

1
src/multiplayer_ui.cpp
View file

@ -16,6 +16,7 @@
#include "network.hpp"
#include "game_config.hpp"
#include "sound.hpp"
#include "gettext.hpp"
#define LOG_NW lg::info(lg::network)
#define ERR_NW lg::err(lg::network)

1
src/multiplayer_wait.cpp
View file

@ -20,6 +20,7 @@
#include "util.hpp"
#include "wassert.hpp"
#include "wml_separators.hpp"
#include "gettext.hpp"
#include "serialization/string_utils.hpp"
#define LOG_NW lg::info(lg::network)

4
src/race.cpp
View file

@ -25,7 +25,7 @@ markov_prefix_map markov_prefixes(const std::vector<std::string>& items, size_t
markov_prefix_map res;
for(std::vector<std::string>::const_iterator i = items.begin(); i != items.end(); ++i) {
add_prefixes(string_to_wstring(*i),length,res);
add_prefixes(utils::string_to_wstring(*i),length,res);
}
return res;
@ -117,7 +117,7 @@ const std::string& unit_race::name() const { return name_; }
std::string unit_race::generate_name(unit_race::GENDER gender) const
{
return wstring_to_string(markov_generate_name(next_[gender],chain_size_,12));
return utils::wstring_to_string(markov_generate_name(next_[gender],chain_size_,12));
}
const config::child_list& unit_race::additional_traits() const

2
src/race.hpp
View file

@ -5,7 +5,7 @@
#include <string>
#include <vector>
#include "language.hpp"
#include "serialization/string_utils.hpp"
#include "config.hpp"
typedef std::map<wide_string, std::vector<wchar_t> > markov_prefix_map;

180
src/serialization/string_utils.cpp
View file

@ -18,6 +18,11 @@
#include <sstream>
#include "serialization/string_utils.hpp"
#include "util.hpp"
#include "log.hpp"
#include "SDL_types.h"
#define ERR_GENERAL lg::err(lg::general)
namespace game_events {
std::string const &get_variable_const(std::string const &varname);
@ -303,4 +308,179 @@ std::pair< int, int > parse_range(std::string const &str)
return res;
}
int byte_size_from_utf8_first(unsigned char ch)
{
int count;
if ((ch & 0x80) == 0)
count = 1;
else if ((ch & 0xE0) == 0xC0)
count = 2;
else if ((ch & 0xF0) == 0xE0)
count = 3;
else if ((ch & 0xF8) == 0xF0)
count = 4;
else if ((ch & 0xFC) == 0xF8)
count = 5;
else if ((ch & 0xFE) == 0xFC)
count = 6;
else
throw invalid_utf8_exception(); /* stop on invalid characters */
return count;
}
utf8_iterator::utf8_iterator() :
current_char(0)
{
}
utf8_iterator::utf8_iterator(const std::string& str)
{
current_substr.first = str.begin();
string_end = str.end();
update();
}
utf8_iterator::utf8_iterator(std::string::const_iterator beg, std::string::const_iterator end)
{
current_substr.first = beg;
string_end = end;
update();
}
utf8_iterator utf8_iterator::end(const std::string& str)
{
utf8_iterator res;
res.current_char = 0;
res.current_substr.first = str.end();
res.string_end = str.end();
return res;
}
bool utf8_iterator::operator==(const utf8_iterator& a)
{
return current_substr.first == a.current_substr.first;
}
utf8_iterator& utf8_iterator::operator++()
{
current_substr.first = current_substr.second;
update();
return *this;
}
wchar_t utf8_iterator::operator*()
{
return current_char;
}
const std::pair<std::string::const_iterator, std::string::const_iterator>& utf8_iterator::substr()
{
return current_substr;
}
void utf8_iterator::update()
{
size_t size = byte_size_from_utf8_first(*current_substr.first);
current_substr.second = current_substr.first + size;
current_char = (unsigned char)(*current_substr.first);
/* Convert the first character */
if (size != 1) {
current_char &= 0xFF >> (size + 1);
}
/* Convert the continuation bytes */
for(std::string::const_iterator c = current_substr.first+1;
c != current_substr.second; ++c) {
// If the string ends occurs within an UTF8-sequence, this is
// bad.
if (c == string_end)
throw invalid_utf8_exception();
if ((*c & 0xC0) != 0x80)
throw invalid_utf8_exception();
current_char = (current_char << 6) | ((unsigned char)*c & 0x3F);
}
}
std::string wstring_to_string(const wide_string &src)
{
wchar_t ch;
wide_string::const_iterator i;
int j;
Uint32 bitmask;
std::string ret;
try {
for(i = src.begin(); i != src.end(); ++i) {
int count;
ch = *i;
/* Determine the bytes required */
count = 1;
if(ch >= 0x80)
count++;
bitmask = 0x800;
for(j = 0; j < 5; ++j) {
if(ch >= bitmask)
count++;
bitmask <<= 5;
}
if(count > 6)
throw invalid_utf8_exception();
if(count == 1) {
push_back(ret,ch);
} else {
for(j = count-1; j >= 0; --j) {
unsigned char c = (ch >> (6*j)) & 0x3f;
c |= 0x80;
if(j == count-1)
c |= 0xff << (8 - count);
push_back(ret,c);
}
}
}
return ret;
}
catch(invalid_utf8_exception e) {
ERR_GENERAL << "Invalid wide character string\n";
return ret;
}
}
std::string wchar_to_string(const wchar_t c) {
wide_string s;
s.push_back(c);
return wstring_to_string(s);
}
wide_string string_to_wstring(const std::string &src)
{
wide_string res;
try {
res.insert(res.end(), utf8_iterator(src), utf8_iterator::end(src));
}
catch(invalid_utf8_exception e) {
ERR_GENERAL << "Invalid UTF-8 string: \"" << src << "\"\n";
return res;
}
return res;
}
}

40
src/serialization/string_utils.hpp
View file

@ -19,6 +19,9 @@
#include <string>
#include <vector>
//the type we use to represent Unicode strings.
typedef std::vector<wchar_t> wide_string;
namespace utils {
bool isnewline(char c);
@ -45,6 +48,43 @@ typedef std::map< std::string, std::string > string_map;
// variables will be used instead
std::string interpolate_variables_into_string(std::string const &str, string_map const *symbols = NULL);
//functions for converting Unicode wide-char strings to UTF-8 encoded
//strings, back and forth
class invalid_utf8_exception : public std::exception {
};
class utf8_iterator
{
public:
typedef std::input_iterator_tag iterator_category;
typedef wchar_t value_type;
typedef ptrdiff_t difference_type;
typedef wchar_t* pointer;
typedef wchar_t& reference;
utf8_iterator();
utf8_iterator(const std::string& str);
utf8_iterator(std::string::const_iterator begin, std::string::const_iterator end);
static utf8_iterator end(const std::string& str);
bool operator==(const utf8_iterator& a);
bool operator!=(const utf8_iterator& a) { return ! (*this == a); }
utf8_iterator& operator++();
wchar_t operator*();
const std::pair<std::string::const_iterator, std::string::const_iterator>& substr();
private:
void update();
wchar_t current_char;
std::string::const_iterator string_end;
std::pair<std::string::const_iterator, std::string::const_iterator> current_substr;
};
std::string wstring_to_string(const wide_string &);
wide_string string_to_wstring(const std::string &);
std::string wchar_to_string(const wchar_t);
}
#endif

16
src/widgets/textbox.cpp
View file

@ -32,7 +32,7 @@ namespace gui {
const int font_size = font::SIZE_PLUS;
textbox::textbox(display& d, int width, const std::string& text, bool editable, size_t max_size, double alpha, double alpha_focus)
: scrollarea(d), max_size_(max_size), text_(string_to_wstring(text)),
: scrollarea(d), max_size_(max_size), text_(utils::string_to_wstring(text)),
cursor_(text_.size()), selstart_(-1), selend_(-1),
grabmouse_(false), text_pos_(0), editable_(editable),
show_cursor_(true), show_cursor_at_(0), text_image_(NULL),
@ -53,14 +53,14 @@ void textbox::set_inner_location(SDL_Rect const &rect)
const std::string textbox::text() const
{
const std::string &ret = wstring_to_string(text_);
const std::string &ret = utils::wstring_to_string(text_);
return ret;
}
// set_text does not respect max_size_
void textbox::set_text(const std::string& text)
{
text_ = string_to_wstring(text);
text_ = utils::string_to_wstring(text);
cursor_ = text_.size();
text_pos_ = 0;
selstart_ = -1;
@ -76,7 +76,7 @@ void textbox::append_text(const std::string& text)
return;
}
const wide_string& wtext = string_to_wstring(text);
const wide_string& wtext = utils::string_to_wstring(text);
const surface new_text = add_text_line(wtext);
const surface new_surface = create_compatible_surface(text_image_,maximum<size_t>(text_image_->w,new_text->w),text_image_->h+new_text->h);
@ -255,7 +255,7 @@ surface textbox::add_text_line(const wide_string& text)
if(char(*itor) == ' ') {
backup_itor = itor;
}
visible_string.append(wchar_to_string(*itor));
visible_string.append(utils::wchar_to_string(*itor));
if(char(*itor) == '\n') {
backup_itor = text.end();
@ -287,7 +287,7 @@ surface textbox::add_text_line(const wide_string& text)
}
}
const std::string s = wstring_to_string(wrapped_text);
const std::string s = utils::wstring_to_string(wrapped_text);
const surface res(font::get_rendered_text(s, font_size, font::NORMAL_COLOUR));
return res;
@ -473,7 +473,7 @@ void textbox::handle_event(const SDL_Event& event)
if(is_selection())
erase_selection();
wide_string s = string_to_wstring(copy_from_clipboard());
wide_string s = utils::string_to_wstring(copy_from_clipboard());
if(text_.size() < max_size_) {
if(s.size() + text_.size() > max_size_) {
s.resize(max_size_ - text_.size());
@ -491,7 +491,7 @@ void textbox::handle_event(const SDL_Event& event)
const size_t end = maximum<size_t>(size_t(selstart_),size_t(selend_));
wide_string ws = wide_string(text_.begin() + beg, text_.begin() + end);
std::string s = wstring_to_string(ws);
std::string s = utils::wstring_to_string(ws);
copy_to_clipboard(s);
}
break;

2
src/widgets/textbox.hpp
View file

@ -16,7 +16,7 @@
#include "../events.hpp"
#include "../key.hpp"
#include "../language.hpp"
#include "../serialization/string_utils.hpp"
#include "../sdl_utils.hpp"
#include "scrollarea.hpp"