Fixed text parser to strip CRs

Made addons upload/downloads escape CR which forces reupload for
anyone having binary data in addon

changelog

@@ -33,6 +33,8 @@ Version 1.5.1+svn:
    * Fixed statitics not add turn data from previus scenarios (bug: #11719)
    * Fixed manager to initialize before gamestate in playcontroller (bug: 11935)
    * Removed persistance from team configuration (bug: #10916)
+   * Fixed text parser again strip cr from configs. Added CR and 254 to be
+     escaped in campaign upload. All UMC has to be reuploaded to server
    * Made automaticaly generated macro reference easier to naviagate and link to
      (patch: #1076)
    * Allow [unit_type] num_traits=0 to override race's num_traits.

src/addon_management.cpp

@@ -105,7 +105,7 @@ namespace {
 	const char escape_char = 1;
 }
 
-static bool needs_escaping(char c) { return c == 0 || c == escape_char; }
+static bool needs_escaping(char c) { return c == 0 || c == escape_char || c == 13 || c == 254; }
 
 static bool IsCR(const char& c)
 {

src/serialization/tokenizer.cpp

@@ -146,10 +146,7 @@ const token& tokenizer::next_token()
 	case '"':
 		token_.type = token::QSTRING;
 		while (1) {
-			/** Have to use next_char_no_strip 
-			 * because we will break image ifwe do striping
-			 **/
-			next_char_no_strip();
+			next_char();
 
 			if(current_ == EOF) {
 				token_.type = token::UNTERMINATED_QSTRING;
@@ -158,20 +155,11 @@ const token& tokenizer::next_token()
 			if(current_ == '"' && peek_char() != '"')
 				break;
 			if(current_ == '"' && peek_char() == '"')
-				next_char_fast_no_strip();
-			if (current_ == 254 && 
-					(peek_char() == 'l' || peek_char() == 't')) {
-				next_char_fast_no_strip();
-				if ((current_ == 'l' && peek_char() == 'i') || (current_ == 't' && peek_char() == 'e'))
-				{
-					skip_comment();
-					--lineno_;
-					continue;
-				}
-				else
-				{
-					token_.value += 254;
-				}
+				next_char_fast();
+			if (current_ == 254 ) {
+				skip_comment();
+				--lineno_;
+				continue;
 			}
 
 			token_.value += current_;

src/serialization/tokenizer.hpp

@@ -71,26 +71,6 @@ protected:
 	int current_;
 	size_t lineno_;
 
-	inline void next_char_no_strip()
-	{
-		if (UNLIKELY(current_ == '\n'))
-			lineno_++;
-		this->next_char_fast_no_strip();
-	}
-
-	inline void next_char_fast_no_strip()
-	{
-		if (LIKELY(in_.good()))
-		{
-			current_ = in_.get();
-		}
-		else
-		{
-			current_ = EOF;
-			return;
-		}
-	}
-
 	inline void next_char()
 	{
 		if (UNLIKELY(current_ == '\n'))