Merge pull request #2647 from AI0867/base64-refactor

Base64 refactor

source_lists/libwesnoth_core

@@ -15,6 +15,7 @@ map/map.cpp
 mt_rng.cpp
 random.cpp
 seed_rng.cpp
+serialization/base64.cpp
 serialization/binary_or_text.cpp
 serialization/parser.cpp
 serialization/preprocessor.cpp

src/campaign_server/campaign_server.cpp

@@ -24,6 +24,7 @@
 #include "filesystem.hpp"
 #include "lexical_cast.hpp"
 #include "log.hpp"
+#include "serialization/base64.hpp"
 #include "serialization/binary_or_text.hpp"
 #include "serialization/parser.hpp"
 #include "serialization/string_utils.hpp"
@@ -76,14 +77,13 @@ bool authenticate(config& campaign, const config::attribute_value& passphrase)
 
 std::string generate_salt(size_t len)
 {
-	static const std::string itoa64 = "./0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
 	boost::mt19937 mt(time(0));
 	std::string salt = std::string(len, '0');
-	boost::uniform_int<> from_str(0, itoa64.length() - 1);
+	boost::uniform_int<> from_str(0, 63); // 64 possible values for base64
 	boost::variate_generator< boost::mt19937, boost::uniform_int<>> get_char(mt, from_str);
 
 	for(size_t i = 0; i < len; i++) {
-		salt[i] = itoa64[get_char()];
+		salt[i] = crypt64::encode(get_char());
 	}
 
 	return salt;

src/hash.cpp

@@ -14,6 +14,8 @@
 
 #include "hash.hpp"
 
+#include "serialization/base64.hpp"
+
 #include <iostream>
 #include <string>
 #include <sstream>
@@ -32,31 +34,12 @@ static_assert(utils::sha1::DIGEST_SIZE == SHA_DIGEST_LENGTH, "Constants mismatch
 
 namespace {
 
-const std::string itoa64 = "./0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" ;
 const std::string hash_prefix = "$H$";
 
 template<size_t len>
-std::string encode_hash(const std::array<uint8_t, len>& input) {
-	std::string encoded_hash;
-
-	unsigned int i = 0;
-	do {
-		unsigned value = input[i++];
-		encoded_hash.append(itoa64.substr(value & 0x3f,1));
-		if(i < len)
-			value |= static_cast<int>(input[i]) << 8;
-		encoded_hash.append(itoa64.substr((value >> 6) & 0x3f,1));
-		if(i++ >= len)
-			break;
-		if(i < len)
-			value |= static_cast<int>(input[i]) << 16;
-		encoded_hash.append(itoa64.substr((value >> 12) & 0x3f,1));
-		if(i++ >= len)
-			break;
-		encoded_hash.append(itoa64.substr((value >> 18) & 0x3f,1));
-	} while (i < len);
-
-	return encoded_hash;
+std::string encode_hash(const std::array<uint8_t, len>& bytes) {
+	utils::byte_string_view view{bytes.data(), len};
+	return crypt64::encode(view);
 }
 
 template<size_t len>
@@ -81,7 +64,7 @@ md5::md5(const std::string& input) {
 }
 
 int md5::get_iteration_count(const std::string& hash) {
-	return itoa64.find_first_of(hash[3]);
+	return crypt64::decode(hash[3]);
 }
 
 std::string md5::get_salt(const std::string& hash) {

src/image.cpp

@@ -29,6 +29,7 @@
 #include "image_modifications.hpp"
 #include "log.hpp"
 #include "preferences/general.hpp"
+#include "serialization/base64.hpp"
 #include "serialization/string_utils.hpp"
 #include "sdl/rect.hpp"
 #include "utils/general.hpp"
@@ -666,36 +667,6 @@ static surface load_image_sub_file(const image::locator& loc)
 	return surf;
 }
 
-static std::string base64_decode(utils::string_view in)
-{
-	std::vector<int> T(256,-1);
-	for(int i=0; i<64; i++) {
-		T["ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"[i]] = i;
-	}
-
-	const int last_char = in.find_last_not_of("=");
-	const int length = last_char * 6 / 8;
-
-	std::string out;
-	out.reserve(length);
-
-	int val = 0, bits = -8;
-	for(unsigned char c: in) {
-		if(T[c] == -1) {
-			break; // Non-base64 character encountered. Should be =
-		}
-		val = (val<<6) + T[c];
-		bits += 6;
-		if(bits >= 0) {
-			out.push_back(static_cast<char>((val >> bits) & 0xFF));
-			bits -= 8;
-			val &= 0xFFFF; // Prevent shifting bits off the left end, which is UB
-		}
-	}
-
-	return out;
-}
-
 static surface load_image_data_uri(const image::locator& loc)
 {
 	surface surf;
@@ -709,8 +680,8 @@ static surface load_image_data_uri(const image::locator& loc)
 	} else if(parsed.mime.substr(0, 5) != "image") {
 		ERR_DP << "Data URI not of image MIME type: " << parsed.mime << std::endl;
 	} else {
-		const std::string image_data = base64_decode(parsed.data);
-		filesystem::rwops_ptr rwops{SDL_RWFromConstMem(image_data.data(), image_data.length()), &SDL_FreeRW};
+		const std::vector<uint8_t> image_data = base64::decode(parsed.data);
+		filesystem::rwops_ptr rwops{SDL_RWFromConstMem(image_data.data(), image_data.size()), &SDL_FreeRW};
 
 		if(parsed.mime == "image/png") {
 			surf = IMG_LoadTyped_RW(rwops.release(), true, "PNG");

src/serialization/base64.cpp

@@ -0,0 +1,232 @@
+/*
+   Copyright (C) 2003 - 2018 by David White <dave@whitevine.net>
+   Part of the Battle for Wesnoth Project http://www.wesnoth.org/
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2 of the License, or
+   (at your option) any later version.
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY.
+
+   See the COPYING file for more details.
+*/
+
+#include "serialization/base64.hpp"
+
+#include <vector>
+
+namespace {
+const std::string base64_itoa_map = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+const std::string crypt64_itoa_map = "./0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
+
+void fill_atoi_map(std::vector<int>& atoi, const std::string& itoa)
+{
+	for(int i=0; i<64; ++i) {
+		atoi[itoa[i]] = i;
+	}
+}
+
+const std::vector<int>& base64_atoi_map()
+{
+	static std::vector<int> atoi64(256, -1);
+	if(atoi64['A'] == -1) {
+		fill_atoi_map(atoi64, base64_itoa_map);
+	}
+	return atoi64;
+}
+const std::vector<int>& crypt64_atoi_map()
+{
+	static std::vector<int> atoi64(256, -1);
+	if(atoi64['A'] == -1) {
+		fill_atoi_map(atoi64, crypt64_itoa_map);
+	}
+	return atoi64;
+}
+char itoa(unsigned value, const std::string& map)
+{
+	return map[value & 0x3f];
+}
+
+std::vector<uint8_t> generic_decode_be(utils::string_view in, const std::vector<int>& atoi_map)
+{
+	const int last_char = in.find_last_not_of("=");
+	const int length = last_char * 6 / 8;
+
+	std::vector<uint8_t> out;
+	out.reserve(length);
+
+	int val = 0, bits = -8;
+	for(unsigned char c: in) {
+		if(atoi_map[c] == -1) {
+			break; // Non-base64 character encountered. Should be =
+		}
+		val = (val<<6) + atoi_map[c];
+		bits += 6;
+		if(bits >= 0) {
+			out.push_back(static_cast<char>((val >> bits) & 0xFF));
+			bits -= 8;
+			val &= 0xFFFF; // Prevent shifting bits off the left end, which is UB
+		}
+	}
+
+	return out;
+}
+
+std::vector<uint8_t> generic_decode_le(utils::string_view in, const std::vector<int>& atoi_map)
+{
+	const int last_char = in.find_last_not_of("=");
+	const int length = last_char * 6 / 8;
+
+	std::vector<uint8_t> out;
+	out.reserve(length);
+
+	for(int i = 0; i <= last_char; i += 4) {
+		//add first char (always)
+		unsigned value = atoi_map[in[i]];
+
+		const bool second_char = i + 1 <= last_char;
+		if(!second_char) {
+			break;
+		}
+		//add second char (if present)
+		value |= atoi_map[in[i+1]] << 6;
+
+		//output first byte (if second char)
+		out.push_back(value & 0xFF);
+
+		const bool third_char = i + 2 <= last_char;
+		if(!third_char) {
+			break;
+		}
+		//add third char (if present)
+		value |= atoi_map[in[i+2]] << 12;
+
+		//output second byte (if third char)
+		out.push_back((value >> 8) & 0xFF);
+
+		const bool fourth_char = i + 3 <= last_char;
+		if(!fourth_char) {
+			break;
+		}
+		//add fourth char (if present)
+		value |= atoi_map[in[i+3]] << 18;
+
+		//output third byte (if fourth char)
+		out.push_back((value >> 16) & 0xFF);
+	}
+
+	return out;
+}
+
+std::string generic_encode_be(utils::byte_string_view in, const std::string& itoa_map, bool pad)
+{
+	const int in_len = in.length();
+	const int groups = (in_len + 2) / 3;
+	const int out_len = groups * 4;
+
+	std::string out;
+
+	int i = 0;
+	out.reserve(out_len);
+	unsigned value = 0;
+	unsigned bits = 0;
+	while(i < in_len) {
+		value <<= 8;
+		value |= in[i++];
+		bits += 8;
+		do {
+			bits -= 6;
+			out.push_back(itoa(value >> bits, itoa_map));
+		} while(bits >= 6);
+	}
+	if(bits > 0) {
+		out.push_back(itoa(value << (6 - bits), itoa_map));
+	}
+
+	if(pad) {
+		// If not round, append = chars
+		out.resize(out_len, '=');
+	}
+
+	return out;
+
+}
+std::string generic_encode_le(utils::byte_string_view in, const std::string& itoa_map, bool pad)
+{
+	const int in_len = in.length();
+	const int groups = (in_len + 2) / 3;
+	const int out_len = groups * 4;
+
+	std::string out;
+
+	int i = 0;
+	out.reserve(out_len);
+	while(i < in_len) {
+		//add first byte (always)
+		unsigned value = in[i];
+		//output first char (always)
+		out.push_back(itoa(value, itoa_map));
+		//add second byte (if present)
+		const bool second_byte = ++i < in_len;
+		if(second_byte) {
+			value |= static_cast<int>(in[i]) << 8;
+		}
+		//output second char (always, contains 2 bits from first byte)
+		out.push_back(itoa(value >> 6, itoa_map));
+		if(!second_byte) {
+			break;
+		}
+		//add third byte (if present)
+		const bool third_byte = ++i < in_len;
+		if(third_byte) {
+			value |= static_cast<int>(in[i]) << 16;
+		}
+		//output third char (if second byte)
+		out.push_back(itoa(value >> 12, itoa_map));
+		//output fourth char (if third byte)
+		if(third_byte) {
+			out.push_back(itoa(value >> 18, itoa_map));
+			++i;
+		}
+	}
+
+	if(pad) {
+		// If not round, append = chars
+		out.resize(out_len, '=');
+	}
+
+	return out;
+
+}
+}
+
+namespace base64 {
+std::vector<uint8_t> decode(utils::string_view in)
+{
+	return generic_decode_be(in, base64_atoi_map());
+}
+std::string encode(utils::byte_string_view bytes)
+{
+	return generic_encode_be(bytes, base64_itoa_map, true);
+}
+}
+namespace crypt64{
+std::vector<uint8_t> decode(utils::string_view in)
+{
+	return generic_decode_le(in, crypt64_atoi_map());
+}
+std::string encode(utils::byte_string_view bytes)
+{
+	return generic_encode_le(bytes, crypt64_itoa_map, false);
+}
+int decode(char encoded_char)
+{
+	size_t pos = crypt64_itoa_map.find(encoded_char);
+	return pos == std::string::npos ? -1 : pos;
+}
+char encode(int value)
+{
+	return itoa(value, crypt64_itoa_map);
+}
+}

src/serialization/base64.hpp

@@ -0,0 +1,33 @@
+/*
+   Copyright (C) 2003 - 2018 by David White <dave@whitevine.net>
+   Part of the Battle for Wesnoth Project http://www.wesnoth.org/
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2 of the License, or
+   (at your option) any later version.
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY.
+
+   See the COPYING file for more details.
+*/
+
+#pragma once
+
+#include "serialization/string_view.hpp"
+
+#include <vector>
+
+// Official Base64 encoding (RFC4648)
+namespace base64 {
+	std::vector<uint8_t> decode(utils::string_view encoded);
+	std::string encode(utils::byte_string_view bytes);
+}
+// crypt()-compatible radix-64 encoding
+namespace crypt64 {
+	std::vector<uint8_t> decode(utils::string_view encoded);
+	std::string encode(utils::byte_string_view bytes);
+	// Single character functions. For special use only
+	int decode(char encoded_char);
+	char encode(int value);
+}

src/serialization/string_view.hpp

@@ -28,6 +28,7 @@ that class. */
 
 namespace utils {
 using string_view = boost::string_view;
+typedef boost::basic_string_view<uint8_t, std::char_traits<uint8_t>> byte_string_view;
 }
 
 #else
@@ -605,6 +606,7 @@ const basic_string_view<charT, traits>& str) {
 }
 
 typedef basic_string_view<char, std::char_traits<char>>        string_view;
+typedef basic_string_view<uint8_t, std::char_traits<uint8_t>>  byte_string_view;
 typedef basic_string_view<wchar_t, std::char_traits<wchar_t>>    wstring_view;
 
 #ifndef BOOST_NO_CXX11_CHAR16_T

src/server/user_handler.cpp

@@ -15,6 +15,7 @@
 #include "server/user_handler.hpp"
 #include "config.hpp"
 #include "random.hpp"
+#include "serialization/base64.hpp"
 #include <openssl/rand.h>
 
 #include <array>
@@ -52,36 +53,7 @@ std::string user_handler::create_unsecure_nonce(int length) {
 	return  ss.str();
 }
 
-// TODO - This really should be a common function.
-// This is duplicated in two or three other places.
-// Some are virtual member functions.
 namespace {
-	const std::string itoa64 = "./0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" ;
-
-	template<size_t len>
-	std::string encode_hash(const std::array<unsigned char, len>& input) {
-		std::string encoded_hash;
-
-		unsigned int i = 0;
-		do {
-			unsigned value = input[i++];
-			encoded_hash.append(itoa64.substr(value & 0x3f,1));
-			if(i < len)
-				value |= static_cast<int>(input[i]) << 8;
-			encoded_hash.append(itoa64.substr((value >> 6) & 0x3f,1));
-			if(i++ >= len)
-				break;
-			if(i < len)
-				value |= static_cast<int>(input[i]) << 16;
-			encoded_hash.append(itoa64.substr((value >> 12) & 0x3f,1));
-			if(i++ >= len)
-				break;
-			encoded_hash.append(itoa64.substr((value >> 18) & 0x3f,1));
-		} while (i < len);
-
-		return encoded_hash;
-	}
-
 	class RAND_bytes_exception: public std::exception
 	{
 	};
@@ -96,6 +68,6 @@ std::string user_handler::create_secure_nonce()
 		throw RAND_bytes_exception();
 	}
 
-	return encode_hash(buf);
+	return base64::encode({buf.data(), buf.size()});
 }
 

src/tests/test_serialization.cpp

@@ -16,6 +16,7 @@
 
 #include <vector>
 #include <string>
+#include "serialization/base64.hpp"
 #include "serialization/string_utils.hpp"
 #include "serialization/unicode.hpp"
 #include <boost/test/auto_unit_test.hpp>
@@ -134,4 +135,50 @@ BOOST_AUTO_TEST_CASE( test_wildcard_string_match )
 	BOOST_CHECK(!utils::wildcard_string_match("", "???"));
 }
 
+BOOST_AUTO_TEST_CASE( test_base64_encodings )
+{
+	const std::vector<uint8_t> empty;
+	const std::string empty_b64;
+	const std::string empty_c64;
+	const std::vector<uint8_t> foo = {'f', 'o', 'o'};
+	const std::string foo_b64 = "Zm9v";
+	const std::string foo_c64 = "axqP";
+	const std::vector<uint8_t> foob = {'f', 'o', 'o', 'b'};
+	const std::string foob_b64 = "Zm9vYg==";
+	const std::string foob_c64 = "axqPW/";
+
+	std::vector<uint8_t> many_bytes;
+
+	many_bytes.resize(1024);
+	for(int i = 0; i < 1024; ++i) {
+		many_bytes[i] = i % 256;
+	}
+
+	BOOST_CHECK(base64::encode({empty.data(), empty.size()}).empty());
+	BOOST_CHECK_EQUAL(base64::encode({foo.data(), foo.size()}), foo_b64);
+	BOOST_CHECK_EQUAL(base64::encode({foob.data(), foob.size()}), foob_b64);
+
+	BOOST_CHECK(base64::decode(empty_b64).empty());
+	// Not using CHECK_EQUAL because vector<uint8_t> is not printable
+	BOOST_CHECK(base64::decode(foo_b64) == foo);
+	BOOST_CHECK(base64::decode(foob_b64) == foob);
+
+	BOOST_CHECK(crypt64::encode({empty.data(), empty.size()}).empty());
+	BOOST_CHECK_EQUAL(crypt64::encode({foo.data(), foo.size()}), foo_c64);
+	BOOST_CHECK_EQUAL(crypt64::encode({foob.data(), foob.size()}), foob_c64);
+
+	BOOST_CHECK(crypt64::decode(empty_c64).empty());
+	// Not using CHECK_EQUAL because vector<uint8_t> is not printable
+	BOOST_CHECK(crypt64::decode(foo_c64) == foo);
+	BOOST_CHECK(crypt64::decode(foob_c64) == foob);
+
+	BOOST_CHECK_EQUAL(crypt64::decode('.'), 0);
+	BOOST_CHECK_EQUAL(crypt64::decode('z'), 63);
+	BOOST_CHECK_EQUAL(crypt64::encode(0), '.');
+	BOOST_CHECK_EQUAL(crypt64::encode(63), 'z');
+
+	BOOST_CHECK(base64::decode(base64::encode({many_bytes.data(), many_bytes.size()})) == many_bytes);
+	BOOST_CHECK(crypt64::decode(crypt64::encode({many_bytes.data(), many_bytes.size()})) == many_bytes);
+}
+
 BOOST_AUTO_TEST_SUITE_END()