made campaign server use gzip for storing addons

added configuraion option to choose gzip compression level

changelog

@@ -12,6 +12,8 @@ Version 1.4.2+svn:
  * campaignd
    * made campaign server use gzip compression for networking
    * made campaign server detect if client can receive gzipped data
+   * made campaign server use gzip for storing addons
+   * added configuraion option to choose gzip compression level
  * miscellaneous and bug fixes:
    * fixed parser bug that prevented loading binary data strings
    * starting a campaing without any installed now gives an error.

src/Makefile.am

@@ -167,6 +167,7 @@ wesnothd_DEPENDENCIES=libwesnoth-core.a
 
 campaignd_SOURCES = \
 	campaign_server/campaign_server.cpp \
+	server/input_stream.cpp \
 	network.cpp \
 	network_worker.cpp \
 	publish_campaign.cpp \

src/campaign_server/campaign_server.cpp

@@ -18,10 +18,14 @@
 #include "network.hpp"
 #include "publish_campaign.hpp"
 #include "util.hpp"
+#include "scoped_resource.hpp"
 #include "serialization/binary_wml.hpp"
+#include "serialization/binary_or_text.hpp"
 #include "serialization/parser.hpp"
 #include "game_config.hpp"
 
+#include "server/input_stream.hpp"
+
 #include "SDL.h"
 
 #include <iostream>
@@ -62,12 +66,17 @@ namespace {
 		public:
 			explicit campaign_server(const std::string& cfgfile,size_t min_thread = 10,size_t max_thread = 0);
 			void run();
+			~campaign_server()
+			{
+				delete input_;
+			}
 		private:
 			/**
 			 * Fires a script, if no script defined it will always return true
 			 * If a script is defined but can't be executed it will return false
 			 */
 			void fire(const std::string& hook, const std::string& addon);
+			void convert_binary_to_gzip();
 			int load_config(); // return the server port
 			const config& campaigns() const { return *cfg_.child("campaigns"); }
 			config& campaigns() { return *cfg_.child("campaigns"); }
@@ -76,6 +85,8 @@ namespace {
 			const network::manager net_manager_;
 			const network::server_manager server_manager_;
 			std::map<std::string, std::string> hooks_;
+			input_stream* input_;
+			int compress_level_;
 
 	};
 
@@ -121,11 +132,14 @@ namespace {
 	{
 		scoped_istream stream = istream_file(file_);
 		read(cfg_, *stream);
+		/** Seems like compression level above 6 is waste of cpu cycle */
+		compress_level_ = lexical_cast_default<int>(cfg_["compress_level"],6);
+		cfg_["compress_level"] = lexical_cast<std::string>(compress_level_);
 		return lexical_cast_default<int>(cfg_["port"], 15005);
 	}
 
 	campaign_server::campaign_server(const std::string& cfgfile,size_t min_thread,size_t max_thread)
-		: file_(cfgfile), net_manager_(min_thread,max_thread), server_manager_(load_config())
+		: file_(cfgfile), net_manager_(min_thread,max_thread), server_manager_(load_config()), input_(0)
 	{
 		if(cfg_.child("campaigns") == NULL) {
 			cfg_.add_child("campaigns");
@@ -163,7 +177,7 @@ namespace {
 			// Campaign contains python scripts.
 			config old_campaign;
 			scoped_istream stream = istream_file(filename);
-			read_compressed(old_campaign, *stream);
+			read_gz(old_campaign, *stream);
 			std::vector<config *> old_scripts = find_scripts(old_campaign, ".py");
 			std::string script_names = "";
 			std::vector<config *>::iterator i, j;
@@ -235,21 +249,66 @@ namespace {
 				", path is \"" << game_config::path << "\"\n";
 		}
 	}
+	void campaign_server::convert_binary_to_gzip()
+	{
+		if (cfg_["converted_to_gzipped_data"] != "yes")
+		{
+			// Convert all addons to gzip
+			config::child_list camps = campaigns().get_children("campaign");
+			LOG_CS << "Converting all stored addons to gzip format. Number of addons: " << camps.size() <<"\n";
+
+			for (config::child_list::iterator itor = camps.begin();
+					itor != camps.end(); ++itor)
+			{
+				LOG_CS << "Converting " << (**itor)["name"] << "\n";
+				scoped_istream binary_stream = istream_file((**itor)["filename"]);
+				config data;
+				if (binary_stream->peek() == 31) //This is gzip file allready
+				{
+					LOG_CS << "All ready converted\n";
+					continue;
+				}
+				read_compressed(data, *binary_stream);
+
+				scoped_ostream gzip_stream = ostream_file((**itor)["filename"]);
+				config_writer writer(*gzip_stream, true, "",compress_level_);
+				writer.write(data);
+			}
+			
+			cfg_["converted_to_gzipped_data"] = "yes";
+		}
+	}
 
 	void campaign_server::run()
 	{
+		
+		convert_binary_to_gzip();
+		
+		if (!cfg_["control_socket"].empty())
+			input_ = new input_stream(cfg_["control_socket"]); 
+		
 		bool gzipped;
+		network::connection sock = 0;
 		for(int increment = 0; ; ++increment) {
 			try {
+				std::string admin_cmd;
+				if (input_ && input_->read_line(admin_cmd))
+				{
+					// process command
+					if (admin_cmd == "shut_down")
+					{
+						break;
+					}
+				}
 				//write config to disk every ten minutes
-				if((increment%(60*10*2)) == 0) {
+				if((increment%(60*10*50)) == 0) {
 					scoped_ostream cfgfile = ostream_file(file_);
 					write(*cfgfile, cfg_);
 				}
 
 				network::process_send_queue();
 
-				network::connection sock = network::accept_connection();
+				sock = network::accept_connection();
 				if(sock) {
 					LOG_CS << "received connection from " << network::ip_address(sock) << "\n";
 				}
@@ -320,15 +379,23 @@ namespace {
 
 							network::send_data(construct_error("Add-on '" + (*req)["name"] + "'not found."), sock, true);
 						} else {
-							config cfg;
+							size_t size = file_size((*campaign)["filename"]);
 							scoped_istream stream = istream_file((*campaign)["filename"]);
-							read_compressed(cfg, *stream);
-							add_license(cfg);
+							if (gzipped)
+							{
+								util::scoped_resource<char*,util::delete_array> buf(new char[size+1]);
+								stream->read(buf,size);
 
-							network::send_data(cfg, sock, gzipped);
+								network::send_raw_data(buf, size, sock);
+							} else {
+								config cfg;
+								read_gz(cfg, *stream);
+								network::send_data(cfg, sock, false);
+							}
 
 							const int downloads = lexical_cast_default<int>((*campaign)["downloads"],0)+1;
 							(*campaign)["downloads"] = lexical_cast<std::string>(downloads);
+
 						}
 
 					} else if(data.child("request_terms") != NULL) {
@@ -375,8 +442,12 @@ namespace {
 								(*campaign).clear_children("translation");
 								find_translations(*data, *campaign);
 
+								add_license(*data);
+
 								scoped_ostream campaign_file = ostream_file(filename);
-								write_compressed(*campaign_file, *data);
+								config_writer writer(*campaign_file, true, "",compress_level_);
+								writer.write(*data);
+//								write_compressed(*campaign_file, *data);
 
 								(*campaign)["size"] = lexical_cast<std::string>(
 										file_size(filename));
@@ -433,8 +504,13 @@ namespace {
 							if ((*campaign)["python"] != "allowed") {
 								message += check_python_scripts(*data, filename);
 							}
+
+							add_license(*data);
+
 							scoped_ostream campaign_file = ostream_file(filename);
-							write_compressed(*campaign_file, *data);
+							config_writer writer(*campaign_file, true, "",compress_level_);
+							writer.write(*data);
+//							write_compressed(*campaign_file, *data);
 
 							(*campaign)["size"] = lexical_cast<std::string>(
 									file_size(filename));
@@ -513,12 +589,14 @@ namespace {
 							// Read the campaign from disk.
 							config campaign_file;
 							scoped_istream stream = istream_file((*campaign)["filename"]);
-							read_compressed(campaign_file, *stream);
+							read_gz(campaign_file, *stream);
 							std::string scripts = validate_all_python_scripts(campaign_file);
 							if (!scripts.empty()) {
 								// Write the campaign with changed filenames back to disk
 								scoped_ostream ostream = ostream_file((*campaign)["filename"]);
-								write_compressed(*ostream, campaign_file);
+								config_writer writer(*ostream, true, "",compress_level_);
+								writer.write(campaign_file);
+//								write_compressed(*ostream, campaign_file);
 
 
 								network::send_data(construct_message("The following scripts have been validated: " +
@@ -540,9 +618,10 @@ namespace {
 				}
 			} catch(config::error& /*e*/) {
 				LOG_CS << "error in receiving data...\n";
+				network::disconnect(sock);
 			}
 
-			SDL_Delay(500);
+			SDL_Delay(20);
 		}
 	}
 

src/serialization/binary_or_text.cpp

@@ -49,7 +49,7 @@ void write_possibly_compressed(std::ostream &out, config &cfg, bool compress)
 }
 
 config_writer::config_writer(
-	std::ostream &out, bool compress, const std::string &textdomain) :
+	std::ostream &out, bool compress, const std::string &textdomain,int level) :
 		filter_(),
 		out_ptr_(compress ? &filter_ : &out), //ternary indirection creates a temporary
 		out_(*out_ptr_), //now MSVC will allow binding to the reference member
@@ -58,7 +58,11 @@ config_writer::config_writer(
 		textdomain_(textdomain)
 {
 	if(compress_) {
-		filter_.push(boost::iostreams::gzip_compressor());
+		if (level >=0)
+			filter_.push(boost::iostreams::gzip_compressor(boost::iostreams::gzip_params(level)));
+		else
+			filter_.push(boost::iostreams::gzip_compressor(boost::iostreams::gzip_params()));
+
 		filter_.push(out);
 	}
 }

src/serialization/binary_or_text.hpp

@@ -39,7 +39,7 @@ void write_possibly_compressed(std::ostream &out, config &cfg, bool compress);
 class config_writer
 {
 public:
-	config_writer(std::ostream &out, bool compress, const std::string &textdomain);
+	config_writer(std::ostream &out, bool compress, const std::string &textdomain, int level = -1);
 
 	void write(const config &cfg);
 	void write_child(const std::string &key, const config &cfg);