Add initial version of the schema validator.

Committing Sytyi's patch.

src/CMakeLists.txt

@@ -185,7 +185,7 @@ if(MSVC)
 	SOURCE_GROUP("src tests gui" REGULAR_EXPRESSION "tests/gui/.*")
 	SOURCE_GROUP("src tests utils" REGULAR_EXPRESSION "tests/utils/.*")
 	SOURCE_GROUP("src tools" REGULAR_EXPRESSION "tools/.*")
-	SOURCE_GROUP("src tools" REGULAR_EXPRESSION "tools/.*")
+	SOURCE_GROUP("src tools schema" REGULAR_EXPRESSION "tools/schema/.*")
 	SOURCE_GROUP("src whiteboard" REGULAR_EXPRESSION "whiteboard/.*")
 
 endif(MSVC)
@@ -794,8 +794,21 @@ set_target_properties(cutter PROPERTIES OUTPUT_NAME ${BINARY_PREFIX}cutter${BINA
 
 install(TARGETS cutter DESTINATION ${BINDIR})
 
-endif(ENABLE_TOOLS)
+set(schema_generator_SRC
+	tools/schema/schema_generator.cpp
+	tools/schema/sourceparser.cpp
+	tools/schema/error_container.cpp
+	tools/schema/tag.cpp
+	filesystem.cpp
+	loadscreen_empty.cpp
+)
 
+add_executable(schema_generator ${schema_generator_SRC})
+target_link_libraries(schema_generator wesnoth-core ${tools-external-libs} ${Boost_REGEX_LIBRARY})
+set_target_properties(schema_generator PROPERTIES OUTPUT_NAME ${BINARY_PREFIX}schema_generator${BINARY_SUFFIX})
+
+install(TARGETS schema_generator DESTINATION ${BINDIR})
+endif(ENABLE_TOOLS)
 ########### Unit tests ###############
 
 if(ENABLE_TESTS)

src/tools/schema/error_container.cpp

@@ -0,0 +1,74 @@
+/*
+   Copyright (C) 2011 - 2011 by Sytyi Nick <nsytyi@gmail.com>
+   Part of the Battle for Wesnoth Project http://www.wesnoth.org/
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2 of the License, or
+   (at your option) any later version.
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY.
+
+   See the COPYING file for more details.
+*/
+
+/**
+ * @file
+ * This file contains implementation of error_container.hpp.
+ */
+
+#include "./tools/schema/error_container.hpp"
+
+namespace schema_generator{
+
+void class_error_container::add_simple_error(const std::string & message){
+	list_.push_back(message);
+}
+
+void class_error_container::add_read_error(const std::string & file,int line){
+	std::ostringstream s;
+	s << file << ":" << line <<": Read error at line "<< line <<".\n";
+	list_.push_back(s.str());
+}
+
+void class_error_container::add_opened_entity_error(
+		const std::string & file,int line,const std::string & name){
+	std::ostringstream s;
+	s << file << ":" << line <<": Entity "<< name
+			<<" is opened but not closed.\n";
+	list_.push_back(s.str());
+}
+
+void class_error_container::add_unopened_entity_error(
+		const std::string & file,int line,const std::string & name){
+	std::ostringstream s;
+	s << file << ":" << line <<": Entity "<< name
+			<<" is being closed  but was not opened.\n";
+	list_.push_back(s.str());
+}
+
+void class_error_container::add_second_parent_error(
+		const std::string & file,int line,const std::string & first,
+		const std::string & second){
+	std::ostringstream s;
+	s << file << ":" << line <<": Parent "<< first <<" is closed due  to parent"
+			<< second << "is opened here. \n";
+	list_.push_back(s.str());
+}
+
+void class_error_container::add_orphan_error(const std::string & file,int line,
+											 const std::string & name){
+	std::ostringstream s;
+	s << file << ":" << line <<": Tag "<< name <<" has no parent \n";
+	list_.push_back(s.str());
+}
+
+void class_error_container::print_errors(std::ostream & s)  const{
+	for (unsigned int i = 0; i< list_.size(); i++){
+		s << list_.at(i);
+	}
+}
+void class_error_container::sort(){
+	std::sort(list_.begin(),list_.end());
+}
+}

src/tools/schema/error_container.hpp

@@ -0,0 +1,113 @@
+/*
+	Copyright (C) 2011 - 2011 by Sytyi Nick <nsytyi@gmail.com>
+	Part of the Battle for Wesnoth Project http://www.wesnoth.org/
+
+	This program is free software; you can redistribute it and/or modify
+	it under the terms of the GNU General Public License as published by
+	the Free Software Foundation; either version 2 of the License, or
+	(at your option) any later version.
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY.
+
+	See the COPYING file for more details.
+*/
+
+/**
+ * @file
+ * This file contains object "error_container", which are used to store error
+ * messages while annotation parsing.
+ * Also the another part of his job is to create GCC:styled error messages.
+ */
+
+#ifndef TOOLS_ERROR_CONTAINER_HPP_INCLUDED
+#define TOOLS_ERROR_CONTAINER_HPP_INCLUDED
+
+#include <algorithm>
+#include <iostream>
+#include <queue>
+#include <sstream>
+#include <string>
+
+namespace schema_generator{
+/**
+ * Container of errors, which are generated while schema generator tool
+ * is parsing source files.
+ * These errors are collected here, to be print on screen after parsing.
+ */
+class class_error_container{
+public:
+	class_error_container(){}
+
+	/**
+	 *Puts simple error message in container
+	 * @param message Message to print.
+	 */
+	void add_simple_error(const std::string & message);
+
+	/**
+	 * Generate and put GCC-style error message about error read file error
+	 * @param file Filename
+	 * @param line Number of line with error
+	 */
+	void add_read_error(const std::string & file,int line);
+
+	/**
+	 * Generate and put GCC-style error message about annotation block
+	 * somebody missed to close
+	 * @param file Filename
+	 * @param line Number of line with error
+	 * @param name Name of open block
+	 */
+	void add_opened_entity_error(const std::string & file,int line,
+								 const std::string & name);
+
+	/**
+	 * Generate and put GCC-style error message about closing block
+	 * that wasn't opened
+	 * @param file Filename
+	 * @param line Number of line with error
+	 * @param name of block
+	 */
+	void add_unopened_entity_error(const std::string & file,int line,
+								   const std::string & name);
+
+	/**
+	 * Generate and put GCC-style error message about opening parent block
+	 * before before closing previous block
+	 * @param file Filename
+	 * @param line Number with error
+	 * @param first Name of open parent
+	 * @param second Name of parent is opening
+	 */
+	void add_second_parent_error(
+			const std::string & file,int line, const std::string & first,
+			const std::string & second);
+
+	/**
+	 * Generate and put GCC-style error message about tag without parent
+	 * @param file Filename
+	 * @param line Number with error
+	 * @param name Name of tag
+	 */
+	void add_orphan_error(const std::string & file,int line,
+						  const std::string & name);
+
+	/**
+	 * Prints errors to output stream
+	 * @param s Output
+	 */
+	void print_errors(std::ostream & s) const;
+	void sort();
+
+	/** Checks, if container is empty.*/
+	bool is_empty() const {
+		return list_.empty();
+	}
+private:
+	/** Container to store error messages.*/
+	std::vector<std::string> list_;
+};
+
+
+}//namespace schema_generator
+#endif // TOOLS_ERROR_CONTAINER_HPP_INCLUDED

src/tools/schema/schema_generator.cpp

@@ -0,0 +1,170 @@
+/*
+  Copyright (C) 2011 - 2011 by Sytyi Nick <nsytyi@gmail.com>
+  Part of the Battle for Wesnoth Project http://www.wesnoth.org/
+
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation; either version 2 of the License, or
+  (at your option) any later version.
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY.
+
+  See the COPYING file for more details.
+*/
+
+/**
+ * @file
+ * This file parses the input parameters, prepares a list of files to be parsed
+ * and calls parser for each of them.
+ *
+ */
+
+#include "tools/schema/sourceparser.hpp"
+
+#include "filesystem.hpp"
+
+#include <iostream>
+#include <fstream>
+#include <string>
+
+#include <string.h>
+
+
+std::string version = "0.6.0";
+
+using namespace schema_generator;
+/**
+ * Parses the command line.
+ * @retval 0 Everything's OK!
+ * @retval 1 Errors found. User decided to exit.
+ * @retval 2 No input files found. Please, check your input directory.
+ * @retval 3 Output file for schema cannot be created.
+ * @retval 4 Output file for regex list cannot be created.
+ */
+int main(int argc, char *argv[]){
+	std::string input_dir = "./src/gui/";
+	std::string output_file = "./data/gui/schema.cfg";
+	std::string regex_file = "./utils/regex_list.txt";
+	for (int arg = 1; arg != argc; ++arg) {
+		const std::string val(argv[arg]);
+		if (val.empty()) {
+			continue;
+		}
+		else if ((val == "--src-dir" || val == "-s") && arg+1 != argc) {
+			input_dir = argv[++arg];
+		}
+		else if ((val == "--output" || val == "-o") && arg+1 != argc) {
+			output_file = argv[++arg];
+		}
+		else if ((val == "--regex" || val == "-r") ) {
+			if (arg+1 != argc){
+				regex_file = argv[++arg];
+			}
+			std::fstream f;
+			f.open(regex_file.c_str(),std::ios::out|std::ios::trunc);
+			if (f.fail()){
+				return 4;
+			}
+			test_regex(f);
+			f.close();
+			return 0;
+		}
+		else if (val == "--help" || val == "-h") {
+			std::cout << "usage: " << argv[0]
+					<< " [-rhV] [-i input_dir] [-o output_file]\n"
+					<< " -r, --regex <regex_file>\t"
+					<< "List of used regexes.\n"
+					<< " -h, --help\t\t\t"
+					<< "Shows this usage message.\n"
+					<< " -s, --src-dir <input_dir>\t"
+					<<"Select the input directory.\n"
+					<< " -o, --output <output_dir>\t"
+					<<"Select the input directory.\n"
+					<< " -V, --version\t"
+					<< "Returns the version of schema generator tool\n";
+			return 0;
+		} else if (val == "--version" || val == "-V") {
+			std::cout << "Battle for Wesnoth schema generator tool, version "
+					<< version << "\n";
+			return 0;
+		}
+	}
+
+
+	if (! file_exists(input_dir)){
+		return 2;
+	}
+
+	std::vector<std::string> files;
+	std::vector<std::string> dirs;
+
+	if (is_directory(input_dir)){
+		get_files_in_dir(input_dir, &files, &dirs, ENTIRE_FILE_PATH);
+
+		if (files.empty() && dirs.empty()){
+			std::cout << "Some problem with input directory "
+					<< input_dir << "\n"
+					<< "No files found\n";
+			return 2;
+		}
+		/** Getting full list of files recursively.*/
+		while (!dirs.empty()){
+			std::string temp_dir = dirs.back();
+			dirs.pop_back();
+			get_files_in_dir(temp_dir, &files, &dirs, ENTIRE_FILE_PATH);
+		}
+	}else{
+		files.push_back(input_dir);
+	}
+	class_source_parser parser;
+	parser.set_output(output_file);
+	std::vector<std::string>::iterator i = files.begin();
+
+
+	for (;i != files.end(); ++i){
+
+		if (file_name((*i)).find(".cpp")!=std::string::npos){
+			parser.set_input((*i));
+			parser.parse_source();
+			continue;
+		}
+		if (file_name((*i)).find(".hpp")!=std::string::npos){
+			parser.set_input((*i));
+			parser.parse_source();
+		}
+	}
+
+	// check errors
+	if (! parser.see_errors().is_empty()) {
+		/**
+	 * Let the user decide whether error are great or just misprints.
+	 * If any error found, waits for a Yy or Nn to continue or not.
+	 */
+		std::cout << "There are some errors\n";
+		parser.see_errors().print_errors(std::cout);
+		// Take user response
+		char c;
+		std::cout << "Continue with errors? Y/N ";
+		while (true) {
+			std::cin.get(c);
+			const char *r = strchr("yYnN",c);
+			if (r == NULL){
+				std::cout << "Please, choose your answer " << std::endl;
+				continue;
+			}
+			if (c == 'n' || c=='N'){
+				std::cout << "I'll take that as a NO" << std::endl;
+				return 1;
+			}
+			if (c == 'y' || c=='Y'){
+				std::cout << "I'll take that as a Yes" << std::endl;
+				break;
+			}
+		}
+	}
+	// save schema information
+	if ( ! parser.save_schema()){
+		return 4;
+	}
+	return 0;
+}

src/tools/schema/sourceparser.cpp

@@ -0,0 +1,501 @@
+/*
+ Copyright (C) 2011 - 2011 by Sytyi Nick <nsytyi@gmail.com>
+ Part of the Battle for Wesnoth Project http://www.wesnoth.org/
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY.
+
+ See the COPYING file for more details.
+*/
+/**
+ * @file
+ * This file contains implementation of sourceparser.cpp.
+ */
+
+#include "./tools/schema/sourceparser.hpp"
+
+#include "boost/regex.hpp"
+
+namespace schema_generator{
+/** Little parts of regex templates used to parse Wml annoations.
+ *For details, look http://wiki.wesnoth.org/WML_Annotation_Format , please
+ */
+/** line is valid*/
+const std::string valid_ = "^\\s*\\*\\s*";
+/** begining of wiki block*/
+const std::string wiki_ ="^\\s*/\\*(?:WIKI|SCHEMA)";
+/** whitespace is possible*/
+const std::string space_ ="\\s*";
+/** sigh "="*/
+const std::string equals_ ="=";
+/** non-mandatory sign "*/
+const std::string quote_ ="\"?";
+/** begining of the block.*/
+const std::string block_begin_="@begin" ;
+/** end of block*/
+const std::string block_end_ ="@end";
+/** allow directive*/
+const std::string allow_="@allow";
+/** sign "{" - curly bracket*/
+const std::string property_open_="\\{";
+/** sign "}" - another curly bracket*/
+const std::string property_close_= "\\}";
+/** type of possible name identificator*/
+const std::string name_type_= "[a-z][a-zA-Z0-9_-]*" ;
+/** type of possible parent indentificator*/
+const std::string parent_type_= "/|(?:[a-z][a-zA-Z0-9_-]*/)+";
+/** type of possible link indentificator*/
+const std::string link_type_="(?:[a-z][a-zA-Z0-9_-]*/)*(?:[a-z][a-zA-Z0-9_-]*)";
+/** template to number regex*/
+const std::string number_="\\d*";
+/** sign "-" - hyphen-minus used to set sign of signed integer*/
+const std::string sign_="-?";
+
+/**
+ * end of line + possible various character before.
+ * Used to close template. ".*" is used cause I dont want to get error
+ * every misprint whitespace
+ * after annotation element
+ */
+const std::string eol_=".*$";
+
+/** Private function to surround an argument with brackets.
+ * This allows substitutions :-)
+ */
+std::string sub (const std::string & s){
+	return "(" + s + ")";
+}
+
+/** Private function to surround argument with not mandatory quotes.
+ * Is used when creating properties
+ */
+std::string quote(const std::string & s){
+	return quote_ + s + quote_ ;
+}
+
+/**
+ * Creates a property template
+ * @param name Name of property
+ * @param value Type of property value
+ * If value is empty creates simple property like {table}
+ * Else creates a named property like {name="[name_type_template]"}
+ */
+std::string property(const std::string & name, const std::string & value = ""){
+	if (value == ""){
+		return property_open_ + name + property_close_;
+	}
+	return property_open_ + name + equals_ + quote(value) + property_close_;
+}
+
+
+const std::string & get_valid() {
+	return valid_;
+}
+
+const std::string & get_wiki() {
+	static std::string wiki = wiki_ + eol_;
+	return wiki;
+}
+
+const std::string & get_parent_begin() {
+	static std::string parent_begin = valid_ + block_begin_
+									  + property("parent")
+									  + property("name",sub(parent_type_))
+									  + eol_;
+	return parent_begin;
+}
+
+const std::string & get_parent_end() {
+	static std::string parent_end = valid_ + block_end_ + property("parent")
+									+ property("name",sub(parent_type_))+ eol_;
+	return parent_end;
+}
+
+const std::string & get_tag_begin() {
+	static std::string tag_begin = valid_ + block_begin_ + property("tag")
+								   + property("name",sub(name_type_))
+								   + property("min",sub(number_))
+								   + property("max",sub(sign_ + number_))
+								   + sub(property("super",sub(link_type_)))
+								   +"?" + eol_;
+	/* sub(property("super"),sub(link_type))+"?"
+	 * property super is not mandatory
+	 */
+	return tag_begin;
+}
+
+
+const std::string & get_tag_end() {
+	static std::string tag_end = valid_ + block_end_ + property("tag")
+								 + property("name",sub(name_type_)) + eol_;
+	return tag_end;
+}
+
+
+const std::string & get_allow_link(){
+	static std::string allow_link = valid_ + allow_ + property("link")
+									+property("name",sub(link_type_)) + eol_;
+	return allow_link;
+}
+
+const std::string & get_allow_global(){
+	static std::string global_link = valid_ + allow_ + property("global")
+									 +property("name",sub(name_type_)) + eol_;
+	return global_link;
+}
+
+
+const std::string & get_table_key_begin() {
+	static std::string keys_begin = valid_ + block_begin_ + property("table")
+									+ property("config")+ eol_;
+	return keys_begin;
+}
+
+const std::string & get_table_end() {
+	static std::string table_end = valid_ + block_end_ + property("table")
+								   + eol_;
+	return table_end;
+}
+
+
+const std::string & get_key_value(){
+	static std::string key_value = valid_ + sub("[a-zA-z][a-zA-Z\\d_+-]*")
+								   + "\\s*&\\s*"+sub("[a-z][a-z_]*")
+								   +"\\s*&\\s?"+ sub(quote("[a-zA-Z._0-9+-]*"))
+								   +"\\s&"+ eol_;
+	return key_value;
+}
+
+void test_regex( std::ostream & f ){
+	f << get_valid() << "\n"
+			<< get_wiki() << "\n"
+			<< get_parent_begin() << "\n"
+			<< get_parent_end() << "\n"
+			<< get_tag_begin() << "\n"
+			<< get_tag_end() << "\n"
+			<< get_allow_link() << "\n"
+			<< get_allow_global() << "\n"
+			<< get_table_key_begin() << "\n"
+			<< get_table_end() << "\n"
+			<< get_key_value() << std::endl;
+}
+
+bool class_source_parser::save_schema(){
+	std::fstream out;
+	if (output_.empty()){
+		return false;
+	}
+	out.open(output_.c_str(),std::ios::out|std::ios::trunc);
+	if (out.fail()){
+		errors_.add_simple_error("Can not open file "+output_+
+								 "\n Output woulfd not be stored\n");
+		return false;
+	}
+	out << "[wml_schema]\n";
+	root_.print(out);
+	out << "[/wml_schema]\n";
+	out.close();
+	return true; // @TODO add error support
+}
+
+
+bool class_source_parser::getline(std::string &s){
+	if (f_.fail()){
+		errors_.add_read_error(input_,line_);
+		return false;
+	}
+	std::getline(f_,s);
+	line_ ++;
+	if (! f_.eof()){
+		if (f_.fail()){
+			errors_.add_read_error(input_,line_);
+			return false;
+		}
+	}
+	return true;
+}
+// call without arg when you want to closethem all
+void class_source_parser::add_open_tag_error(int i = INT_MAX){
+	std::vector<class_tag>::iterator it;
+	for (it = current_.begin(); it != current_.end() && i > 0; ++it){
+		errors_.add_opened_entity_error(input_,line_,it->get_name());
+		--i;
+	}
+}
+// call without arg when you want to closethem all
+void class_source_parser::close_opened_tags(int i = INT_MAX){
+	if (current_.empty()){
+		return;
+	}
+	while (current_.size() > 1){
+		if (i==0){
+			return;
+		}
+		class_tag tag (current_.back());
+		current_.pop_back();
+		current_.back().add_tag(tag);
+		i--;
+	}
+	if (i==0){
+		return;
+	}
+	//adding to parent
+	if (parent_name_.empty()) {
+		orphan_tags_.push_back(current_.back());
+		errors_.add_orphan_error(input_,line_,current_.back().get_name());
+	}else{
+		std::string path = parent_name_;
+		do {
+			path = root_.add_tag(path,current_.back());
+			//std::cout << path;
+		}while (! path.empty());
+	}
+	current_.pop_back();
+}
+
+
+bool class_source_parser::parse_source(){
+	if (input_.empty()){
+		errors_.add_simple_error("File was not defined\n");
+		// Use  to hack sorting errors.
+		// Item with  will be at the end of error list.
+		return false;
+	}
+	f_.open(input_.c_str(),std::ios::in);
+	if (f_.fail()){
+		errors_.add_simple_error("File "+input_ + " can not be opened\n");
+		return false;
+	}
+	line_ = 0;
+	bool result = true;
+	while (!f_.eof()){
+		std::string line;
+		if (! getline(line) ) {
+			close_opened_tags();
+			parent_name_.clear();
+			return false;
+		} // is used to avoid exceptions.
+
+		if (check_wiki(line)) {
+			result = parse_block();
+			if (! result) {
+				break;
+			}
+		}
+	}
+	f_.close();
+	close_opened_tags();
+	parent_name_.clear();
+	return result;
+}
+
+
+bool class_source_parser::parse_block(){
+	while (!f_.eof()){
+		std::string line;
+		if (! getline(line) ) { return false; }
+		if ( check_valid(line)) {
+			if (check_parent_begin(line)) continue;
+
+			if (check_tag_begin(line)){
+				parse_tag();
+				continue;
+			}
+			check_parent_end(line);
+		}
+		else{
+			// wiki-block is closed. checking stack of opened tags
+			if (current_.size() != 0 ){
+				add_open_tag_error();
+				close_opened_tags();
+				// continue working
+			}
+			// block is closed and all tags are closed.
+			return true;
+
+		}
+	}// end while
+	return false;
+}
+
+bool class_source_parser::parse_tag(){
+	while (!f_.eof()){
+		std::string line;
+		if (! getline(line) ) { return false; }
+		if (check_valid(line)) {
+			if (check_tag_begin(line)){
+				parse_tag();
+			}
+			if (check_tag_end(line)){
+				return true;
+			}
+			if (check_keys_begin(line)){
+				parse_keys();
+			}
+			if (check_allow_link(line)){
+			}
+			if (check_allow_global(line)){
+			}
+		}else{
+			if (current_.size() != 0 ){
+				// adding error messages for each unclosed entity
+				add_open_tag_error();
+				close_opened_tags();
+			}
+			return true;
+		}
+	}
+	return true;
+}
+
+
+bool class_source_parser::parse_keys(){
+	std::string line;
+	do{
+		if (! getline(line) ) { return false; }
+		if (! check_valid(line)) {
+			errors_.add_opened_entity_error(input_,line_,"Table config");
+			add_open_tag_error();
+			close_opened_tags();
+			return false;
+		}
+		static const boost::regex value (get_key_value() );
+		boost::smatch sub;
+		bool res = boost::regex_match(line,sub,value);
+		if (res){
+			class_key key;
+			key.set_name(sub[1]);
+			key.set_type(sub[2]);
+			key.set_default(sub[3]);
+			current_.back().add_key(key);
+		}
+	}while (! check_keys_end(line));
+	return true;
+}
+
+
+bool class_source_parser::check_valid(const std::string &s){
+	// s must be like " *" or "*"
+	static const boost::regex valid (get_valid());
+	return boost::regex_search(s,valid);
+}
+
+bool class_source_parser::check_wiki(const std::string& s){
+	boost::regex wiki (get_wiki());
+	return boost::regex_match(s,wiki);
+}
+
+bool class_source_parser::check_tag_begin(const std::string &s){
+	// read tag;
+	static boost::regex tag (get_tag_begin());
+	boost::smatch sub;
+	bool res = boost::regex_match(s,sub,tag);
+	if (res){
+		class_tag new_tag;
+		new_tag.set_name(sub[1]);
+		new_tag.set_min(sub[2]);
+		new_tag.set_max(sub[3]);
+		new_tag.set_super(sub[5]);
+		current_.push_back(new_tag);
+		return true;
+	}
+	return false;
+}
+
+bool class_source_parser::check_tag_end(const std::string &s){
+	static const boost::regex endtag (get_tag_end());
+	boost::smatch sub;
+	bool res = boost::regex_match(s,sub,endtag);
+	if (res){
+		std::string name = sub[1];
+		if (current_.empty()){
+			errors_.add_unopened_entity_error(input_,line_,name);
+			return false;
+		}
+		std::vector<class_tag>::iterator ii = current_.end();
+		int count_opened = 0;
+		do{
+			--ii;
+			if (ii->get_name() == name){
+				add_open_tag_error(count_opened);
+				close_opened_tags(++count_opened);
+				return true;
+			}else{
+				count_opened ++;
+			}
+		}while (ii != current_.begin()) ;
+	}
+	return false;
+}
+
+bool class_source_parser::check_allow_link(const std::string &s){
+	static const boost::regex allow_link (get_allow_link());
+	boost::smatch sub;
+	bool res = boost::regex_match(s,sub,allow_link);
+	if (res){
+		if (!current_.empty()){
+			current_.back().add_link(sub[1]);
+		}
+	}
+	return res;
+}
+
+bool class_source_parser::check_allow_global(const std::string &s){
+	static const boost::regex allow_global (get_allow_global());
+	boost::smatch sub;
+	bool res = boost::regex_match(s,sub,allow_global);
+	if (res){
+		if (!current_.empty()){
+			current_.back().add_link("global/"+sub[1]);
+		}
+	}
+	return res;
+}
+
+bool class_source_parser::check_parent_begin(const std::string &s){
+	static const boost::regex parent (get_parent_begin());
+	boost::smatch sub;
+	bool res = boost::regex_match(s,sub,parent);
+	if (res){
+		std::string name = sub[1];
+		if (!parent_name_.empty()) {
+			errors_.add_second_parent_error(input_,line_,parent_name_,name);
+		}
+		parent_name_ = name;
+	}
+	return res;
+}
+
+bool class_source_parser::check_parent_end(const std::string &s){
+	static const boost::regex parent (get_parent_end());
+	boost::smatch sub;
+	bool res = boost::regex_match(s,sub,parent);
+	if (res){
+		std::string name = sub[1];
+		if (parent_name_ == name) {
+			parent_name_.clear();
+		}else{
+			errors_.add_unopened_entity_error(input_,line_,name);
+		}
+	}
+	return true;
+}
+
+bool class_source_parser::check_keys_begin(const std::string &s){
+	static const boost::regex keys (get_table_key_begin());
+	return boost::regex_match(s,keys);
+
+}
+
+bool class_source_parser::check_keys_end(const std::string &s){
+	static const boost::regex endkeys (get_table_end());
+	bool res = boost::regex_match(s,endkeys);
+	return res;
+}
+
+
+} // namespace schema_generator

src/tools/schema/sourceparser.hpp

@@ -0,0 +1,194 @@
+/*
+   Copyright (C) 2011 - 2011 by Sytyi Nick <nsytyi@gmail.com>
+   Part of the Battle for Wesnoth Project http://www.wesnoth.org/
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2 of the License, or
+   (at your option) any later version.
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY.
+
+   See the COPYING file for more details.
+*/
+/**
+ * @file
+ * This file contains sourceparser object, collecting annotations
+ * and building a tag tree.
+ * Also here are declared fuctions that return regex templates.
+ * And a fuction to create a file with list of templates.
+ */
+
+#ifndef TOOLS_SCHEMA_SOURCEPARSER_HPP_INCLUDED
+#define TOOLS_SCHEMA_SOURCEPARSER_HPP_INCLUDED
+
+#include "./tools/schema/error_container.hpp"
+#include "./tools/schema/tag.hpp"
+
+#include <iostream>
+#include <fstream>
+#include <map>
+#include <queue>
+#include <string>
+#include <vector>
+
+namespace schema_generator{
+/** A few regex templates. Please notice, that adding any regex template,
+ * schould be called in test_regexes()
+ */
+
+/** Template to check line of beeing valid*/
+const std::string & get_valid();
+/** Template to check line is beginnnig of Wiki block*/
+const std::string & get_wiki();
+/** Template to check begining of parent block*/
+const std::string & get_parent_begin() ;
+/** Template to check closing of parent block*/
+const std::string & get_parent_end() ;
+/** Template to check if line contains opening of tag block*/
+const std::string & get_tag_begin() ;
+/** Template to check end of tag block*/
+
+const std::string & get_tag_end() ;
+
+/** Template to check allow{link} block*/
+const std::string & get_allow_link();
+/** Template to check allow{global} block*/
+const std::string & get_allow_global();
+
+/** Template to check begining of table{config} storing key values*/
+const std::string & get_table_key_begin() ;
+/** Template to check if table is closed*/
+const std::string & get_table_end() ;
+
+/** Template to get key value*/
+const std::string & get_key_value();
+/** Writes to the file regex templates list.*/
+void test_regex(std::ostream & f );
+
+class class_source_parser{
+public:
+	class_source_parser():
+			input_ (""),
+			output_(""),
+			root_(class_tag("root",1,1)),
+			parent_name_("")
+	{}
+
+	~class_source_parser(){
+	}
+	void set_input(const std::string &s){
+		input_ = s;
+	}
+	void set_output(const std::string &s){
+		output_ = s;
+	}
+	/**
+	 * Parses file line-by-line, checking every line to open WIKI block
+	 * Please, notice that main input work is made in check_*** methods.
+	 * Methods parse_*** are used to organize alhoritm and set which
+	 * regex template can be used in this context.
+	 * The only exception is parse_keys, where table of keys is read
+	 * and add to top of stack tags.
+	 */
+	bool parse_source();
+	/**
+	 * Saves tag tree to schema file.
+	 */
+	bool save_schema();
+	const std::vector<class_tag> & see_orphans() const{
+		return orphan_tags_;
+	}
+	/** Grants access to error container*/
+	const class_error_container & see_errors() {
+		errors_.sort();
+		return errors_;
+	}
+private:
+	/** name of input file to be parsed*/
+	std::string input_;
+	/** name of output file to print schema*/
+	std::string output_;
+	std::fstream f_;
+	//current line number
+	/** number of current read line. Is used when generating errors*/
+	int line_;
+	//vector-based stack. The element on top is current opened tag.
+	/** Stack of opened tags.*/
+	std::vector<class_tag> current_;
+	/** Root of the schema tree*/
+	class_tag root_;
+	/** Name of current parent*/
+	std::string parent_name_;
+	/** List of tags without parents.*/
+	std::vector<class_tag> orphan_tags_;
+	/** used to store errors*/
+	class_error_container  errors_;
+
+	/**
+	 * Parses WIKI block line-by-line, checking every line
+	 * to open annotation block
+	 */
+	bool parse_block();
+	/**
+	 * Parses lines inside tag block.
+	 * Calls checkers that are allowed in tag block
+	 */
+	bool parse_tag();
+	/**
+	 * Read key table and add keys to tag on the top of the stack.
+	 */
+	bool parse_keys();
+
+	/** check the input line with a template
+	 * check if the line is valid (still in block)
+	 */
+	bool check_valid(const std::string& s);
+	/**
+	 * Gets a line from file and returns it.
+	 * Is used to manage exceptions while IO.
+	 */
+	bool getline(std::string & s);
+	/**
+	 * Сhecks stack of opened tags. If any tags is opened - closes it
+	 *and adds to sublist of next tag in stack.
+	 * Add last tag in stack to parent
+	 * @param i number of tags in stack to close.
+	 */
+	void close_opened_tags(int i);
+	/** Generates errors for each opened tag.
+	 * @param i number of tags in stack to complain.
+	 */
+	void add_open_tag_error(int i);
+
+
+	/**
+	 * Read tag form the line and add it to stack
+	 */
+	bool check_wiki(const std::string& s);
+
+	/** Checks line for tag annotation. Reads tag and puts in into stack.*/
+	bool check_tag_begin(const std::string& s);
+	/**
+	 * Puts closed tag to child list of previous tag.
+	 * Also closes all opened child tags, if they are, and generates warnings.
+	 */
+	bool check_tag_end(const std::string& s);
+
+	/** Opens parrent block*/
+	bool check_parent_begin(const std::string& s);
+	/** Closes parent block*/
+	bool check_parent_end(const std::string& s);
+	/** Checks beginning of keys*/
+	bool check_keys_begin(const std::string&s);
+	/** Checks end of keys*/
+	bool check_keys_end(const std::string&s);
+	/** Checks links*/
+	bool check_allow_link(const std::string & s);
+	/** Checks allowed global tags*/
+	bool check_allow_global(const std::string &s);
+
+};
+} // namespace SchemaGenerator
+
+#endif // TOOLS_SCHEMA_SOURCEPARSER_HPP_INCLUDED

src/tools/schema/tag.cpp

@@ -0,0 +1,127 @@
+/*
+   Copyright (C) 2011 - 2011 by Sytyi Nick <nsytyi@gmail.com>
+   Part of the Battle for Wesnoth Project http://www.wesnoth.org/
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2 of the License, or
+   (at your option) any later version.
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY.
+
+   See the COPYING file for more details.
+*/
+
+/**
+ * @file
+ * Implementation of tag.hpp.
+ */
+
+#include "./tools/schema/tag.hpp"
+
+namespace schema_generator{
+void class_key::print(std::ostream& os,int level) const {
+	std::string s;
+	for (int j=0;j<level;j++){
+		s.append(" ");
+	}
+	os << s << "[key]\n"
+			<< s << "name=\""<< name_ <<"\"\n"
+			<< s << "type=\""<< type_ <<"\"\n";
+	if (is_mandatory()){
+		os << s << "mandatory=\"true\"\n";
+	}else{
+		os << s <<"default="<< default_ <<"\n";
+	}
+	os << s << "[/key]\n";
+}
+
+void class_tag::print(std::ostream& os){
+	printl(os,4,4);
+}
+
+
+void class_tag::printl(std::ostream &os,int level, int step){
+	std::sort (keys_.begin(), keys_.end());
+	std::sort (tags_.begin(), tags_.end());
+	std::sort (links_.begin(), links_.end());
+	std::string s;
+
+	for (int j=0;j<level;j++){
+		s.append(" ");
+	}
+	os << s << "[tag]\n"
+			<< s <<"name=\""<< name_ <<"\"\n"
+			<< s <<"min=\""<< min_ <<"\"\n"
+			<< s <<"max=\""<< max_ <<"\"\n";
+	if (! super_.empty() ){
+		os<< s <<"super=\""<< super_ <<"\"\n";
+	}
+	for (std::vector<class_tag>::iterator i = tags_.begin();
+	i != tags_.end(); ++i ){
+		i->printl(os,level+step,step);
+	}
+	for (std::vector<std::string>::iterator i = links_.begin();
+	i != links_.end(); ++i ){
+		os << s << "" << "[link]\n"
+				<< s << "" << "name=\"" <<*i << "\"\n"
+				<< s << "" << "[/link]\n";
+	}
+	for (std::vector<class_key>::iterator i = keys_.begin();
+	i != keys_.end(); ++i ){
+		i->print(os,level+step);
+	}
+	os<< s << "[/tag]\n";
+}
+
+const std::string  class_tag::add_tag(const std::string &path,
+									  const class_tag &tag){
+	if ( path.empty() || path == "/" ){
+		std::vector<class_tag>::iterator it =
+				std::find(tags_.begin(),tags_.end(),tag);
+		if (it == tags_.end()){
+			tags_.push_back(tag);
+		}else{
+			it->add_tags(tag.tags_);
+			it->set_min(tag.min_);
+			it->set_max(tag.max_);
+			it->add_keys(tag.keys_);
+		}
+		return "";
+	}
+	int pos = path.find('/');
+	std::string name = path.substr(0,pos);
+	std::string next_path = path.substr(++pos,path.length());
+	std::vector<std::string>::iterator it_links= links_.begin();
+	for (;it_links != links_.end();++it_links){
+		std::string::size_type pos_last = it_links->rfind('/');
+		//	if (pos_last > it_links->length()) {
+		//continue;
+		//}
+		std::string name_link = it_links->substr(++pos_last,path.length());
+		if (name_link == name) {
+			break;
+		}
+	}
+	if (it_links != links_.end()){
+		return  *it_links + "/" + next_path;
+	}
+	std::vector<class_tag>::iterator it_tags = tags_.begin();
+	// = std::find(tags_.begin(),tags_.end(),class_tag(name,0,0);
+	for (;it_tags!=tags_.end(); ++it_tags){
+		if ( it_tags->name_ == name ){
+			break;
+		}
+	}
+	if (it_tags == tags_.end()){
+		class_tag subtag;
+		subtag.set_name(name);
+		subtag.add_tag(next_path,tag);
+		tags_.push_back(subtag);
+		return "";
+	}
+	return it_tags->add_tag(next_path,tag);
+}
+
+
+}//namespace schema_generator

src/tools/schema/tag.hpp

@@ -0,0 +1,275 @@
+
+/*
+   Copyright (C) 2011 - 2011 by Sytyi Nick <nsytyi@gmail.com>
+   Part of the Battle for Wesnoth Project http://www.wesnoth.org/
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2 of the License, or
+   (at your option) any later version.
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY.
+
+   See the COPYING file for more details.
+*/
+
+/**
+ * @file
+ * This file contains objects "tag" and "key", which are used to store
+ * information about tags and keys while annotation parsing.
+ */
+
+#ifndef TOOLS_SCHEMA_TAG_HPP_INCLUDED
+#define TOOLS_SCHEMA_TAG_HPP_INCLUDED
+
+#include <algorithm>
+#include <iostream>
+#include <sstream>
+#include <string>
+#include <vector>
+
+
+
+namespace schema_generator{
+/**
+  * class_key is used to save the information about one key.
+  * Key has next info: name, type, default value or key is mandatory.
+  */
+class class_key{
+public:
+	class_key():name_(""),type_(""),default_(""),mandatory_(false)
+	{ }
+	class_key(const std::string & name,
+			  const std::string &type,
+			  const std::string &def,
+			  bool mandatory)
+	{
+		name_ = name;
+		type_ = type;
+		default_ = def;
+		mandatory_ = mandatory;
+	}
+
+	const std::string &  get_name() const{
+		return name_ ;
+	}
+	const std::string &  get_type() const{
+		return type_ ;
+	}
+	const std::string &  get_default() const{
+		return default_;
+	}
+	bool  is_mandatory () const{
+		return mandatory_ ;
+	}
+
+	void  set_name(const std::string& name){
+		name_ = name;
+	}
+	void  set_type(const std::string& type){
+		type_ = type;
+	}
+	void  set_default(const std::string& def){
+		default_ = def;
+		if (def.empty()){
+			mandatory_ = true;
+		}
+	}
+	void  set_mandatory(bool mandatory){
+		mandatory_ = mandatory;
+	}
+	/** is used to print key info
+ * the format is next
+ *  [key]
+ *      name="name"
+ *      type="type"
+ *      default="default"
+ *      mandatory="true/false"
+ *  [/key]
+*/
+	void  print(std::ostream& os,int level) const;
+
+
+	/**
+	 *Compares keys by name. Used in std::sort, i.e.
+	 */
+	bool  operator < ( const class_key& k) const{
+		return (get_name() < k.get_name());
+	}
+private:
+	/** Name of key*/
+	std::string name_;
+	/** Type of key*/
+	std::string type_;
+	/** Default value*/
+	std::string default_;
+	/** Shows, if key is a mandatory key.*/
+	bool mandatory_;
+};
+
+/**
+  * Stores information about tag.
+  * Each tags is an element of great tag tree. This tree is close to filesystem:
+  * you can use links and special include directory global/
+  * Normally root is not mentioned in path.
+  * Each tag has name, minimum and maximum occasions number,
+  * and lists of subtags, keys and links.
+  */
+class class_tag{
+public:
+	class_tag():name_(""),min_(0),max_(0),super_("")
+	{ }
+
+	class_tag(const std::string & name,
+			  int min,
+			  int max
+			  )
+	{
+		name_ = name;
+		min_ = min;
+		max_ = max;
+	}
+	class_tag(const std::string & name,
+			  int min,
+			  int max,
+			  const std::string & super
+			  )
+	{
+		name_ = name;
+		min_ = min;
+		max_ = max;
+		super_ = super;
+	}
+	~class_tag(){  }
+
+	/** Prints information about tag to outputstream, recursively
+	 * is used to print tag info
+	 * the format is next
+	 *  [tag]
+	 *      subtags
+	 *      keys
+	 *      name="name"
+	 *      min="min"
+	 *      max="max"
+	 *  [/tag]
+	*/
+	void print(std::ostream& os);
+
+	const std::string & get_name() const{
+		return name_ ;
+	}
+	int get_min() const{
+		return min_;
+	}
+	int get_max() const{
+		return max_;
+	}
+	const std::string & get_super() const{
+		return super_ ;
+	}
+	bool is_extension() const{
+		return ! super_.empty();
+	}
+
+	void set_name(const std::string& name){
+		name_ = name;
+	}
+	void set_min(int o){
+		min_ = o;
+	}
+	void set_max(int o){
+		max_ = o;
+	}
+	void set_min( std::string const& s){
+		std::istringstream i(s);
+		if (!(i >> min_)){
+			min_ = 0;
+		}
+	}
+	void set_max( std::string const & s){
+		std::istringstream i(s);
+		if (!(i >> max_)){
+			max_ = 0;
+		}
+	}
+	void set_super(std::string const & s){
+		super_= s;
+	}
+	void add_key(const class_key& new_key){
+		keys_.push_back(new_key);
+	}
+	void add_tag(const class_tag& new_tag){
+		tags_.push_back(new_tag);
+	}
+	void add_link(const std::string & link){
+		links_.push_back(link);
+	}
+
+	/**
+	 * Tags are usually organized in tree.
+	 * This fuction helps to add tag to his exact place in tree
+	 * @param path - path in subtree to exact place of tag
+	 * @param tag  - tag to add
+	 * @todo make support of adding to link. Probably by returning exact path,
+	 * i.e we wan add to foo/bar/example, but /foo/bar is a link to bar.
+	 * Probably fuction schould return right path , i.this.e. bar/example
+	 * and caller schould call one's more
+	 * Path is getting shotter and shoter with each call.
+	 * Path schould look like tag1/tag2/parent/ Slash at end is mandatory.
+	 * @return Pointer to name of the full path.
+	 * @retval NULL  add successful.
+	 * @retval Not NULL  you try add a child to a link,
+	 *                   here is a right path, please, use it.
+	 */
+	const std::string  add_tag (const std::string & path,
+								const class_tag & tag);
+
+	bool operator < ( const class_tag& t) const{
+		return name_ < t.name_;
+	}
+	bool operator == (const class_tag & other){
+		return name_ == other.name_;
+	}
+
+private:
+	/** name of tag*/
+	std::string name_;
+	/** number of minimum occasions*/
+	int min_;
+	/** number of maximum occasions*/
+	int max_;
+	/**
+	 * name of tag to extend "super-tag"
+	 * Extension is smth like inheritance and is used in case
+	 * when you need to use another tag with all his
+	 * keys, childs, etc. But you also want to allow extra subtags of that tags,
+	 * so just linking that tag wouldn't help at all.
+	 */
+	std::string super_;
+	/** children tags*/
+	std::vector<class_tag> tags_;
+	/** keys*/
+	std::vector<class_key> keys_;
+	/** links to possible children.
+	 * @todo make support of key section. which takes mandatory links usually.
+	 */
+	std::vector<std::string> links_;
+	/**
+	 * the same as class_tag::print(std::ostream&)
+	 * but indents different levels with step space.
+	 * @param os stream to print
+	 * @param level  current level of indentation
+	 * @param step   step to next indent
+	 */
+	void printl(std::ostream &os,int level, int step);
+
+	void add_tags (const std::vector<class_tag> & list){
+		tags_.insert(tags_.end(),list.begin(),list.end());
+	}
+	void add_keys (const std::vector<class_key> & list){
+		keys_.insert(keys_.end(),list.begin(),list.end());
+	}
+};
+
+}
+#endif // TOOLS_SCHEMA_TAG_HPP_INCLUDED

utils/wiki_grabber.py

@@ -108,7 +108,9 @@ if __name__ == "__main__":
 
         # strip
         data = re.sub(" \*(?: |)", "", data)
-
+        
+        #annotation
+        data = re.sub(r'@(?:begin|end|allow)\{(?:parent|tag|link|global)\}(?:\{.*\})',"",data)
         return data
 
     def get_value(data, key):