346 lines
9.8 KiB
Python
Executable file
346 lines
9.8 KiB
Python
Executable file
#!/usr/bin/env python2
|
|
from subprocess import Popen, PIPE
|
|
from time import clock, time
|
|
import datetime
|
|
import sqlite3
|
|
import ConfigParser
|
|
import os
|
|
import string
|
|
import random
|
|
import sys
|
|
|
|
# Wording conversations:
|
|
# We have one 'Test'
|
|
# One 'Test' includes multiple 'Executions'
|
|
# One 'Execution' has one or more 'Games'
|
|
|
|
class Test:
|
|
ai_config1 = ''
|
|
ai_config2 = ''
|
|
ai_ident1 = ''
|
|
ai_ident2 = ''
|
|
map = ''
|
|
version_string = ''
|
|
faction1 = ''
|
|
faction2 = ''
|
|
time = ''
|
|
test_id = 0
|
|
title = ''
|
|
game_results = []
|
|
|
|
def __init__(self, _ai_config1, _ai_config2, _faction1, _faction2, _map, _title):
|
|
self.ai_config1 = _ai_config1
|
|
self.ai_config2 = _ai_config2
|
|
self.faction1 = _faction1
|
|
self.faction2 = _faction2
|
|
self.map = _map
|
|
self.title = _title
|
|
|
|
today = datetime.datetime.now()
|
|
self.time = today.strftime('%Y-%m-%d %H:%M')
|
|
|
|
def filter_non_printable(str):
|
|
return ''.join(c for c in str if ord(c) > 31 or ord(c) == 9)
|
|
|
|
def construct_command_line(cfg, test, switched_side):
|
|
wesnoth = cfg.get('default', 'path_to_wesnoth_binary')
|
|
options = cfg.get('default', 'additional_arguments')
|
|
repeats = cfg.getint('default', 'repeat')
|
|
repeats_param = '--multiplayer-repeat ' + str(repeats)
|
|
if repeats > 1:
|
|
print 'Be patient, ' + str(repeats) + ' repeats are going to take a while.'
|
|
|
|
side1 = test.ai_config1 if not switched_side else test.ai_config2
|
|
side2 = test.ai_config2 if not switched_side else test.ai_config1
|
|
faction1 = test.faction1 if not switched_side else test.faction2
|
|
faction2 = test.faction2 if not switched_side else test.faction1
|
|
ai_param1 = '--ai-config 1:' + side1 if side1 else ''
|
|
ai_param2 = '--ai-config 2:' + side2 if side2 else ''
|
|
faction_param1 = '--side 1:"' + faction1 + '"' if faction1 else ''
|
|
faction_param2 = '--side 2:"' + faction2 + '"' if faction2 else ''
|
|
map_param = '--scenario=' + test.map if test.map else ''
|
|
|
|
if len(sys.argv) > 1 and sys.argv[1] == '-p':
|
|
gui = '--debug'
|
|
else:
|
|
gui = '--nogui'
|
|
|
|
statics = '--log-info=ai/testing,mp/connect/engine --multiplayer'
|
|
return (wesnoth + ' ' + options + ' ' + map_param + ' ' + ai_param1 + ' ' + ai_param2 +
|
|
' ' + faction_param1 + ' ' + faction_param2 + ' ' + gui + ' ' + repeats_param + ' ' + statics)
|
|
|
|
def do_filter(str, substring):
|
|
n = str.find(substring)
|
|
if (n > -1):
|
|
temp = str[n + len(substring):].strip()
|
|
c = temp.find(',')
|
|
if (c > -1):
|
|
return n, temp[:c].strip()
|
|
else:
|
|
return n, temp
|
|
return n, ''
|
|
|
|
def run_game(cfg, test, switched_side):
|
|
command_line = construct_command_line(cfg, test, switched_side)
|
|
print 'Running: ' + command_line
|
|
|
|
game_results = []
|
|
game_result = None
|
|
faction1 = ''
|
|
faction2 = ''
|
|
debugout = ''
|
|
|
|
p = Popen(command_line, shell=True, bufsize=10000000, stdout=PIPE, stderr=PIPE)
|
|
|
|
for line in p.stderr:
|
|
l = filter_non_printable(line.strip())
|
|
debugout += l + '\n'
|
|
|
|
n, s = do_filter(l , 'side 1: faction=')
|
|
if (n > -1):
|
|
faction1 = s
|
|
continue
|
|
|
|
n, s = do_filter(l , 'side 2: faction=')
|
|
if (n > -1):
|
|
faction2 = s
|
|
continue
|
|
|
|
n, s = do_filter(l , 'info ai/testing: VERSION:')
|
|
if (n > -1):
|
|
test.version_string = s
|
|
continue
|
|
|
|
n, s = do_filter(l , 'info ai/testing: AI_IDENTIFIER1:')
|
|
if(n > -1):
|
|
if switched_side:
|
|
test.ai_ident2 = s
|
|
else:
|
|
test.ai_ident1 = s
|
|
|
|
# this is the first line of a game.
|
|
# We'll do some initializations here.
|
|
|
|
game_result = {}
|
|
game_result['switched_side'] = switched_side
|
|
game_result['is_success'] = False
|
|
continue
|
|
|
|
n, s = do_filter(l , 'info ai/testing: AI_IDENTIFIER2:')
|
|
if(n > -1):
|
|
if switched_side:
|
|
test.ai_ident1 = s
|
|
else:
|
|
test.ai_ident2 = s
|
|
continue
|
|
|
|
n, s = do_filter(l , 'info ai/testing: WINNER:')
|
|
if (n > -1):
|
|
if (int(s) == 1) != switched_side:
|
|
winner = 1
|
|
else:
|
|
winner = 2
|
|
print 'AND THE WINNER IS: ' + str(winner)
|
|
if game_result.has_key('winner'):
|
|
game_result['is_success'] = False
|
|
break
|
|
game_result['winner'] = winner
|
|
game_result['is_success'] = True
|
|
continue
|
|
|
|
n, s = do_filter(l , 'info ai/testing: DRAW:')
|
|
if(n > -1):
|
|
print 'AND THE WINNER IS: DRAW'
|
|
if game_result.has_key('winner'):
|
|
game_result['is_success'] = False
|
|
break
|
|
game_result['winner'] = 0
|
|
game_result['is_success'] = True
|
|
continue
|
|
|
|
n, s = do_filter(l , 'info ai/testing: GAME_END_TURN:')
|
|
if (n > -1):
|
|
# this is the last line printed in a game
|
|
# so we do some checking here and adding
|
|
# game_result to game_results.
|
|
|
|
print 'AND THE VICTORY_TURN IS: ' + s
|
|
|
|
if game_result.has_key('end_turn'):
|
|
game_result['is_success'] = False
|
|
break
|
|
|
|
game_result['end_turn'] = int(s)
|
|
game_result['faction1'] = faction1 if not switched_side else faction2
|
|
game_result['faction2'] = faction2 if not switched_side else faction1
|
|
|
|
if not game_result['is_success']:
|
|
print_error(debugout)
|
|
|
|
game_results.append(game_result)
|
|
continue
|
|
|
|
n, s = do_filter(l , 'error')
|
|
if(n > -1):
|
|
# forward errors from stderr.
|
|
print 'stderr give: error ' + s
|
|
continue
|
|
|
|
|
|
if game_result is None or not game_result['is_success']:
|
|
print_error(debugout)
|
|
return game_results
|
|
|
|
def print_error(debugout):
|
|
print 'Warning: not success!'
|
|
print '===================='
|
|
print 'stderr:'
|
|
print debugout
|
|
print '===================='
|
|
|
|
def save_result_logfile(cfg, test, game_result, log_file):
|
|
print 'Saving to log file....'
|
|
log_file.write('"' + test.ai_config1 + '", "' +
|
|
test.ai_config2 + '", "' +
|
|
test.ai_ident1 + '", "' +
|
|
test.ai_ident2 + '", "' +
|
|
str(game_result['switched_side']) + '", "' +
|
|
game_result['faction1'] + '", "' +
|
|
game_result['faction2'] + '", "' +
|
|
str(game_result['is_success']) + '", "' +
|
|
test.map + '", "' +
|
|
str(game_result['end_turn']) + '", "' +
|
|
str(test.version_string) + '", "' +
|
|
str(game_result['winner']) + '"\n');
|
|
|
|
log_file.flush();
|
|
print 'Saved to log file'
|
|
|
|
def save_result_database(cfg, test, game_result, sqlite_file):
|
|
print 'Saving to DB....'
|
|
query = ('INSERT INTO games("test_id","faction1","faction2","switched_side","is_success","end_turn","winner")' +
|
|
'VALUES (?,?,?,?,?,?,?)')
|
|
|
|
conn = sqlite3.connect(sqlite_file)
|
|
cur = conn.cursor()
|
|
cur.execute(query, (
|
|
test.test_id,
|
|
game_result['faction1'],
|
|
game_result['faction2'],
|
|
game_result['switched_side'],
|
|
game_result['is_success'],
|
|
game_result['end_turn'],
|
|
game_result['winner']))
|
|
conn.commit()
|
|
conn.close()
|
|
print 'Saved to DB'
|
|
|
|
def executions(cfg, test):
|
|
structured = cfg.getboolean('default', 'structured_test')
|
|
if structured:
|
|
factions = ['Drakes', 'Rebels', 'Undead', 'Northerners', 'Knalgan Alliance', 'Loyalists']
|
|
i = 1
|
|
for faction1 in factions:
|
|
for faction2 in factions:
|
|
print 'EXECUTION: ' + str(i) + '/36 --- ' + faction1 + ' against ' + faction2
|
|
test.faction1 = faction1
|
|
test.faction2 = faction2
|
|
game_results = run_game(cfg, test, False)
|
|
yield game_results
|
|
i = i + 1
|
|
test.faction1 = 'structured'
|
|
test.faction2 = 'structured'
|
|
else:
|
|
n = cfg.getint('default', 'number_of_tests')
|
|
randomize = cfg.getboolean('default', 'randomize_sides')
|
|
random.seed()
|
|
|
|
for i in range(0, n):
|
|
switched_side = (random.randint(0, 1) == 1) if randomize else False
|
|
print 'EXECUTION ' + str(i + 1)
|
|
game_results = run_game(cfg, test, switched_side)
|
|
yield game_results
|
|
|
|
if test.faction1 == '':
|
|
test.faction1 = 'random'
|
|
if test.faction2 == '':
|
|
test.faction2 = 'random'
|
|
|
|
# main
|
|
|
|
cfg = ConfigParser.ConfigParser()
|
|
cfg.read('ai_test.cfg')
|
|
|
|
ai1 = cfg.get('default', 'ai_config1').strip()
|
|
ai2 = cfg.get('default', 'ai_config2').strip()
|
|
faction1 = cfg.get('default', 'faction1').strip()
|
|
faction2 = cfg.get('default', 'faction2').strip()
|
|
map = cfg.get('default', 'map').strip()
|
|
if cfg.has_option('default', 'title'):
|
|
title = cfg.get('default', 'title')
|
|
else:
|
|
title = ''
|
|
|
|
test = Test(ai1, ai2, faction1, faction2, map, title)
|
|
|
|
# only 'test the test' with GUI / start one game then exit
|
|
if len(sys.argv) > 1 and sys.argv[1] == '-p':
|
|
executions(cfg, test).next()
|
|
sys.exit(0)
|
|
|
|
log_file = None
|
|
if cfg.has_option('default', 'log_file'):
|
|
log_file = open(datetime.datetime.now().strftime(cfg.get('default', 'log_file').strip()) , 'w')
|
|
log_file.write('"ai_config1", "ai_config2", "ai_ident1", "ai_ident2", "switched_side", ' +
|
|
'"faction1", "faction2", "is_success", "local_modifications", ' +
|
|
'"map", "repo_release", "end_turn", "version_string", "winner_side"\n');
|
|
log_file.flush();
|
|
sqlite_file = None
|
|
if cfg.has_option('default', 'sqlite_file'):
|
|
sqlite_file = cfg.get('default', 'sqlite_file')
|
|
conn = sqlite3.connect(sqlite_file)
|
|
cur = conn.cursor()
|
|
cur.execute('INSERT INTO tests ("title","ai_config1","ai_config2","map","time") ' +
|
|
'VALUES ("' + test.title + '","' + test.ai_config1 + '","' + test.ai_config2 + '","' +
|
|
test.map + '","' + test.time + '")')
|
|
test.test_id = cur.lastrowid
|
|
conn.commit()
|
|
conn.close();
|
|
|
|
# the following variables are for generating a print output only
|
|
total = 0
|
|
ai1_won = 0
|
|
ai2_won = 0
|
|
draw = 0
|
|
|
|
for game_results in executions(cfg, test):
|
|
for game_result in game_results:
|
|
if log_file:
|
|
save_result_logfile(cfg, test, game_result, log_file)
|
|
if sqlite_file:
|
|
save_result_database(cfg, test, game_result, sqlite_file)
|
|
|
|
total = total + 1
|
|
if game_result['winner'] == 0:
|
|
draw = draw + 1
|
|
elif game_result['winner'] == 1:
|
|
ai1_won = ai1_won + 1
|
|
elif game_result['winner'] == 2:
|
|
ai2_won = ai2_won + 1
|
|
print '\n=====Status====='
|
|
print 'Total games: ' + str(total)
|
|
print 'AI1(' + ai1 + ') won: ' + str(ai1_won) + "/" + str(ai1_won * 100 / total) + '%'
|
|
print 'AI2(' + ai2 + ') won: ' + str(ai2_won) + "/" + str(ai2_won * 100 / total) + '%'
|
|
print 'Draws: ' + str(draw) + "/" + str(draw * 100 / total) + '%\n'
|
|
if sqlite_file:
|
|
conn = sqlite3.connect(sqlite_file)
|
|
cur = conn.cursor()
|
|
query = "UPDATE tests SET ai_ident1 = ?, ai_ident2 = ?, version = ? , faction1 = ?, faction2 = ? WHERE id = ?;"
|
|
cur.execute(query, (test.ai_ident1,
|
|
test.ai_ident2,
|
|
test.version_string,
|
|
test.faction1,
|
|
test.faction2,
|
|
test.test_id))
|
|
conn.commit()
|
|
conn.close();
|