refactored everything

got rid of the handlers & custom lists, in favor of standard lists &
filtering methods.
handling requests & responses through the Request object now.
this is still work-in-progress, mainly missing callback capabilities

listall.py

-#!/usr/bin/python
+#!/usr/bin/env python2
 
 from __future__ import print_function
-from teeworlds import Teeworlds
-import sys
 
-# set up stuff
-tw = Teeworlds(timeout=2)
+from teeworlds.teeworlds import Teeworlds
+from teeworlds.server import Server
 
-# ask the masters for servers
-tw.query_masters()
+t = Teeworlds()
+#t.socket._idle_limit = 10
+t.lookup_masters()
+for m in t.masterlist:
+	m.on_server_add = lambda s: s.request_info()
+t.query_masters()
+t.run()
+servers = []
+for m in t.masterlist:
+	servers += m.serverlist
 
-# query servers, wait for responses
-# stops if no packet is received for `timeout` seconds
-tw.run_loop()
-
-servers = tw.serverlist
 
 # sort by ping
 servers.sort(key=lambda s: s.address)
 
 # display a nice list
 for server in servers:
-        print("{server: <64} {address: <15} [{gametype: ^16}] on {master}: {clients: >2}/{max_clients: <2} - {latency: >4.0f} ms" \
-        .format(server=server.name, address=server.address, gametype=server.gametype, master=server.master.name, clients=server.clients, \
-                max_clients=server.max_clients, latency=server.latency*1000))
+	if server.latency:
+		print("{server: <64} {address: <15} [{gametype: ^16}] on {master}: {clients: >2}/{max_clients: <2} - {latency: >4.0f} ms" \
+			.format(server=server.name, address=server.address, gametype=server.gametype, master=server.master.name, clients=server.clients, \
+				max_clients=server.max_clients, latency=server.latency*1000))
\ No newline at end of file

servercount.py

+from teeworlds.teeworlds import Teeworlds
+
+t = Teeworlds()
+t.lookup_masters()
+t.query_masters_servercount()
+#for m in t.masterlist:
+#	m.on_server_add = lambda s: s.request_info()
+#t.query_masters()
+t.run()
+

teeworlds/base.py

+import sys
+import socket
+import select
+import Queue as queue
+from struct import unpack
+from time import time
+from collections import defaultdict
+import logging
+
+logging.basicConfig(format="[%(asctime)s] %(levelname)s: %(funcName)s: %(message)s", level=logging.DEBUG)
+L = logging.getLogger(__name__)
+
+def get_address(host, port=8303, family=0):
+	try:
+		info = socket.getaddrinfo(host, port, family, socket.SOCK_DGRAM)
+		return info[0][4]
+	except socket.gaierror as e:
+		L.warning('getaddrinfo failed: ' + str(e))
+		return None
+
+def is_ipv6(address):
+	if isinstance(address, tuple): address = address[0]
+	# TODO: should be more solid
+	return True if ':' in address else False
+
+def ip_from_data(data):
+	"""takes 6 or 18 bytes of data and extracts IPv4/v6 addresses from it
+	returns a tuple (family, address)
+	"""
+	# ::ffff:0:0/96 == IPv4 mapping
+	if data[0:12] == b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff":
+		data = data[12:18]
+
+	if len(data) == 6:
+		return (socket.inet_ntoa(data[:4]), unpack("!H", data[4:])[0])
+	elif len(data) == 18:
+		address = None
+		port = unpack("!H", data[16:])[0]
+		if sys.platform == "win32":
+			segments = []
+			for (a, b) in (data[:16:2], data[1:16:2]):
+				segments.append("{:x}".format((ord(a)<<8) + ord(b)))
+			address = ':'.join(segments)
+		else:
+			address = socket.inet_ntop(socket.AF_INET6, data[:16])
+		return (address, port)
+	else:
+		raise Exception("Invalid IP data")
+
+
+class Request(object):
+	"""network communication data wrapper"""
+	address = None
+
+	def sent(self):
+		"""callback: request has been processed and sent"""
+		pass
+
+	def response_received(self, data):
+		"""callback: response received
+		return True if more data is expected"""
+		return False
+
+	def get_address(self):
+		"""must return destination address as a tuple (host, port)"""
+		return self.address
+
+	def get_data(self):
+		"""must return data to be sent"""
+		raise NotImplementedError()
+
+	def __str__(self):
+		return "<Request to {} with {} bytes>".format(self.get_address(), len(self.get_data()))
+
+
+class EventSocket(object):
+	"""handles low-level network communication, supports queuing"""
+	def __init__(self, packets_per_second=200, idle_limit=10):
+		self._max_packets_per_second = packets_per_second
+		self._idle_counter = 0
+		self._idle_limit = idle_limit
+		self._packet_rate = 0
+		self._packet_rate_last_update = 0
+		self._sockets = {}
+		# dict of sent requets, so we know where to return received data to
+		self._requests = defaultdict(set)
+		self._queue = queue.Queue()
+		self._sockets[socket.AF_INET] = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.SOL_UDP)
+		self.has_ipv6 = socket.has_ipv6
+		if self.has_ipv6:
+			self._sockets[socket.AF_INET6] = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM, socket.SOL_UDP)
+
+	def _packet_rate_update(self):
+		"""Update the rate limiting counter"""
+		cur_time = time()
+		diff = cur_time - self._packet_rate_last_update
+		self._packet_rate_last_update = cur_time
+		self._packet_rate -= diff*self._max_packets_per_second
+		if self._packet_rate < 0:
+			self._packet_rate = 0
+
+	def _select(self):
+		"""select wrapper raising exception on timeout"""
+		timeout = 1.0/self._max_packets_per_second
+		if self._queue.empty():
+			timeout = 1.0
+		ret = select.select(self._sockets.values(), [], [], timeout)
+		#L.debug("selected: r={} w={} x={}".format(*ret))
+		#if ret == ([], [], []):
+		#	raise socket.timeout('select timed out')
+		#else:
+		#	return ret
+		return ret
+
+	def _send(self, request):
+		"""Actually send request"""
+		socket_type = socket.AF_INET
+		if is_ipv6(request.get_address()):
+			if not self.has_ipv6:
+				raise socket.error("Cannot send IPv6 packet without IPv6 socket")
+			socket_type = socket.AF_INET6
+		self._requests[request.get_address()].add(request)
+		length = self._sockets[socket_type].sendto(request.get_data(), request.get_address())
+		L.debug("Sent {}".format(request))
+		request.sent()
+		self._packet_rate += 1
+		return length
+
+	def send(self, request):
+		"""Queue Request for sending"""
+		assert(isinstance(request, Request))
+		self._queue.put(request)
+
+	def run(self):
+		"""Main loop"""
+		while True:
+			self._packet_rate_update()
+
+			if not self._queue.empty() and self._packet_rate < self._max_packets_per_second:
+				request = self._queue.get()
+				data = request.get_data()
+				try:
+					length = self._send(request)
+					if length != len(data):
+						# TODO: resend?
+						L.warning('Sent {} of {} bytes of {}'.format(length, len(data)))
+				except socket.error as e:
+					if e.errno == 10054: # ICMP port unreachable
+						L.debug('ICMP port unreachable, {} discarded'.format(request))
+					else: raise
+
+			(r, _, _) = self._select()
+
+			# read from all receivable sockets
+			for sock in r:
+				(data, address) = sock.recvfrom(65535)
+				rem = []
+				for request in self._requests[address]:
+					if not request.response_received(data):
+						rem.append(request)
+				if len(self._requests[address]) == 0:
+					L.warning("Nothing sent to {} but received response".format(address))
+				self._requests[address].difference_update(rem)
+
+			if not r and self._queue.empty():
+				self._idle_counter += 1
+				remaining_requests = 0
+				for requestlist in self._requests.values():
+					remaining_requests += len(requestlist)
+				if remaining_requests == 0:
+					L.info("No more outstanding requests, returning")
+					return
+				if self._idle_counter > self._idle_limit:
+					L.info("Idle limit hit, returning")
+					remaining_requests = []
+					for requestlist in self._requests.values():
+						remaining_requests += requestlist
+					L.debug("Remaining sent requests: {}".format(remaining_requests))
+					return
+			else:
+				self._idle_counter = 0

teeworlds/list.py

+import re
+import logging
+logging.basicConfig(format="[%(asctime)s] %(levelname)s: %(funcName)s: %(message)s", level=logging.DEBUG)
+L = logging.getLogger(__name__)
+
+from .server import Server
+from .player import Player
+
+class ServerList(object):
+	def __init__(self):
+		self.servers = []
+
+	def add(self, server):
+		if not isinstance(server, Server):
+			raise Exception('Trying to add non-Server object')
+		self.servers.append(server)
+
+	def find(self, **kwargs):
+		output = ServerList()
+		for server in self.servers:
+			if server.match(**kwargs):
+				output.add(server)
+		return output
+
+	def sort(self, cmp=None, key=None, reverse=False):
+		self.servers.sort(cmp, key, reverse)
+
+	def reverse(self):
+		self.players.reverse()
+
+	def __iter__(self):
+		return iter(self.servers)
+
+	def __repr__(self):
+		return str(self.servers)
+
+
+class PlayerList(object):
+	def __init__(self):
+		self.players = []
+
+	def add(self, player):
+		if not isinstance(player, Player):
+			raise Exception('Trying to add non-Player-object')
+		self.players.append(player)
+
+	def find(self, name=None, clan=None, country=None, playing=None, server=None):
+		output = PlayerList()
+		if name: name = re.compile(name, re.IGNORECASE)
+		if clan: clan = re.compile(clan, re.IGNORECASE)
+		for player in self.players:
+			if (name == None or name.search(player.name)) and \
+				(clan == None or clan.search(player.clan)) and \
+				(country == None or player.country == country) and \
+				(server == None or player.server == server) and \
+				(playing == None or player.playing == playing):
+				output.add(player)
+		return output
+
+	def sort(self, cmp=None, key=None, reverse=False):
+		self.players.sort( cmp, key, reverse)
+
+	def reverse(self):
+		self.players.reverse()
+
+	def __iter__(self):
+		return iter(self.players)
+
+	def __repr__(self):
+		return str(self.players)
\ No newline at end of file

teeworlds/master.py

+import time
+import logging
+logging.basicConfig(format="[%(asctime)s] %(levelname)s: %(funcName)s: %(message)s", level=logging.DEBUG)
+L = logging.getLogger(__name__)
+
+from .base import *
+from .server import *
+
+class MasterServer(object):
+
+	class Count(Request):
+		packet_count_request = 10*b'\xff' + b'cou2'
+		packet_count_response = 10*b'\xff' + b'siz2'
+
+		def __init__(self, address, data_cb):
+			self.address = address
+			self.time_sent = None
+			self.latency = None
+			self.data_cb = data_cb
+
+		def get_data(self):
+			return self.packet_count_request
+
+		def sent(self):
+			self.time_sent = time()
+
+		def response_received(self, data):
+			if len(data) <= len(self.packet_count_response) or not data.startswith(self.packet_count_response):
+				return True # that's not it, wait for more
+			self.latency = time() - self.time_sent
+			L.debug("received count response from {} in {} seconds".format(self.get_address(), self.latency))
+			self.data_cb(data[len(self.packet_count_response):])
+
+	class List(Request):
+		packet_list_request = 10*b'\xff' + b'req2'
+		packet_list_response = 10*b'\xff' + b'lis2'
+		serveraddr_size = 18
+
+		def __init__(self, address, data_cb):
+			self.address = address
+			self.time_sent = None
+			self.latency = None
+			self.data_cb = data_cb
+
+		def get_data(self):
+			return self.packet_list_request
+
+		def sent(self):
+			self.time_sent = time()
+
+		def response_received(self, data):
+			if len(data) <= len(self.packet_list_response) or not data.startswith(self.packet_list_response):
+				return True # that's not it, wait for more
+			self.latency = time() - self.time_sent
+			L.debug("received list response from {} in {} seconds".format(self.get_address(), self.latency))
+			self.data_cb(data[len(self.packet_list_response):])
+			return True # needs more data - eventually
+
+	def __init__(self, socket, address, name=None):
+		self._socket = socket
+		self._address = address
+		self.address = ("[{host}]:{port}" if is_ipv6(address) else "{host}:{port}") \
+			.format(host=address[0], port=address[1])
+		self.name = name or "None"
+		self.latency = None
+		self.serverlist = []
+		self.count = None
+		self._request_count = self.Count(self._address, self._count_response)
+		self._request_list = self.List(self._address, self.add_from_serverlist)
+
+	def request_list(self):
+		self._socket.send(self._request_list)
+
+	def request_count(self):
+		self._socket.send(self._request_count)
+
+	def add_from_serverlist(self, data):
+		"""
+		Add servers from a binary list as received from master servers
+		"""
+		if len(data) % self.List.serveraddr_size != 0:
+			L.warning("Serverlist data length is not a multiple of {}".format(self.List.serveraddr_size))
+		for i in range(0, len(data), self.List.serveraddr_size):
+			server = Server(self._socket, ip_from_data(data[i:i+self.List.serveraddr_size]), master=self)
+			self.serverlist.append(server)
+			self.on_server_add(server)
+
+	def on_server_add(self, server):
+		"""called when a new server is added by add_to_serverlist"""
+		pass
+
+	def _count_response(self, data):
+		self.count = unpack('!H', data[0:2])[0]
+		L.debug("Count: {}".format(self.count))
+
+	def __repr__(self):
+		return "<MasterServer name='{name}' address='{address}' servers='{server_count}'>".format(name=self.name, address=self.address, server_count=len(self.serverlist))

teeworlds/player.py

+import logging
+logging.basicConfig(format="[%(asctime)s] %(levelname)s: %(funcName)s: %(message)s", level=logging.DEBUG)
+L = logging.getLogger(__name__)
+
+class Player(object):
+	def __init__(self):
+		self.name = None
+		self.clan = None
+		self.country = None
+		self.score = None
+		self.server = None
+		self.playing = None
+
+	def __repr__(self):
+		return "<Player name='{name}'>".format(name=self.name)

teeworlds/server.py

+from random import randint
+from time import time
+import re
+import logging
+logging.basicConfig(format="[%(asctime)s] %(levelname)s: %(funcName)s: %(message)s", level=logging.DEBUG)
+L = logging.getLogger(__name__)
+
+from .base import *
+from .player import Player
+
+class Server(object):
+	class Info(Request):
+		packet_request = 10*b'\xff' + b'gie3'
+		packet_response = 10*b'\xff' + b'inf3'
+
+		def __init__(self, address, data_cb):
+			self.address = address
+			self.time_sent = None
+			self.latency = None
+			self.data_cb = data_cb
+			self.token = randint(1,255)
+
+		def get_data(self):
+			return self.packet_request + chr(self.token)
+
+		def sent(self):
+			self.time_sent = time()
+
+		def response_received(self, data):
+			if len(data) <= len(self.packet_response) or not data.startswith(self.packet_response):
+				return True # that's not it, wait for more
+			pieces = data[len(self.packet_response):].split(b'\x00')
+			try:
+				if pieces.pop(0).decode("ascii") != str(self.token):
+					L.warning("Invalid token received, waiting for more data")
+					return True
+			except ValueError:
+				L.warning("Invalid token received, waiting for more data")
+				return True
+			self.latency = time() - self.time_sent
+			self.data_cb(pieces)
+
+	def __init__(self, socket, address, master=None):
+		self._address = address
+		self.address = ("[{host}]:{port}" if is_ipv6(address) else "{host}:{port}") \
+			.format(host=address[0], port=address[1])
+		self._socket = socket
+		self.master = master
+		self.data = None
+		self.reset()
+
+	def reset(self):
+		self._request = self.Info(self._address, self.parse)
+		self.latency = None
+		self.playerlist = []
+		self.version = None
+		self.name = self.address
+		self.map = None
+		self.gametype = None
+		self.password = None
+		self.players = None
+		self.max_players = None
+		self.clients = None
+		self.max_clients = None
+
+	def request_info(self):
+		#L.debug("Requesting {}".format(self.address))
+		self._socket.send(self._request)
+
+	def parse(self, pieces):
+		L.debug("Response received from {}".format(self.address))
+		self.latency = self._request.latency
+		it = iter(pieces)
+		try:
+			self.version = it.next() #.decode('utf8')
+			self.name = it.next() #.decode('utf8')
+			self.map = it.next() #.decode('utf8')
+			self.gametype = it.next() #.decode('utf8')
+			self.password = (it.next()=='1')
+			self.players = int(it.next())
+			self.max_players = int(it.next())
+			self.clients = int(it.next())
+			self.max_clients = int(it.next())
+			for _ in range(self.clients):
+				player = Player()
+				player.name=it.next() #.decode('utf8')
+				player.clan=it.next() #.decode('utf8')
+				player.country = int(it.next())
+				player.score = int(it.next())
+				player.playing = (it.next()=='1')
+				player.server = self
+				self.playerlist.append(player)
+		except StopIteration:
+			L.debug(repr(pieces))
+			#raise
+			self.reset()
+			# TODO: raise?
+			L.warning('unexpected end of data for server {}'.format(self))
+		except Exception:
+			L.debug(repr(pieces))
+			raise
+		self.on_info_received()
+
+	def match(self, **kwargs):
+		if kwargs.has_key("hideInvalid") and kwargs["hideInvalid"] and self.latency == None:
+			return False
+		if kwargs.has_key("_address") and kwargs["_address"] != self._address:
+			return False
+		if kwargs.has_key("_data") and kwargs["_data"][0:len(self.data)] != self.data:
+			return False
+		if kwargs.has_key("name") and not re.search(kwargs["name"], self.name):
+			return False
+		if kwargs.has_key("address") and not re.search(kwargs["address"], self.address):
+			return False
+		if kwargs.has_key("gametype") and not re.search(kwargs["gametype"], self.gametype):
+			return False
+		if kwargs.has_key("maxping") and self.latency > kwargs["maxping"]:
+			return False
+		return True
+
+	def on_info_received(self):
+		pass
+
+	def __repr__(self):
+		return "<Server name='{name}' address='{address}' master='{master}'>".format(**self.__dict__)

teeworlds/teeworlds.py

+#!/usr/bin/env python2
+# coding: utf-8
+#
+#  A library to get the serverlist & information for Teeworlds servers
+#  Copyright (C) 2011  m!nus <m1nus@online.de>
+#
+#  This software is provided 'as-is', without any express or implied
+#  warranty.  In no event will the authors be held liable for any damages
+#  arising from the use of this software.
+#  
+#  Permission is granted to anyone to use this software for any purpose,
+#  including commercial applications, and to alter it and redistribute it
+#  freely, subject to the following restrictions:
+#  
+#  1. The origin of this software must not be misrepresented; you must not
+#     claim that you wrote the original software. If you use this software
+#     in a product, an acknowledgment in the product documentation would be
+#     appreciated but is not required.
+#  2. Altered source versions must be plainly marked as such, and must not be
+#     misrepresented as being the original software.
+#  3. This notice may not be removed or altered from any source distribution.
+
+
+import logging
+logging.basicConfig(format="[%(asctime)s] %(levelname)s: %(funcName)s: %(message)s", level=logging.DEBUG)
+L = logging.getLogger(__name__)
+
+from .base import EventSocket, get_address
+from .master import MasterServer
+
+class Teeworlds(object):
+	def __init__(self):
+		self.masterlist = []
+		self.socket = EventSocket()
+
+	def lookup_masters(self):
+		if self.masterlist:
+			return
+		for mastername in ["master{}.teeworlds.com".format(i) for i in range(1, 4+1)]:
+			# resolves host and picks the first address
+			master_addr = get_address(mastername, port=8300)
+			if master_addr:
+				L.debug("requesting {} ({})".format(mastername, master_addr))
+				master = MasterServer(self.socket, master_addr, mastername.partition(".")[0])
+				self.masterlist.append(master)
+
+	def query_masters(self):
+		for master in self.masterlist:
+			master.request_list()
+
+	def query_masters_servercount(self):
+		for master in self.masterlist:
+			master.request_count()
+
+	def run(self):
+		self.socket.run()