#!/usr/bin/python
# tz_web_server.py
# --copyright-- Copyright 2011 (C) Tranzoa, Co. All rights reserved. Warranty: You're free and on your own here. This code is not necessarily up-to-date or of public quality.
# --url-- http://www.tranzoa.net/tzpython/
# --email-- pycode is the name to send to. tranzoa.com is the place to send to.
# --bodstamps--
# March 23, 2011 bar spun off from poll_server.py
# March 24, 2011 bar comments or whatever
# March 25, 2011 bar get rid of a wrong comment
# April 7, 2011 bar load_file does 'em relative to root_dir
# September 29, 2011 bar return true if favicon is sent out
# get_arg_number()
# October 3, 2011 bar a_client_hit is 'is_new' now
# October 4, 2011 bar client server has verbose option
# load file can get 'em from elsewhere
# October 10, 2011 bar a_client.when
# a_client.ip_adr
# October 13, 2011 bar learn_new_client() and get new_client_hit() back to calling write_new_client()
# October 28, 2011 bar handle favicon completely in generic_hit()
# November 7, 2011 bar HTTP_PORT has moved - change a goofed ref to it
# don't call close_log()
# November 16, 2011 bar client_name_list() and known_ip_adrs[]
# November 19, 2011 bar use our log routine, not hit.server's for client-change logging
# client 'when' in tzlib.elapsed_time, not time.time
# November 20, 2011 bar charset
# November 29, 2011 bar cleanup
# December 16, 2011 bar scary way of getting the charset in send_http_headers()
# February 22, 2012 bar verbose properly set from param
# March 19, 2012 bar gif files
# March 27, 2012 bar put the latest hit time out in the client list
# routine to delete clients (though this didn't look reliable when I originally thought it out)
# May 27, 2012 bar doxygen namespace
# July 10, 2012 bar log the user agent on new hits and next hits
# August 12, 2012 bar name handler threads if needed
# May 28, 2014 bar put thread id in threads
# November 19, 2015 bar able to be given a log routine besides a logger(with .log)
# November 22, 2015 bar uh - bad news if someone didn't give us a logger
# verbize the log output with our name
# let log work for both a tz_simple_logger and for tz_server_logger, we hope
# November 30, 2015 bar sanitize() and sanitize the log msgs
# December 26, 2015 bar compress_for_user_agent()
# January 9, 2018 bar track when the server was last hit
# --eodstamps--
## \file
# \namespace tzpython.tz_web_server
#
#
# This is a generic web server.
# It has code to handle random, 62 bit client numbers, too.
#
# Note: When I spun this code off from poll_server.py, I looked at the Python web server situation out there.
# Bottom line: Tornado ( http://www.tornadoweb.org/ ) looks like they've done pretty much where this code would head if I were to spend time on it.
#
#
import os
import random
import re
import socket
import threading
import time
import urlparse
import SocketServer
SocketServer.TCPServer.request_queue_size = 150 # how many clients can hit us at the same time
import strip_files
import tz_http_server
import tzlib
HTTP_PORT = 12599 # for main program testing
def get_arg_number(args, k, i = 0) :
if k in args :
try :
return(float(args[k][i or 0]))
except ( ValueError, TypeError, IndexError ) :
pass
pass
return(0.0)
macro_start = '{{' # macro start string
no_macro_start = " {_{ " # what to replace macro-starts with
macro_end = '}}' # macro end string
no_macro_end = " }_} " # what to replace macro-ends with
def compress_for_user_agent(s, mime_type = 'text/html') :
"""
Optimize the given string knowing its mime type.
This strips comments and blank lines, for instance.
"""
mime_type = mime_type or 'text/html'
if mime_type.startswith('text/html') :
s = strip_files.strip_angle_string(s)[0]
elif mime_type.startswith('text/css') or mime_type.startswith('application/javascript') :
s = strip_files.strip_curly_string(s)[0]
return(s)
def sanitize(s) :
""" Get dangerous things out of the given string. """
if not isinstance(s, basestring) :
s = str(s)
s = tzlib.best_ascii(s).encode('ascii', 'replace')
s = s.replace(macro_start, no_macro_start)
s = s.replace(macro_end, no_macro_end)
s = s.replace('&', '&')
s = s.replace('<', '<')
s = re.sub(r'\%([0-9a-fA-F]{2})', r'[X \1]', s)
return(s)
def sanitize_args(args) :
""" Get the garbage out of args dictionary. """
for a, v in args.items() :
args[a] = sanitize(v)
return(args)
class a_hit(object) :
def __init__(me, server, httpd, handler, args, is_head) :
me.server = server
me.httpd = httpd
me.handler = handler
me.args = args
me.is_head = is_head
# a_hit
class a_server(object) :
def __init__(me, server = None, http_port = HTTP_PORT, root_dir = None, charset = None, logger = None, log_generic = False, show_exceptions = False, allow_remote_clients = True) :
me.root_dir = root_dir or "."
me.server = server or me
me.http_port = http_port or HTTP_PORT
me.charset = charset or 'utf8'
me.logger = logger
me._log = (callable(me.logger) and me.logger) or getattr(me.logger, 'log', None)
me.log_generic = log_generic
me.a_hit = a_hit
me._stop = False
try :
if allow_remote_clients :
host = ""
else :
host = None # note: this parameter name, 'host', tells what host name the clients can use. If it's empty, then any host name is cool. If it's 'localhost' (that is, in this case, None), then only the local machine can get to the server (at http://localhost or http:127.0.0.1)
me.http_server = tz_http_server.a_threaded_http_server(host = host, port = me.http_port, do_http_args_rtn = me.do_http_args, show_exceptions = show_exceptions)
except ( socket.error, socket.herror, socket.gaierror, socket.timeout ) :
raise ValueError("I cannot create a server on port %u to communicate with the browser!\r\n" % ( me.http_port ))
pass
def set_when(me, t = None) :
me.when = t or tzlib.elapsed_time()
def get_when(me) :
return(me.when)
def get_time_when(me) :
return(me.when + time.time() - tzlib.elapsed_time())
def start(me) :
me.set_when()
me.http_server.start()
def stop(me) :
me._stop = True
me.http_server.stop()
time.sleep(0.1) # should be in tz_http_server to let the real guy shut down (better way?)
def log(me, *args, **kwargs) :
if me._log :
if len(args) :
args = list(args)
li = args.pop(0)
else :
li = kwargs.get('li', "")
me._log(li, *args, **kwargs)
pass
# @staticmethod
def send_http_headers(me, handler, is_head = False, content_type = None, response = 200, hdrs = {}) :
content_type = content_type or ("text/html; charset=%s" % me.charset)
response = response or 200
try :
handler.send_response(response)
handler.send_header("Cache-control", "no-store")
handler.send_header("Content-type", content_type)
for hdr in hdrs.keys() :
handler.send_header(hdr, hdrs[hdr])
handler.end_headers()
except ( socket.error, socket.herror, socket.gaierror, socket.timeout ) : # can happen if he hits the refresh button
is_head = True
me.set_when()
return(not is_head)
def do_http_args(me, httpd, handler, args, is_head = False) :
if getattr(threading.current_thread(), 'name', "Thread-").startswith("Thread-") :
threading.current_thread().name = "tz_web_server.a_server.do_http_args"
threading.current_thread().tid = tzlib.get_tid()
handler.close_connection = 1 # for our purposes, override keep-alives (but it doesn't fix the linux problem)
if me._stop :
handler.send_404(is_head, "Server is stopped")
return(True)
ip_adr = handler.headers.getheader('X-Forwarded-For') # note: if we're not proxied by apache, then this can be spoofed. Maybe even anyway. Gotta check.
if not ip_adr :
ip_adr = handler.client_address[0]
handler.client_address = ( ip_adr, handler.client_address[1] )
handler.path = re.sub(r"/+", "/", handler.path)
uri = urlparse.urlparse(handler.path)
handler.file_only = re.sub(r".*/", "", uri.path.strip(' .\\/'))
handler.file_only_lc = handler.file_only.lower()
hit = me.a_hit(me, httpd, handler, args, is_head)
me.server.pre_process_hit(hit)
r = me.server.do_hit(hit)
me.set_when()
return(r)
def pre_process_hit(me, hit) :
me.set_when()
def do_hit(me, hit) :
me.set_when()
return(False)
def load_file(me, fn, ext, root_dir = None) :
"""
A bit of logic that allows the generic files to be given by extended logic as file names rather than as the file data.
Saves the outsider some typing at the possible expense of reading the files over and over again from the disk (cache).
"""
me.set_when()
fd = ""
if fn :
fn = os.path.normpath(fn)
if root_dir == None :
root_dir = me.root_dir
if not fn.replace("\\", "/").startswith(root_dir.replace("\\", "/")) :
fn = os.path.join(root_dir, fn)
if fn.replace("\\", "/").startswith(root_dir.replace("\\", "/")) :
if fn.lower().endswith(ext) and os.path.isfile(fn) :
try :
fd = tzlib.read_whole_binary_file(fn)
except ( IOError, OSError ) :
fd = "" # don't accidently send the file name to the client when the disk is funny (network drive?)
pass
pass
pass
return(fd)
#
#
# Usual-replace-land starts here
#
#
def favicon_ico(me, hit) :
return("favicon.ico")
def favicon_png(me, hit) :
return("favicon.png")
def favicon_gif(me, hit) :
return("favicon.gif")
def possible_file(me, hit) :
return(os.path.normpath(os.path.join(me.root_dir, hit.handler.file_only)))
# a_server
CI_ARG = 'ci' # FORM hidden input name for the client id
CN_ARG = 'cn' # FORM field for putting in a client name
class a_client(object) :
def __init__(me, hit) :
me.ci = hit.ci
me.name = hit.cn
me.host = None
me.ip_adr = None
me.set_ipadr()
me.set_when()
@staticmethod
def new_cid() :
return((random.randint(1, 0x7fffFFFF) * 0x80000000) + random.randint(1, 0x7fffFFFF))
def set_when(me, t = None) :
me.when = t or tzlib.elapsed_time()
def get_when(me) :
return(me.when)
def get_time_when(me) :
return(me.when + time.time() - tzlib.elapsed_time())
def set_ipadr(me, ip_adr = None) :
ip_adr = str(ip_adr or '')
if ip_adr != me.ip_adr :
a = tz_http_server.get_remote_address_name(ip_adr)
me.host = ''
if len(a) :
me.host = a[0].name
pass
me.ip_adr = ip_adr
# a_client
class a_client_hit(a_hit) :
def __init__(me, server, httpd, handler, args, is_head) :
super(a_client_hit, me).__init__(server, httpd, handler, args, is_head)
me.cl = None
me.ci = ""
me.cn = ""
me.is_new = False
srv = me.server
if srv and hasattr(srv.server, 'clients') :
srv = srv.server # legacy !!!! (unless there's a reason, fix poll_server(_linear) to extend a_server rather than having a_server keep a pointer to the server)
if srv and (CI_ARG in me.args) :
me.ci = srv.clients.valid_client_number(me.args[CI_ARG][0])
if srv and (CN_ARG in me.args) :
me.cn = srv.clients.valid_client_name( me.args[CN_ARG][0])
pass
def update_client(me, cl) :
me.cl = cl
if me.cl :
me.cl.set_when()
me.ci = cl.ci
me.cn = cl.name
# a_client_hit
class a_client_db(dict) :
def __init__(me, *args, **kwargs) :
super(a_client_db, me).__init__(*args, **kwargs)
me.a_client = a_client
def new_client_number(me) :
""" Generate a random, practical-matter-unique string. """
while True :
ci = me.a_client.new_cid()
if str(ci) not in me : break
return(str(ci))
def add_client(me, cl) :
""" Add this client to our memory. """
me[cl.ci] = cl
def delete_client(me, cl) :
""" Get rid of a client from our memory. (Presumably, he's not talked to us in a long while.) """
if cl and (cl.ci in me) :
del(me[cl.ci])
pass
@staticmethod
def valid_client_number(ci) :
""" Protect this string from being dangerous in HTML, at the least. """
try :
ci = str(int(ci))
except ( ValueError, TypeError ) :
ci = ""
return(ci)
@staticmethod
def valid_client_name(cn) :
""" Protect this string from being dangerous in HTML, at the least. """
return(tzlib.printable(cn))
# a_client_db
class a_known_ip_adr(object) :
""" Regx and name for known IP addresses. """
def __init__(me, regx, name) :
me.re = re.compile(regx)
me.name = name
# a_known_ip_adr
class a_client_server(a_server) :
def __init__(me, known_ip_adrs = [], verbose = 0, *args, **kwargs) :
super(a_client_server, me).__init__(*args, **kwargs)
me.verbose = verbose or 0
me.a_hit = a_client_hit
me.clients = a_client_db()
me.known_ip_adrs = known_ip_adrs or [] # make the client list know some IP addresses
def pre_process_hit(me, hit) :
""" Update the hit with known information about the client, and our client data with information from the user. """
# print "@@@@", me.clients, "-----", hit.ci, hit.cn, hit.cn
me.set_when()
if hit.ci in me.clients :
hit.cl = me.clients.get(hit.ci, None) # get the known client
if hit.cl :
hit.cl.set_when()
hit.cl.set_ipadr(hit.handler.client_address[0])
if hit.cn :
if hit.cl.name != hit.cn :
me.log(sanitize('tz_web_server_client_name "%s" was "%s" is "%s"' % ( hit.ci, hit.cl.name, hit.cn )))
hit.cl.name = hit.cn # learn a new name if he gives one, though the boiler plate doesn't allow it as it gives the form entry only if the name is unknown
pass
hit.cn = hit.cl.name # and update the hit, itself, from memory
pass
def do_hit(me, hit) :
me.set_when()
if getattr(threading.current_thread(), 'name', "Thread-").startswith("Thread-") :
threading.current_thread().name = "tz_web_server.a_client_server.do_hit"
threading.current_thread().tid = tzlib.get_tid()
r = me.generic_hit(hit) # handle hits that don't need client information (static images and the like)
if r :
if me.log_generic :
me.log( sanitize("tz_web_server_generic %s [%s] %s" % ( hit.handler.client_address[0], hit.handler.path, str(hit.args) )))
pass
else :
if not hit.ci :
hit.ci = me.clients.new_client_number()
me.log( sanitize("tz_web_server_new %s#%s [%s] %s [%s]" % ( hit.handler.client_address[0], hit.ci, hit.handler.path, str(hit.args), hit.handler.headers.getheader('User-agent'), )))
r = me.new_client_hit(hit)
else :
if me.verbose :
me.log( sanitize("tz_web_server_next %s#%s [%s] %s [%s]" % ( hit.handler.client_address[0], hit.ci, hit.handler.path, str(hit.args), hit.handler.headers.getheader('User-agent'), )))
r = me.client_hit(hit)
pass
if not r :
r = me.final_hit(hit)
return(r)
def generic_hit(me, hit) :
""" Override this to pump out the static resources. """
if hit.handler.file_only_lc == 'favicon.ico' :
fd = me.load_file(me.favicon_ico(hit), ".ico")
if fd :
if hit.handler.send_http_headers(hit.is_head, content_type = "image/x-icon") :
hit.handler.write_data(fd)
return(True)
else :
hit.handler.send_404(hit.is_head, "No favicon.ico")
return(True)
pass
if hit.handler.file_only_lc == 'favicon.png' :
fd = me.load_file(me.favicon_png(hit), ".png")
if fd :
if hit.handler.send_http_headers(hit.is_head, content_type = "image/png") :
hit.handler.write_data(fd)
return(True)
else :
hit.handler.send_404(hit.is_head, "No favicon.png")
return(True)
pass
if hit.handler.file_only_lc == 'favicon.gif' :
fd = me.load_file(me.favicon_gif(hit), ".gif")
if fd :
if hit.handler.send_http_headers(hit.is_head, content_type = "image/gif") :
hit.handler.write_data(fd)
return(True)
else :
hit.handler.send_404(hit.is_head, "No favicon.gif")
return(True)
pass
return(False)
def learn_new_client(me, hit) :
if not hit.ci :
raise IndexError('new_client_hit() without hit.ci - tz_web_server should have called new_client_number()!')
if hit.cl :
raise IndexError('new_client_hit() with hit.cl - pre_process_hit() should only set hit.cl for known clients!')
hit.is_new = True
me.clients.add_client(me.clients.a_client(hit)) # learn a new client (probably without a name at this point - if he hit the generic link, but with a name if he has hit us after a server restart)
if hit.cn :
me.log(sanitize('tz_web_server_client_new "%s" "%s"' % ( hit.ci, hit.cn ) ))
hit.update_client(me.clients[hit.ci])
hit.cl.set_ipadr(hit.handler.client_address[0]) # note: set_when() is done in update_client()
def new_client_hit(me, hit) :
""" Our server thinks this is a new client hitting us for something. """
if hit.ci in me.clients :
return(me.client_hit(hit))
me.learn_new_client(hit)
if hit.server.send_http_headers(hit.handler, is_head = hit.is_head) :
me.write_new_client(hit)
return(True)
def client_hit(me, hit) :
""" Our server thinks this is a known client hitting us for something. """
if hit.ci not in me.clients :
return(me.new_client_hit(hit))
hit.cl.set_when() # note: probably redundant because they are done in pre_process_hit() if the client was known then and it was called
hit.cl.set_ipadr(hit.handler.client_address[0])
if hit.server.send_http_headers(hit.handler, is_head = hit.is_head) :
me.write_hit(hit)
return(True)
def final_hit(me, hit) :
""" The client hasn't been handled. Bail or something. """
hit.handler.send_404(hit.is_head, """Final dummy server hit Path: %s
Args: %s
ci: %s
""" % ( sanitize(str(hit.handler.path)), sanitize(str(hit.args)), sanitize(str(hit.ci)), ) )
return(True)
#
#
# Replace these
#
#
def write_new_client(me, hit) :
""" This is a new client's first hit for anything. Give 'em the main page."""
me.write_hit(hit)
def write_hit(me, hit) :
""" We know this client. """
hit.handler.wfile.write(("""""") % ( sanitize(str(hit.ci)), sanitize(str(hit.ci)), ) )
def client_name_list(me) :
""" Return a list of strings for each client (IP address/name and client ID. """
vals = [ cl for cl in me.clients.values() if cl.ip_adr ]
vals.sort(lambda a, b : cmp(b.get_when(), a.get_when()))
lst = [ str(cl.ip_adr) + ((cl.host and (' ' + cl.host)) or '') + (' %19s' % str(cl.ci)) + ((cl.name and (' ' + cl.name)) or '') + ' ' + time.asctime(time.localtime(cl.get_time_when())) for cl in vals ]
for i in range(len(lst)) :
for a in me.known_ip_adrs :
lst[i] = a.re.sub(r"\1_%s" % a.name, lst[i]) # translate the IP adr to a known name
pass
return(lst)
# a_client_server
if __name__ == '__main__' :
import TZKeyReady
import tz_server_logger
logger = tz_server_logger.a_logger(http_port = HTTP_PORT) # , file_name = __file__ + ".log")
me = a_client_server(http_port = HTTP_PORT, logger = logger, show_exceptions = True) # , allow_remote_clients = False)
me.verbose = me.verbose + 1
me.start()
print "I am listening on %u" % HTTP_PORT
while True :
k = TZKeyReady.key_ready()
if k :
break
time.sleep(0.1)
me.stop()
logger.close()
#
#
#
# eof