#!/usr/bin/python

# tz_web_server.py
#       --copyright--                   Copyright 2011 (C) Tranzoa, Co. All rights reserved.    Warranty: You're free and on your own here. This code is not necessarily up-to-date or of public quality.
#       --url--                         http://www.tranzoa.net/tzpython/
#       --email--                       pycode is the name to send to. tranzoa.com is the place to send to.
#       --bodstamps--
#       March 23, 2011          bar     spun off from poll_server.py
#       March 24, 2011          bar     comments or whatever
#       March 25, 2011          bar     get rid of a wrong comment
#       April 7, 2011           bar     load_file does 'em relative to root_dir
#       September 29, 2011      bar     return true if favicon is sent out
#                                       get_arg_number()
#       October 3, 2011         bar     a_client_hit is 'is_new' now
#       October 4, 2011         bar     client server has verbose option
#                                       load file can get 'em from elsewhere
#       October 10, 2011        bar     a_client.when
#                                       a_client.ip_adr
#       October 13, 2011        bar     learn_new_client() and get new_client_hit() back to calling write_new_client()
#       October 28, 2011        bar     handle favicon completely in generic_hit()
#       November 7, 2011        bar     HTTP_PORT has moved - change a goofed ref to it
#                                       don't call close_log()
#       November 16, 2011       bar     client_name_list() and known_ip_adrs[]
#       November 19, 2011       bar     use our log routine, not hit.server's for client-change logging
#                                       client 'when' in tzlib.elapsed_time, not time.time
#       November 20, 2011       bar     charset
#       November 29, 2011       bar     cleanup
#       December 16, 2011       bar     scary way of getting the charset in send_http_headers()
#       February 22, 2012       bar     verbose properly set from param
#       March 19, 2012          bar     gif files
#       March 27, 2012          bar     put the latest hit time out in the client list
#                                       routine to delete clients (though this didn't look reliable when I originally thought it out)
#       May 27, 2012            bar     doxygen namespace
#       July 10, 2012           bar     log the user agent on new hits and next hits
#       August 12, 2012         bar     name handler threads if needed
#       May 28, 2014            bar     put thread id in threads
#       November 19, 2015       bar     able to be given a log routine besides a logger(with .log)
#       November 22, 2015       bar     uh - bad news if someone didn't give us a logger
#                                       verbize the log output with our name
#                                       let log work for both a tz_simple_logger and for tz_server_logger, we hope
#       November 30, 2015       bar     sanitize() and sanitize the log msgs
#       December 26, 2015       bar     compress_for_user_agent()
#       January 9, 2018         bar     track when the server was last hit
#       --eodstamps--
##      \file
#       \namespace              tzpython.tz_web_server
#
#
#       This is a generic web server.
#       It has code to handle random, 62 bit client numbers, too.
#
#       Note:   When I spun this code off from poll_server.py, I looked at the Python web server situation out there.
#               Bottom line: Tornado ( http://www.tornadoweb.org/ ) looks like they've done pretty much where this code would head if I were to spend time on it.
#
#

import  os
import  random
import  re
import  socket
import  threading
import  time
import  urlparse

import  SocketServer
SocketServer.TCPServer.request_queue_size   = 150           # how many clients can hit us at the same time

import  strip_files
import  tz_http_server
import  tzlib


HTTP_PORT   = 12599         # for main program testing



def get_arg_number(args, k, i = 0) :
    if  k in args :
        try :
            return(float(args[k][i or 0]))
        except ( ValueError, TypeError, IndexError ) :
            pass
        pass
    return(0.0)



macro_start     = '{{'                                              # macro start string
no_macro_start  = " {_{ "                                           # what to replace macro-starts with
macro_end       = '}}'                                              # macro end string
no_macro_end    = " }_} "                                           # what to replace macro-ends with


def compress_for_user_agent(s, mime_type = 'text/html') :
    """
        Optimize the given string knowing its mime type.

        This strips comments and blank lines, for instance.

    """
    mime_type   = mime_type or 'text/html'
    if      mime_type.startswith('text/html') :
        s   = strip_files.strip_angle_string(s)[0]
    elif    mime_type.startswith('text/css') or mime_type.startswith('application/javascript') :
        s   = strip_files.strip_curly_string(s)[0]
    return(s)


def sanitize(s) :
    """ Get dangerous things out of the given string. """
    if  not isinstance(s, basestring) :
        s   = str(s)
    s       = tzlib.best_ascii(s).encode('ascii', 'replace')
    s       = s.replace(macro_start, no_macro_start)
    s       = s.replace(macro_end,   no_macro_end)
    s       = s.replace('&', '&amp;')
    s       = s.replace('<', '&lt;')
    s       = re.sub(r'\%([0-9a-fA-F]{2})', r'[X \1]', s)
    return(s)


def sanitize_args(args) :
    """ Get the garbage out of args dictionary. """
    for a, v in args.items() :
        args[a]     = sanitize(v)
    return(args)


class   a_hit(object) :
    def __init__(me, server, httpd, handler, args, is_head) :
        me.server   = server
        me.httpd    = httpd
        me.handler  = handler
        me.args     = args
        me.is_head  = is_head
    #   a_hit



class   a_server(object)    :

    def __init__(me, server = None, http_port = HTTP_PORT, root_dir = None, charset = None, logger = None, log_generic = False, show_exceptions = False, allow_remote_clients = True) :
        me.root_dir         = root_dir or "."
        me.server           = server            or me
        me.http_port        = http_port         or HTTP_PORT
        me.charset          = charset           or 'utf8'
        me.logger           = logger
        me._log             = (callable(me.logger) and me.logger) or getattr(me.logger, 'log', None)
        me.log_generic      = log_generic

        me.a_hit            = a_hit

        me._stop            = False

        try :
            if  allow_remote_clients :
                host        = ""
            else            :
                host        = None              # note: this parameter name, 'host', tells what host name the clients can use. If it's empty, then any host name is cool. If it's 'localhost' (that is, in this case, None), then only the local machine can get to the server (at http://localhost or http:127.0.0.1)
            me.http_server  = tz_http_server.a_threaded_http_server(host = host, port = me.http_port, do_http_args_rtn = me.do_http_args, show_exceptions = show_exceptions)
        except ( socket.error, socket.herror, socket.gaierror, socket.timeout ) :
            raise ValueError("I cannot create a server on port %u to communicate with the browser!\r\n" % ( me.http_port ))

        pass


    def set_when(me, t  = None) :
        me.when         = t or tzlib.elapsed_time()

    def get_when(me)    :
        return(me.when)

    def get_time_when(me) :
        return(me.when + time.time() - tzlib.elapsed_time())


    def start(me) :
        me.set_when()
        me.http_server.start()


    def stop(me)    :
        me._stop    = True
        me.http_server.stop()
        time.sleep(0.1)         # should be in tz_http_server to let the real guy shut down (better way?)


    def log(me, *args, **kwargs) :
        if  me._log :
            if  len(args) :
                args    = list(args)
                li      = args.pop(0)
            else        :
                li      = kwargs.get('li', "")
            me._log(li, *args, **kwargs)
        pass


    # @staticmethod
    def send_http_headers(me, handler, is_head = False, content_type = None, response = 200, hdrs = {}) :
        content_type    = content_type or ("text/html; charset=%s" % me.charset)
        response        = response or 200
        try :
            handler.send_response(response)
            handler.send_header("Cache-control",    "no-store")
            handler.send_header("Content-type",     content_type)
            for hdr in hdrs.keys() :
                handler.send_header(hdr, hdrs[hdr])
            handler.end_headers()
        except ( socket.error, socket.herror, socket.gaierror, socket.timeout ) :           # can happen if he hits the refresh button
            is_head = True

        me.set_when()

        return(not is_head)


    def do_http_args(me, httpd, handler, args, is_head = False) :
        if  getattr(threading.current_thread(), 'name', "Thread-").startswith("Thread-") :
            threading.current_thread().name = "tz_web_server.a_server.do_http_args"
        threading.current_thread().tid      = tzlib.get_tid()

        handler.close_connection = 1                                                        # for our purposes, override keep-alives (but it doesn't fix the linux problem)

        if  me._stop :
            handler.send_404(is_head, "Server is stopped")

            return(True)

        ip_adr                  = handler.headers.getheader('X-Forwarded-For')              # note: if we're not proxied by apache, then this can be spoofed. Maybe even anyway. Gotta check.
        if  not ip_adr          :
            ip_adr              = handler.client_address[0]
        handler.client_address  = ( ip_adr, handler.client_address[1] )

        handler.path            = re.sub(r"/+", "/", handler.path)

        uri                     = urlparse.urlparse(handler.path)
        handler.file_only       = re.sub(r".*/", "", uri.path.strip(' .\\/'))
        handler.file_only_lc    = handler.file_only.lower()

        hit                     = me.a_hit(me, httpd, handler, args, is_head)
        me.server.pre_process_hit(hit)

        r                       = me.server.do_hit(hit)

        me.set_when()

        return(r)


    def pre_process_hit(me, hit) :
        me.set_when()

    def do_hit(me, hit) :
        me.set_when()
        return(False)


    def load_file(me, fn, ext, root_dir = None) :
        """
            A bit of logic that allows the generic files to be given by extended logic as file names rather than as the file data.
            Saves the outsider some typing at the possible expense of reading the files over and over again from the disk (cache).
        """

        me.set_when()
        fd              = ""
        if  fn          :
            fn          = os.path.normpath(fn)
            if  root_dir   == None :
                root_dir    = me.root_dir
            if  not fn.replace("\\", "/").startswith(root_dir.replace("\\", "/")) :
                fn      = os.path.join(root_dir, fn)
            if      fn.replace("\\", "/").startswith(root_dir.replace("\\", "/")) :
                if  fn.lower().endswith(ext) and os.path.isfile(fn) :
                    try     :
                        fd  = tzlib.read_whole_binary_file(fn)
                    except ( IOError, OSError ) :
                        fd  = ""                                        # don't accidently send the file name to the client when the disk is funny (network drive?)
                    pass
                pass
            pass

        return(fd)


    #
    #
    #   Usual-replace-land starts here
    #
    #
    def favicon_ico(me, hit) :
        return("favicon.ico")
    def favicon_png(me, hit) :
        return("favicon.png")
    def favicon_gif(me, hit) :
        return("favicon.gif")
    def possible_file(me, hit) :
        return(os.path.normpath(os.path.join(me.root_dir, hit.handler.file_only)))

    #   a_server



CI_ARG                      = 'ci'                          # FORM hidden input name for the client id
CN_ARG                      = 'cn'                          # FORM field for putting in a client name


class   a_client(object) :

    def __init__(me, hit) :
        me.ci       = hit.ci
        me.name     = hit.cn
        me.host     = None
        me.ip_adr   = None
        me.set_ipadr()
        me.set_when()

    @staticmethod
    def new_cid() :
        return((random.randint(1, 0x7fffFFFF) * 0x80000000) + random.randint(1, 0x7fffFFFF))

    def set_when(me, t  = None) :
        me.when         = t or tzlib.elapsed_time()

    def get_when(me)    :
        return(me.when)

    def get_time_when(me) :
        return(me.when + time.time() - tzlib.elapsed_time())

    def set_ipadr(me, ip_adr = None) :
        ip_adr      =  str(ip_adr or '')
        if  ip_adr != me.ip_adr :
            a       = tz_http_server.get_remote_address_name(ip_adr)
            me.host = ''
            if  len(a)  :
                me.host = a[0].name
            pass
        me.ip_adr       = ip_adr

    #   a_client



class   a_client_hit(a_hit) :

    def __init__(me, server, httpd, handler, args, is_head) :
        super(a_client_hit, me).__init__(server, httpd, handler, args, is_head)

        me.cl       = None
        me.ci       = ""
        me.cn       = ""
        me.is_new   = False

        srv         = me.server
        if  srv and hasattr(srv.server, 'clients') :
            srv     = srv.server                            # legacy !!!! (unless there's a reason, fix poll_server(_linear) to extend a_server rather than having a_server keep a pointer to the server)

        if  srv and (CI_ARG in me.args) :
            me.ci   = srv.clients.valid_client_number(me.args[CI_ARG][0])
        if  srv and (CN_ARG in me.args) :
            me.cn   = srv.clients.valid_client_name(  me.args[CN_ARG][0])

        pass

    def update_client(me, cl) :
        me.cl       = cl
        if  me.cl   :
            me.cl.set_when()
        me.ci       = cl.ci
        me.cn       = cl.name

    #   a_client_hit




class   a_client_db(dict) :

    def __init__(me, *args, **kwargs) :
        super(a_client_db, me).__init__(*args, **kwargs)
        me.a_client = a_client

    def new_client_number(me)   :
        """ Generate a random, practical-matter-unique string. """

        while True  :
            ci      = me.a_client.new_cid()
            if  str(ci) not in me : break
        return(str(ci))


    def add_client(me, cl) :
        """ Add this client to our memory. """

        me[cl.ci]   = cl


    def delete_client(me, cl) :
        """ Get rid of a client from our memory. (Presumably, he's not talked to us in a long while.) """

        if  cl and (cl.ci in me) :
            del(me[cl.ci])
        pass


    @staticmethod
    def valid_client_number(ci) :
        """ Protect this string from being dangerous in HTML, at the least. """

        try :
            ci  = str(int(ci))
        except ( ValueError, TypeError ) :
            ci  = ""
        return(ci)


    @staticmethod
    def valid_client_name(cn) :
        """ Protect this string from being dangerous in HTML, at the least. """

        return(tzlib.printable(cn))


    #   a_client_db




class   a_known_ip_adr(object) :
    """ Regx and name for known IP addresses. """

    def __init__(me, regx, name) :
        me.re   = re.compile(regx)
        me.name = name

    #   a_known_ip_adr




class   a_client_server(a_server) :

    def __init__(me, known_ip_adrs = [], verbose = 0, *args, **kwargs) :
        super(a_client_server, me).__init__(*args, **kwargs)
        me.verbose          = verbose or 0
        me.a_hit            = a_client_hit
        me.clients          = a_client_db()
        me.known_ip_adrs    = known_ip_adrs or []               # make the client list know some IP addresses


    def pre_process_hit(me, hit) :
        """ Update the hit with known information about the client, and our client data with information from the user. """

        # print "@@@@", me.clients, "-----", hit.ci, hit.cn, hit.cn

        me.set_when()
        if  hit.ci     in me.clients :
            hit.cl      = me.clients.get(hit.ci, None)          # get the known client
        if  hit.cl      :
            hit.cl.set_when()
            hit.cl.set_ipadr(hit.handler.client_address[0])
            if  hit.cn  :
                if  hit.cl.name != hit.cn :
                    me.log(sanitize('tz_web_server_client_name "%s" was "%s" is "%s"' % ( hit.ci, hit.cl.name, hit.cn )))
                    hit.cl.name  = hit.cn                       # learn a new name if he gives one, though the boiler plate doesn't allow it as it gives the form entry only if the name is unknown
                pass
            hit.cn               = hit.cl.name                  # and update the hit, itself, from memory
        pass


    def do_hit(me, hit) :
        me.set_when()
        if  getattr(threading.current_thread(), 'name', "Thread-").startswith("Thread-") :
            threading.current_thread().name = "tz_web_server.a_client_server.do_hit"
        threading.current_thread().tid      = tzlib.get_tid()

        r               = me.generic_hit(hit)                   # handle hits that don't need client information (static images and the like)
        if  r           :
            if  me.log_generic :
                me.log(     sanitize("tz_web_server_generic %s [%s] %s"         % ( hit.handler.client_address[0],         hit.handler.path, str(hit.args) )))
            pass
        else            :
            if  not hit.ci :
                hit.ci  = me.clients.new_client_number()
                me.log(     sanitize("tz_web_server_new     %s#%s [%s] %s [%s]" % ( hit.handler.client_address[0], hit.ci, hit.handler.path, str(hit.args), hit.handler.headers.getheader('User-agent'), )))
                r       = me.new_client_hit(hit)
            else        :
                if  me.verbose :
                    me.log( sanitize("tz_web_server_next    %s#%s [%s] %s [%s]" % ( hit.handler.client_address[0], hit.ci, hit.handler.path, str(hit.args), hit.handler.headers.getheader('User-agent'), )))
                r       = me.client_hit(hit)
            pass
        if  not r       :
            r           = me.final_hit(hit)

        return(r)


    def generic_hit(me, hit) :
        """ Override this to pump out the static resources. """

        if  hit.handler.file_only_lc == 'favicon.ico' :
            fd  = me.load_file(me.favicon_ico(hit), ".ico")
            if  fd  :
                if  hit.handler.send_http_headers(hit.is_head, content_type = "image/x-icon") :
                    hit.handler.write_data(fd)
                return(True)
            else    :
                hit.handler.send_404(hit.is_head, "No favicon.ico")
                return(True)
            pass

        if  hit.handler.file_only_lc == 'favicon.png' :
            fd  = me.load_file(me.favicon_png(hit), ".png")
            if  fd  :
                if  hit.handler.send_http_headers(hit.is_head, content_type = "image/png") :
                    hit.handler.write_data(fd)
                return(True)
            else    :
                hit.handler.send_404(hit.is_head, "No favicon.png")
                return(True)
            pass

        if  hit.handler.file_only_lc == 'favicon.gif' :
            fd  = me.load_file(me.favicon_gif(hit), ".gif")
            if  fd  :
                if  hit.handler.send_http_headers(hit.is_head, content_type = "image/gif") :
                    hit.handler.write_data(fd)
                return(True)
            else    :
                hit.handler.send_404(hit.is_head, "No favicon.gif")
                return(True)
            pass

        return(False)



    def learn_new_client(me, hit) :
        if  not hit.ci  :
            raise IndexError('new_client_hit() without hit.ci - tz_web_server should have called new_client_number()!')

        if  hit.cl :
            raise IndexError('new_client_hit() with hit.cl - pre_process_hit() should only set hit.cl for known clients!')

        hit.is_new          = True
        me.clients.add_client(me.clients.a_client(hit))                 # learn a new client (probably without a name at this point - if he hit the generic link, but with a name if he has hit us after a server restart)
        if  hit.cn :
            me.log(sanitize('tz_web_server_client_new "%s" "%s"' % ( hit.ci, hit.cn ) ))
        hit.update_client(me.clients[hit.ci])
        hit.cl.set_ipadr(hit.handler.client_address[0])                 # note: set_when() is done in update_client()


    def new_client_hit(me, hit) :
        """ Our server thinks this is a new client hitting us for something. """

        if  hit.ci in me.clients :
            return(me.client_hit(hit))

        me.learn_new_client(hit)

        if  hit.server.send_http_headers(hit.handler, is_head = hit.is_head) :
            me.write_new_client(hit)

        return(True)


    def client_hit(me, hit) :
        """ Our server thinks this is a known client hitting us for something. """

        if  hit.ci not in me.clients :
            return(me.new_client_hit(hit))

        hit.cl.set_when()                                               # note: probably redundant because they are done in pre_process_hit() if the client was known then and it was called
        hit.cl.set_ipadr(hit.handler.client_address[0])

        if  hit.server.send_http_headers(hit.handler, is_head = hit.is_head) :
            me.write_hit(hit)

        return(True)


    def final_hit(me, hit) :
        """ The client hasn't been handled. Bail or something. """

        hit.handler.send_404(hit.is_head, """Final dummy server hit <p>Path: %s<p>Args: %s<p>ci: %s<hr>""" % ( sanitize(str(hit.handler.path)), sanitize(str(hit.args)), sanitize(str(hit.ci)), ) )
        return(True)



    #
    #
    #   Replace these
    #
    #
    def write_new_client(me, hit) :
        """ This is a new client's first hit for anything. Give 'em the main page."""
        me.write_hit(hit)

    def write_hit(me, hit) :
        """ We know this client. """

        hit.handler.wfile.write(("""<html><body><script language='javascript'>var ci = '%s';</script><form ="ci_form" id='ci_form' action='ci_form.htm' method='post'><input type='submit' name='ok' value='Click'><input type="hidden" name='""" + CI_ARG + """' value='%s'></form></body></html>""") % ( sanitize(str(hit.ci)), sanitize(str(hit.ci)), ) )



    def client_name_list(me) :
        """ Return a list of strings for each client (IP address/name and client ID. """

        vals    = [ cl for cl in me.clients.values() if cl.ip_adr ]
        vals.sort(lambda a, b : cmp(b.get_when(), a.get_when()))
        lst     = [ str(cl.ip_adr) + ((cl.host and (' ' + cl.host)) or '') + (' %19s' % str(cl.ci)) + ((cl.name and (' ' + cl.name)) or '') + ' ' + time.asctime(time.localtime(cl.get_time_when()))    for cl in vals ]
        for i  in range(len(lst)) :
            for a in me.known_ip_adrs :
                lst[i]  = a.re.sub(r"\1_%s" % a.name, lst[i])           # translate the IP adr to a known name
            pass

        return(lst)


    #   a_client_server



if  __name__ == '__main__' :

    import  TZKeyReady
    import  tz_server_logger


    logger      = tz_server_logger.a_logger(http_port = HTTP_PORT)     # , file_name = __file__ + ".log")
    me          = a_client_server(http_port = HTTP_PORT, logger = logger, show_exceptions = True)   # , allow_remote_clients = False)
    me.verbose  = me.verbose + 1
    me.start()

    print "I am listening on %u" % HTTP_PORT

    while True :
        k   = TZKeyReady.key_ready()
        if  k :
            break
        time.sleep(0.1)

    me.stop()
    logger.close()


#
#
#
# eof
