#!/usr/bin/python

# tz_multi_run.py
#       --copyright--                   Copyright 2010 (C) Tranzoa, Co. All rights reserved.    Warranty: You're free and on your own here. This code is not necessarily up-to-date or of public quality.
#       --url--                         http://www.tranzoa.net/tzpython/
#       --email--                       pycode is the name to send to. tranzoa.com is the place to send to.
#       --bodstamps--
#       December 19, 2010       bar
#       November 29, 2011       bar     pyflake cleanup
#       May 27, 2012            bar     doxygen namespace
#       --eodstamps--
##      \file
#       \namespace              tzpython.tz_multi_run
#
#
#       Run a gob of command line programs using as many CPUs has we have.
#
#

import  sys
import  multiprocessing
import  Queue
import  subprocess
import  threading
import  time

import  tz_os_priority


class   a_cmd_runner(object) :
    def __init__(me, cmd) :
        me.cmd              = cmd

        me.working          = False
        me.done             = False

        me.exit_code        = -1
        me.so               = None
        me.se               = None
        me.n                = -1
        me.nn               = -1

        me.q                = None
        me.p                = None


    def start(me, n)        :
        me.n                = n
        me.q                = multiprocessing.Queue()
        me.p                = multiprocessing.Process(target = me.run, args = ( ( me.cmd, me.n, ) ) )
        me.p.daemon         = True
        me.working          = True
        me.p.start()


    def run(me, cmd, n)     :
        tz_os_priority.set_proc_to_idle_priority(0)
        if  True :
            if  sys.platform   == 'win32' :
                p               = subprocess.Popen(cmd, shell = True, stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
            else                :
                p               = subprocess.Popen(cmd, shell = True, stdin = subprocess.PIPE, stdout = subprocess.PIPE, stderr = subprocess.STDOUT, close_fds = True)
            ( so, se )          = p.communicate()
            so                  = so or ""
            se                  = se or ""
            r                   = p.returncode
        else :
            r                   = 1
            so                  = "Huh?\n"
            se                  = ""
        # print "@@@@", cmd.strip()
        me.q.put([ r, so, se, n ])


    def get_status(me)              :
        if  not me.done             :
            if  me.q != None        :
                try                 :
                    result          = me.q.get_nowait()
                    me.so           = result[1]
                    me.se           = result[2]
                    me.nn           = result[3]

                    me.exit_code    = result[0]

                    me.done         = True
                    me.working      = False
                    me.p.join()
                    q               = me.q
                    me.q            = None
                    if  q           :               # satisfy pyflakes
                        del(q)                      # so we don't run out
                    pass
                except Queue.Empty  :
                    pass
                pass
            pass

        return(me.done)


    #   a_cmd_runner



class   a_herd(object) :

    def __init__(me) :
        me.lock     = threading.RLock()             # so we can be in a thread and have add_runners() be called while we're in run_em()
        me.runners  = []

    def add_runner(me, runner) :
        if  runner :
            me.lock.acquire()
            me.runners.append(runner)
            me.lock.release()
        pass

    def append(me, runner) :
        me.add_runner(runner)

    def run_em(me, done_rtn = None) :
        bcnt    = 0
        ecnt    = 0
        wrka    = []
        while True  :
            for i,  p in enumerate(wrka) :
                if  p.get_status()       :
                    del(wrka[i])
                    ecnt += 1
                    # print "@@@@ endproc", ecnt, len(wrka), repr(so.strip())
                    if  done_rtn :
                        if  not done_rtn(p) :

                            return(ecnt)

                        pass

                    elif    p.exit_code != 0 :

                        return(ecnt)

                    pass
                pass
            pass

            while len(wrka) < multiprocessing.cpu_count() :
                fnd = False
                me.lock.acquire()
                for pi, p in enumerate(me.runners) :
                    if  (not p.done) and (not p.working) :
                        fnd     = True
                        wrka.append(p)
                        bcnt   += 1
                        # print "@@@@ starting proc", bcnt, len(wrka)
                        p.start(pi)
                        break
                    pass
                me.lock.release()
                if  not fnd :
                    break
                pass

            if  not len(wrka) :
                break

            time.sleep(0.1)

        return(ecnt)

    #   a_herd


if __name__ == '__main__' :

    me  = a_herd()

    for  i in range(1000) :
        me.add_runner(a_cmd_runner("echo Hello %d" % (i + 1)))


    def cb(p) :
        print "Done %i r=%d so=[%s] se=[%s]" % ( p.nn + 1, p.exit_code, p.so.strip(), p.se.strip() )
        return(True)

    me.run_em(done_rtn = cb)

#
#
#
# eof