#!/usr/bin/python

# portfolio_track.py
#       --copyright--                   Copyright 2007 (C) Tranzoa, Co. All rights reserved.    Warranty: You're free and on your own here. This code is not necessarily up-to-date or of public quality.
#       --url--                         http://www.tranzoa.net/tzpython/
#       --email--                       pycode is the name to send to. tranzoa.com is the place to send to.
#       --bodstamps--
#       February 18, 2006       bar
#       February 19, 2006       bar     percentages
#       February 27, 2006       bar     change a column heading
#       March 7, 2006           bar     get DIV working
#                                       print em in symbol alpha order
#       March 22, 2006          bar     use tzpython version of get_yahoo_historical_csv
#                               bar     move to tzpython
#       March 23, 2006          bar     private/public info printout
#                                       allow any symbol for the market_symbol
#       April 8, 2006           bar     FEE (untested) and --html
#       April 10, 2006          bar     more flexible dates allowed
#       April 15, 2006          bar     --after
#       April 21, 2006          bar     expose parse_date and parse_order
#       April 21, 2006          bar     ++show_info
#       April 23, 2006          bar     show percentage of portfolio
#                                       put the dash if the market was down, but not flat
#                                       --sort
#       April 24, 2006          bar     MBUY op
#       April 27, 2006          bar     use pow() function instead of multiply to normalize to a year
#                                       more date parsing
#       April 28, 2006          bar     more robust in face of bad command line args
#                                       change MBUY to require money amount (so small dividends can be reinvested)
#                                       DRIP
#       April 29, 2006          bar     put out dollar days and the alternate way of computing the percent per year gain
#                               bar     test for all raise's
#       July 12, 2006           bar     CASH SPEND INTerest
#       September 7, 2006       bar     print the printout time
#       February 27, 2007       bar     SPLIT
#       February 28, 2007       bar     sort the orders by date
#                                       get split going
#                                       DRIPS
#       April 23, 2007          bar     print when the closing date is
#       April 26, 2007          bar     cmd line help "--after"
#       June 7, 2007            bar     random tracking portfolios
#       November 18, 2007       bar     turn on doxygen
#       November 27, 2007       bar     insert boilerplate copyright
#       May 17, 2008            bar     email adr
#       September 3, 2008       bar     python 2.5 can't do raise string
#       May 25, 2010            bar     tz_os_priority
#       November 29, 2011       bar     pyflake cleanup
#       May 27, 2012            bar     doxygen namespace
#       November 5, 2015        bar     remove unneeded call to strip_comment_fields()
#                                       fix, but did not test, the --buy and --sell command line args
#       --eodstamps--
##      \file
#       \namespace              tzpython.portfolio_track
#
#
#       Given a text (file) with BUY and SELL (etc) records, track how well it's doing - specifically against a one of the "market" numbers. DJ, S&P, NASDAQ, etc.
#
#       Notes:
#
#           YrDollar sum isn't right when displayed because of round-offs in the stock displays.
#
#           Why is the old-computed 1YrGain not the same as the YrDlrG for both the market and the portfolio? !!!!
#
#           Why does the random tracker 50th percentile (using my history) come out 2% above the market? !!!!
#               Note: To check for a "random" stock always being the market, the key for symdata must be uniquified.
#               The cumulative effect because of the power function being applied to the money if it's only been invested for a part of a year?
#                   Test with a single stock, one-year transaction history.
#                   Comes out after a *lot* of runs to be relatively ok. See rp1.cfg
#               Survivor bias?
#                   Not likely. A one year run with random_portfolio_results.py doesn't have that happen. In fact, after the non-survivors were stripped, it lost 2% or so. Interesting, that.
#               Well timed transactions?
#                   Not likely, considering that they are being compared to the market being bought at the same time.
#
#

import  copy
import  math
import  random
import  re
import  time

import  get_yahoo_historical_csv
import  tgcmsg


#                               e.g. 30-Jan-2002 or 30-Jan-02
dt_re           =   re.compile(r"(0\d|1\d|2\d|30|31|\d)-([A-Z]{3,})-(\d{2,4})", re.IGNORECASE)
#                               e.g. 1/30/2002   or 1/30/02
dt_re_dash      =   re.compile(r"(0\d|10|11|12|\d)/(0\d|1\d|2\d|30|31|\d)/(\d{2,4})")
#                               e.g. 1-30-2002   or 1-30-02
dt_re_slash     =   re.compile(r"(0\d|10|11|12|\d)-(0\d|1\d|2\d|30|31|\d)-(\d{2,4})")
#                               e.g. Jan-30-2002 or Jan-30-02
dt_re_nm_dash   =   re.compile(r"([A-Z]{3,})/(0\d|1\d|2\d|30|31|\d)/(\d{2,4})", re.IGNORECASE)
#                               e.g. Jan/30/2002 or Jan/30/02
dt_re_nm_slash  =   re.compile(r"([A-Z]{3,})-(0\d|1\d|2\d|30|31|\d)-(\d{2,4})", re.IGNORECASE)



TOTALS_SYMBOL   = "____"                    # used to store the processed totals

CASH_SYMBOL     = "--CASH--"                # used to mark the cash




def parse_date(s) :
    s               = s.strip()
    g               = dt_re.match(s)
    if  not g : g   = dt_re_dash.match(s)
    if  not g : g   = dt_re_slash.match(s)
    if  not g : g   = dt_re_nm_dash.match(s)
    if  not g : g   = dt_re_nm_slash.match(s)

    if  not g :
        es  = "Bad date: " + s
        raise ValueError(es)

    d   = g.group(1)
    m   = g.group(2)
    if  not m.isdigit() :
        d   = int(d)
    else :
        d   = int(m)
        m   = g.group(1)

    if  not m.isdigit() :
        if  not get_yahoo_historical_csv.a_csv.month_names_to_num.has_key(m.lower()[0:3]) :
            s   = "No such month: " + m
            raise ValueError(s)
        m   = get_yahoo_historical_csv.a_csv.month_names_to_num[m.lower()[0:3]]
    else :
        m   = int(m)

    y   = int(g.group(3))
    if  y < 28 :  y += 100
    if  y < 1900 :
        y +=  1900

    if  (y < 1900) or (y > 2040) :
        es  = "Bad year: " + s
        raise ValueError(es)

    return( [ y, m, d ] )


def _get_msg_sym(msg) :
    sym     = ""
    if  len(msg) >= 2 :
        sym = get_yahoo_historical_csv.clean_symbol(msg[1])
    if  not sym :
        raise   ValueError("No symbol: " + msg[0] + " " + msg[1])

    return(sym)


def parse_order(msg) :

    msg     = copy.copy(msg)

    msg[0]  = msg[0].upper()


    sym     = ""

    if  (msg[0] == "BUY") or (msg[0] == "SELL") :
        if  len(msg) != 5 :
            s   = "BUY/SELL transaction needs symbol/share_count/price/date: " + str(msg)
            raise   ValueError(s)

        sym     = _get_msg_sym(msg)
        msg[2]  = float(msg[2])
        msg[3]  = float(msg[3])

    elif msg[0] == "DRIP" :
        if  len(msg) != 5 :
            s   = "DRIP transaction needs symbol/share_count/price/date: " + str(msg)
            raise   ValueError(s)

        sym     = _get_msg_sym(msg)
        msg[2]  = float(msg[2])
        msg[3]  = float(msg[3])

    elif msg[0].startswith("DIV") or msg[0].startswith("FEE") or msg[0].startswith("MBUY") :
        if  len(msg) != 4 :
            s   = "DIVIDEND/FEE/MBUY transaction needs symbol/amount/date: " + str(msg)
            raise   ValueError(s)

        sym     = _get_msg_sym(msg)
        msg[2]  = float(msg[2])

    elif msg[0].startswith("ALLS") :
        if  len(msg) != 3 :
            s   = "ALLSELL transaction needs symbol/date: " + str(msg)
            raise   ValueError(s)
        sym     = _get_msg_sym(msg)

    elif msg[0].startswith("CASH") or msg[0].startswith("SPEND") or msg[0].startswith("INT") :
        if  len(msg) != 3 :
            s   = "CASH/SPEND needs amount/date: " + str(msg)
            raise   ValueError(s)
        sym     = CASH_SYMBOL
        msg.insert(1, sym)
        msg[2]  = float(msg[2])

    elif msg[0].startswith("SPLIT") :
        if  len(msg) != 5 :
            s   = "SPLIT needs from/to: " + str(msg)
            raise   ValueError(s)
        sym     =  _get_msg_sym(msg)
        msg[2]  = float(msg[2])
        msg[3]  = float(msg[3])

    else :
        s       = "Order is not BUY, SELL, FEE, MBUY, ALLSELL, or DIV(IDEND): " + str(msg)
        raise   ValueError(s)

    msg[1]      = sym
    msg[-1]     = parse_date(msg[-1])

    return(msg)                         # op symbol ...




def find_price(sym, when, csv_dir, hit_web) :
    refresh = False
    if  not hit_web :
        refresh = None

    p   =   get_yahoo_historical_csv.find_price(sym, when, csv_dir, refresh)
    if  p == None :
        s = "Cannot find price of %s on %s" % ( sym, get_yahoo_historical_csv.make_date_str(when) )
        raise   KeyError(s)

    return(p)



def find_date_vals(sym, when, csv_dir, hit_web) :
    refresh     = False
    if  not hit_web :
        refresh = None

    ymd_array   =   get_yahoo_historical_csv.find_date_vals(sym, when, csv_dir, refresh)

    return(ymd_array)



def yearly_percentage(value, yr_dollars) :
    if  yr_dollars != 0.0 :
        pc  =   value * 100.0 / yr_dollars
    else :
        pc  =   0.0

    return(pc)


def market_adjusted_yearly_percentage(value, yr_dollars, mkt_val) :

    mpc = 0.0

    if  yr_dollars != 0.0 :
        if  mkt_val != None :
            mpc = yearly_percentage(value - mkt_val, yr_dollars)
        pass

    return(mpc)







class a_transaction :

    def __init__(me, shares, price, when) :

        me.shares   = shares
        me.price    = price
        me.when     = when

    pass


class a_stock :

    def __init__(me, sym, csv_dir, hit_web) :

        me.sym          = sym
        me.shares       = 0.0                                   # how many shares are owned right now
        me.value        = 0.0                                   # how many dollars are sunk in to the stock (buying raises the value, selling lowers it - a closed stock position with a negative 'value' is good)
        me.yr_dollars   = 0.0                                   # how many dollar-years were put up for the stock ownership (for instance, $500 for a half year is the same as $250 for a whole year)
        me.transactions = []                                    # record of buys, so that sells can be applied to a reasonable buys

        me.real_shares  = 0.0                                   # the number of shares in the real company in case there is a random substitute (so that the proper number of shares can be sold from the random substitute company on a sell order)

        me.prev_shares  = 0.0                                   # how many shares there were before the most recent trade

        me.csv_dir      = csv_dir
        me.hit_web      = hit_web

        me.of_portfolio = 0.0                                   # set by process_portfolio_history()
        me.yr_percent   = None                                  # set by process_portfolio_history()

        me.dollar_days  = 0.0                                   # e.g. if $100 is bought and held for 1 day, then this value is 100. If held for 2 days, then this value would be 200
        me.dollar_years = None                                  # set by process_portfolio_history()
        me.dy_percent   = None                                  # set by process_portfolio_history()

        me.today_vals   = None                                  # set by process_portfolio_history() as the ymd_array that everything is "sold" - today. Actually, it contains more. All that day's numbers.




    def buy(me, shares, price, when) :
        me.prev_shares  = me.shares
        me.shares      += shares
        p               = shares * price
        me.value       += p

        t               = a_transaction(shares, price, when)
        me.transactions.append(t)

        return(p)


    def dividend(me, how_much, when) :
        me.value       -= how_much                              # subtract, because 'value' is actually how much we paid for it.  !!!! for now, we'll ignore when it happened and not worry about interest or whatever


    def sell(me, shares, price, when) :
        if  shares > me.shares :
            s  = "Too many shares sold of %s: %f is more than %f" % ( me.sym, shares, me.shares )
            if  shares <= me.shares + 0.000000001 :
                shares      = me.shares
            else :
                raise   ValueError(s)
            pass

        mr              = 0.0
        if  me.shares  != 0.0 :
            mr              = shares / me.shares

        me.prev_shares  = me.shares
        me.shares      -= shares
        me.value       -= shares * price

        while shares   >  0 :
            if  len(me.transactions) == 0 :
                if  shares > 0.000000001 :
                    s = "Remaining shares of ", me.sym, shares
                    raise   ValueError(s)
                break;
            txidx       = 0
            tx          = me.transactions[txidx]                    # apply the sell to the oldest buy

            cnt         = min(tx.shares, shares)
            shares     -= cnt

            dd          = get_yahoo_historical_csv.unix_date(when) - get_yahoo_historical_csv.unix_date(tx.when)
            if  dd     <  0 :
                s       =   "Sell date before buy date: " + me.sym + " " + str(when) + str(tx.when)
                raise   ValueError(s)


            dd         /= (24.0*60.0*60.0)                          # days

            me.dollar_days     += cnt * tx.price * dd

            if  dd < 2.0 :
                me.yr_dollars   = cnt * price                       # buy was today or yesterday, so just sort of ignore the whole thing as much as we can (to avoid divide by zero problem)
            else :

                txp         = float(cnt) * tx.price                 # total price

                ygm         = math.pow(cnt * price / txp, 365.0 / dd)

                if  True and (ygm != 1.0) :                         # use power function to get the right value
                    me.yr_dollars  += txp * (cnt * price - txp) / (ygm * txp - txp)
                else :
                    me.yr_dollars  += txp * (dd / 365.0)

                pass

            tx.shares  -= cnt
            if  tx.shares == 0.0 :
                del(me.transactions[txidx])
            pass

        return(mr)                                              # return the ratio of shares that have been sold


    def split(me, frm, to, when) :
        me.shares   = (me.shares * float(to)) / frm
        for i in xrange(len(me.transactions)) :
            me.transactions[i].shares   =   (me.transactions[i].shares * float(to)) / frm
        pass


    def find_price(me, when) :
        if  me.sym == CASH_SYMBOL :
            return(1.0)

        return(find_price(me.sym, when, me.csv_dir, me.hit_web))


    def find_date_vals(me, when) :
        if  me.sym == CASH_SYMBOL :
            return(when)

        return(find_date_vals(me.sym, when, me.csv_dir, me.hit_web))


    def add_stock_value_to_stock(me, sd) :
        me.value               +=   sd.value
        me.yr_dollars          +=   sd.yr_dollars
        me.dollar_days         +=   sd.dollar_days


    def calculate_dollar_years(me) :
        me.dollar_years         =   me.dollar_days / 365.0
        me.dy_percent           =                   yearly_percentage(-me.value, me.dollar_years)


    def calculate_yr_percents(me) :
        me.yr_percent           =                   yearly_percentage(-me.value, me.yr_dollars)




    def invest(me, how_much, when) :
        me.buy(how_much, 1, when)


    def interest(me, how_much, when) :
        me.dividend(how_much, when)


    def deinvest(me, how_much, when) :
        me.sell(how_much, 1, when)



    pass




class a_real_stock(a_stock) :

    def __init__(me, sym, market_sym, csv_dir, hit_web) :

        a_stock.__init__(me, sym, csv_dir, hit_web)

        me.market       = a_stock(market_sym, csv_dir, hit_web)



    def buy(me, shares, price, when) :
        p               = a_stock.buy(me, shares, price, when)
        mp              = me.market.find_price(when)
        me.market.buy(p / mp, mp, when)

        return(p)



    def sell(me, shares, price, when) :
        mr              = a_stock.sell(me, shares, price, when)
        ms              = me.market.shares * mr                 # how many market shares we must sell
        me.market.sell(ms, me.market.find_price(when), when)

        return(mr)


    def add_stock_value_to_stock(me, sd) :
        a_stock.add_stock_value_to_stock(me, sd)
        me.market.add_stock_value_to_stock(sd.market)


    def calculate_dollar_years(me) :
        a_stock.calculate_dollar_years(me)
        me.market.calculate_dollar_years()


    def calculate_yr_percents(me) :
        a_stock.calculate_yr_percents(me)
        me.market.calculate_yr_percents()


    pass




def process_portfolio_history(market_sym, orders, csv_dir, hit_web = True, after_when = [ 0, 0, 0 ], today = None, random_portfolio = None ) :

    if  today  == None :
        today   = get_yahoo_historical_csv.make_today_date(time.time() + 24 * 60 * 60)


    def _cmp_order_dates(a, b) :
        return(cmp(a[-1], b[-1]))

    orders.sort(_cmp_order_dates)                   # in case the orders are out of order


    market_sym  = get_yahoo_historical_csv.clean_symbol(market_sym)

    symdata     = {}
    ignores     = {}

    port_sym    = {}                                # mapping 'tween portfolio symbol and a random symbol if there is one in it's place

    for od in orders :
        # print od

        dt      = od[-1]

        sym     = get_yahoo_historical_csv.clean_symbol(od[1])

        if  random_portfolio == None :
            port_sym[sym]   = sym                   # just use the real symbol if we're not generating random portfolios with the same ops as the real one
        elif port_sym.has_key(sym) :
            sym             = port_sym[sym]
        elif len(random_portfolio) == 0 :
            port_sym[sym]   = sym                   # use the real symbol if we're simulating the random portfolio logic
        else :
            stoday          = find_date_vals(market_sym, today, csv_dir, hit_web)
            sdt             = find_date_vals(market_sym, dt,    csv_dir, hit_web)

            nsym            = sym
            while stoday and sdt :
                nsym        = random.choice(random_portfolio)
                if  not symdata.has_key(nsym) :

                    ymd         = find_date_vals(nsym, today, csv_dir, hit_web)

                    if            ymd and (ymd[0] == stoday[0]) and (ymd[1] == stoday[1]) and (ymd[2] == stoday[2]) :
                        ymd     = find_date_vals(nsym, dt,    csv_dir, hit_web)
                        if        ymd and (ymd[0] == sdt[0]) and (ymd[1] == sdt[1]) and (ymd[2] == sdt[2]) :
                            break
                        pass
                    pass
                pass

            port_sym[sym]   = nsym
            sym             = nsym


        if  not symdata.has_key(sym) :
            symdata[sym]    = a_real_stock(sym, market_sym, csv_dir, hit_web)


        sd  = symdata[sym]

        if  od[0]   == "BUY" :
            if  random_portfolio == None :
                sd.buy( od[2], od[3], dt)
            else :
                p   = sd.find_price(dt)
                sd.buy((od[2] * od[3]) / float(p), p, dt)
            sd.real_shares += od[2]                 # remember how many shares of the real company he bought

        elif od[0]  == "SELL" :
            if  random_portfolio == None :
                sd.sell(od[2], od[3], dt)
            else :
                p   = sd.find_price(dt)
                sc  = sd.shares * (od[2] / float(sd.real_shares))   # sell the same fraction of the real company's shares is the real transaction did
                sd.sell(sc, p, dt)
            sd.real_shares -= od[2]

        elif od[0].startswith("DIV") :
            if  random_portfolio == None :
                sd.dividend(od[2], dt)
            pass

        elif od[0]  == "DRIP" :
            if  random_portfolio == None :
                sd.dividend(od[3], dt)
                sd.buy(od[2], od[3] / od[2], dt)
            pass

        elif od[0]  == "FEE" :
            sd.dividend(-od[2], dt)

        elif od[0]  == "MBUY" :
            p       = sd.find_price(dt)
            sd.buy(od[2] / p, +p, dt)
            sd.real_shares += od[2] / p             # remember how many shares of the real company he bought

        elif od[0].startswith("ALLS") :
            p       = sd.find_price(dt)
            sd.sell(sd.shares, p, dt)
            sd.real_shares  = 0.0

        elif od[0].startswith("CASH") :
            sd.invest(od[2], dt)
            sd.real_shares += od[2]

        elif od[0].startswith("SPEND") :
            sd.deinvest(od[2], dt)
            sd.real_shares -= od[2]

        elif od[0].startswith("INT") :
            sd.interest(od[2], dt)
            sd.invest(od[2], dt)
            sd.real_shares += od[2]

        elif od[0].startswith("SPLIT") :
            if  random_portfolio == None :
                sd.split(od[2], od[3], dt)
            pass

        else :
            ps  = ""
            for i in xrange(len(od) - 1) :
                ps += " " + od[i]
            s = "Bad op:%s %s" % ( ps, get_yahoo_historical_csv.make_date_str(dt) )
            raise   ValueError(s)


        if  dt < after_when :
            ignores[sym]    = sym
            # print "ignore", sym
        pass


    #
    #   First throw away the stocks that he told us to ignore (not correct logic, yet - ignores whole stock, not just trades before a particular date)
    #
    for sym in ignores.keys() :
        del(symdata[sym])


    #
    #   Tell each stock how much it is of the total portfolio value
    #
    pv          = 0.0
    prices      = {}
    for sym in symdata.keys() :
        sd      = symdata[sym]

        shares  = sd.shares
        if  shares >= .0001 :
            if  sym == CASH_SYMBOL :
                prices[sym] = 1.0
            else :
                prices[sym]     =   find_price(     sym, today, csv_dir, hit_web)
                sd.today_vals   =   find_date_vals( sym, today, csv_dir, hit_web)
            pv += sd.shares * prices[sym]
        pass


    #
    #   Sell off the portfolio right now so we can find out where it stands.
    #   And figure out things.
    #
    md          = a_real_stock(TOTALS_SYMBOL, market_sym, csv_dir, hit_web)
    for sym in symdata.keys() :
        sd      = symdata[sym]

        shares  = sd.shares
        if  shares >= .0001 :
            sd.of_portfolio = (sd.shares * prices[sym]) / pv
            sd.sell(shares, prices[sym], today)
        else :
            sd.shares               =   sd.prev_shares          = 0.0
            if  hasattr(sd, 'market') :
                sd.market.shares    =   sd.market.prev_shares   = 0.0
            pass

        sd.calculate_yr_percents()
        sd.calculate_dollar_years()

        md.add_stock_value_to_stock(sd)

    md.calculate_yr_percents()
    md.calculate_dollar_years()

    symdata[md.sym] =   md                              # put the totals in with the data under TOTALS_SYMBOL

    return(symdata)




def show_sym_str(sym, do_html = False) :
    if  do_html :
        s   = '<A HREF="http://finance.yahoo.com/q?s=%s">%s</A>%s' % ( sym, sym, ' ' * max(0, 8 - len(sym)) )
    else :
        s   = "%-8s" % ( sym )

    return(s)


def html_percent(percent, do_html) :
    fs  =   fe  = ""
    if  do_html :
        if  percent >=  15.0 :
            fs  = '<FONT COLOR="GREEN">'
            fe  = '</FONT>'
        if  percent >=  50.0 :
            fs  = "<B>" + fs
            fe += "</B>"
        if  percent <    0.0 :
            fs  = '<FONT COLOR="RED">'
            fe  = '</FONT>'
        if  percent <= -15.0 :
            fs  = "<B>" + fs
            fe += "</B>"
        pass
    return( ( fs, fe ) )


def show_position_str(sym, fraction_of_portfolio, shares, value, yr_percent, show_info = 0, do_html = False) :

    sym     = show_sym_str(sym, do_html)

    ps      = "     "
    if  fraction_of_portfolio > 0.0 :
        ps  = " %3.0f%%" % ( fraction_of_portfolio * 100.0 )

    ( fs, fe ) = html_percent(yr_percent, do_html)
    if  show_info < 2 :
        s   =   "%s%s %s%7.1f%%%s"             % ( sym, ps,                fs, yr_percent, fe )
    else :
        s   =   "%s%s %7.1f %9.2f %s%7.1f%%%s" % ( sym, ps, shares, value, fs, yr_percent, fe )

    return(s)


def show_market_adjust_position_str(mkt_sym, mkt_yr_percent, yr_percent, do_html = False) :

    s   = ""
    if  mkt_sym :
        ( fs, fe )  = html_percent(mkt_yr_percent, do_html)
        s           =   " %s%7.1f%%%s ~ %s" % ( fs, yr_percent - mkt_yr_percent, fe, show_sym_str(mkt_sym, do_html) )

    return(s)


def show_dollar_days(dollar_years, percent, do_html = False) :
    ( fs, fe )   = html_percent(percent, do_html)
    s            = " %7.0f %s%7.1f%%%s" % ( dollar_years, fs, percent, fe )

    return(s)


def cmp_val(v) :
    if  v < 0 : return(-1)
    if  v > 0 : return( 1)
    return(0)


def html_u(c, do_html) :
    if  not do_html :   return(c)
    return("<U>" + c + "</U>")


def show_portfolio_history_str(symdata, market_sym, sort_by = None, show_info = 0, do_html = False) :

    crlf        = "\n"

    retval      = ""

    market_sym  = get_yahoo_historical_csv.clean_symbol(market_sym)

    dashes  = "-----------------------------------"
    if  show_info < 2 : dashes = ""
    dashes += "-----------------------------------------------------" + crlf

    if  show_info != 0 :

        if  do_html :   retval += "<HTML><BODY><PRE>" + crlf

        retval                 += time.asctime() + crlf

        if  do_html :   retval += "<B>"

        if  show_info < 2 :
            retval             += "Symbo" +                                         \
                                  html_u("l", do_html) + "     " +                  \
                                  html_u("P", do_html) + "%  " +                    \
                                  html_u("1", do_html) + "yrGain  " +               \
                                  html_u("M", do_html) + "arket-relative    -"
        else :
            retval             += "Symbo" +                                         \
                                  html_u("l", do_html) + "     " +                  \
                                  html_u("P", do_html) + "%  " +                    \
                                  html_u("S", do_html) + "hares     $" +            \
                                  html_u("G", do_html) + "ain  " +                  \
                                  html_u("1", do_html) + "yrGain  " +               \
                                  html_u("M", do_html) + "arket-relative   " +      \
                                  html_u("Y", do_html) + "rDollar   Yr" +           \
                                  html_u("D", do_html) + "lrG -"
        if  do_html :   retval += "</B>"

        retval                 += crlf
        retval                 += dashes

    skeys   = symdata.keys()


    def cmp_syms(k1, k2, cv) :
        if  cv != 0.0 : return(cv)
        return(cmp(symdata[k1].sym, symdata[k2].sym))


    if  sort_by[-1]  == 'P' :
        def cmp_of_portfolio(k1, k2) :          return(cmp_syms(k1, k2, cmp_val(symdata[k2].of_portfolio                                 - symdata[k1].of_portfolio)))
        skeys.sort(cmp_of_portfolio)
    elif (sort_by[-1] == 'S') and (show_info >= 2) :
        def cmp_shares(k1, k2) :                return(cmp_syms(k1, k2, cmp_val(symdata[k2].prev_shares                                  - symdata[k1].prev_shares)))
        skeys.sort(cmp_shares)
    elif (sort_by[-1] == 'G') and (show_info >= 2) :
        def cmp_gain(k1, k2) :                  return(cmp_syms(k1, k2, cmp_val(symdata[k1].value                                        - symdata[k2].value)))
        skeys.sort(cmp_gain)
    elif sort_by[-1] == '1' :
        def cmp_1yr_gain(k1, k2) :              return(cmp_syms(k1, k2, cmp_val(symdata[k2].yr_percent                                   - symdata[k1].yr_percent)))
        skeys.sort(cmp_1yr_gain)
    elif sort_by[-1] == 'M' :
        def cmp_market_relative_gain(k1, k2) :  return(cmp_syms(k1, k2, cmp_val((symdata[k2].yr_percent - symdata[k2].market.yr_percent) - (symdata[k1].yr_percent - symdata[k1].market.yr_percent))))
        skeys.sort(cmp_market_relative_gain)
    elif (sort_by[-1] == 'Y') and (show_info >= 2) :
        def cmp_market_relative_gain(k1, k2) :  return(cmp_syms(k1, k2, cmp_val(symdata[k2].dollar_years                                 - symdata[k1].market.dollar_years)))
        skeys.sort(cmp_market_relative_gain)
    elif (sort_by[-1] == 'D') and (show_info >= 2) :
        def cmp_market_relative_gain(k1, k2) :  return(cmp_syms(k1, k2, cmp_val(symdata[k2].dy_percent                                   - symdata[k1].dy_percent)))
        skeys.sort(cmp_market_relative_gain)
    elif sort_by[-1] == 'L' :
        skeys.sort()
    else :
        skeys.sort()
    if  sort_by[0:1] == '-' :
        skeys.reverse()


    for sym in skeys :
        if  sym    !=  TOTALS_SYMBOL :
            sd      = symdata[sym]
            s       = show_position_str(sym, sd.of_portfolio, sd.prev_shares, -sd.value, sd.yr_percent, show_info, do_html)
            s      += show_market_adjust_position_str(sd.market.sym, sd.market.yr_percent, sd.yr_percent, do_html)
            if  show_info >= 2 :
                s  += show_dollar_days(sd.dollar_years, sd.dy_percent, do_html)
            if  sd.market.yr_percent < 0.0 :
                s  += " -"                                                  # market was down overall
            else :
                s  += "  "
            if  sd.today_vals :
                s  += get_yahoo_historical_csv.make_date_str(sd.today_vals)

            retval += s + crlf
        pass

    md  = symdata[TOTALS_SYMBOL]

    if  show_info  != 0 :
        s           = ""
        if  md.market.yr_dollars != 0.0 :
            s      += dashes
            s      += show_position_str(market_sym, 0.0, 0.0, -md.market.value, md.market.yr_percent, show_info, do_html)
            if  show_info >= 2 :
                s  += "                    "
                s  += show_dollar_days(md.market.dollar_years, md.market.dy_percent, do_html)
            s      += crlf
        retval     += s + dashes

    if  show_info != 0 :
        if  show_info < 2  :
            av  = rv  = ""
            ps  = "     "
        else :
            ps  = "           "
            av  = " $%9.2f " % (  -md.value                    )
            rv  = " $%9.2f " % ( -(md.value - md.market.value) )

        ( fs, fe )  = html_percent(md.yr_percent, do_html)
        retval     += "%sAbsolute:%s%s%7.1f%%%s"               % ( ps, av, fs, md.yr_percent, fe )
        if  show_info >= 2 :
            retval += "                    "
            retval += show_dollar_days(md.dollar_years, md.dy_percent, do_html)
        retval     += crlf

        ( fs, fe )  = html_percent(md.yr_percent - md.market.yr_percent, do_html)
        retval     += "%sRelative:%s         %s%7.1f%%%s ~ %s" % ( ps, rv, fs, md.yr_percent - md.market.yr_percent, fe, show_sym_str(market_sym, do_html)  )
        if  show_info >= 2 :
            ( fs, fe ) = html_percent(md.dy_percent - md.market.dy_percent, do_html)
            retval += "         %s%7.1f%%%s" % ( fs, md.dy_percent - md.market.dy_percent, fe )
        retval     += crlf

        if  do_html :
            retval += "</PRE></BODY></HTML>" + crlf
        pass

    return(retval)



def read_portfolio_history_file(fname) :
    msgs    = tgcmsg.amp_messages_from_file(fname)
    if  msgs == None :      return([])
    return(map(lambda msg : parse_order(msg), msgs))




if  __name__ == '__main__' :
    import  sys
    import  os

    if  len(sys.argv) < 2 :

        print   "Tell me portfolio information"

    else :

        import  TZCommandLineAtFile
        import  TZKeyReady
        import  tz_os_priority
        import  tzlib


        program_name        = sys.argv.pop(0)
        TZCommandLineAtFile.expand_at_sign_command_line_files(sys.argv)

        orders              =   []
        show_info           =   0
        timeout             =   None
        csv_dir             =   ""
        hit_web             =   True
        market_sym          =   "^gspc"                 # S&P 500
        do_html             =   False
        after_when          =   parse_date("1-Jan-1900")
        sort_by             =   'N'
        random_portfolio    =   []
        today               =   None

        if  (tzlib.array_find(sys.argv, "--help") >= 0) or (tzlib.array_find(sys.argv, "-h") >= 0) or (tzlib.array_find(sys.argv, "-?") >= 0) :
            print """
python %s   Track how well buy and sell history has done - against a market average.

--csv_dir       dir     Directory name to put the history .csv file(s) in to.
--show_info             Print progress/debugging info - one time, public info - twice, private.
--no_web_hit            Do not hit the web for price history unless absolutely needed.
--timeout       seconds Set the web-bit timeout.
--after         D-Mon-Y Ignore all operations on stocks with operations before given date.
--market_symbol symbol  Set the market symbol wanted to compare against ([D]ow [S]&P500 [N]asdaq [R]ussell3000) (default: S&P500)
--html                  Print out data in HTML form.
--sort    (-)[NPSG1MYD] Sort on various columns (default: N). Leading '-' reverses order.
--after MMM/DD/YYYY     Just show how things have done since the given date.
--random_portfolio file Do metrics with a random portfolio drawn from the symbols in 'file'.
--today         D-Mon-Y Set when today is (for testing).

Input files are in text line format:

    ;                                 comment

    BUY         symbol shares price date    ; buy  given shares at given price
    SELL        symbol shares price date    ; sell given shares at given price
    FEE         symbol        money date    ; subtract money from holding
    DIVIDEND    symbol        money date    ; add money to holding
    DRIP        symbol shares money date    ; add dividend as reinvested in given shares for given money (implies DIV)
    MBUY        symbol        money date    ; buy shares at previous day's ending price with given money
    ALLSELL     symbol              date    ; sell all shares owned at previous day's ending price
    SPLIT       symbol from   to    date    ; the stock split on this date (a normal split might be from=1 to=2)

e.g.:

    buy     msft 100 27.34 3-Jan-2005
    DIV     MSFT     10.14 8-dec-2005

    Dates can be in these formats:

        30-Jan-2002 or 30-Jan-02
        1/30/2002   or 1/30/02
        1-30-2002   or 1-30-02
        Jan-30-2002 or Jan-30-02
        Jan/30/2002 or Jan/30/02


Market comparisions are done against most recent previous day's closing market price.

""" % os.path.basename(program_name)
            sys.exit(254)



        while True :
            oi  = tzlib.array_find(sys.argv, "--after")
            if  oi < 0 :    break
            del sys.argv[oi]
            after_when      = parse_date(sys.argv.pop(oi))

        while True :
            oi  = tzlib.array_find(sys.argv, "--timeout")
            if  oi < 0 :    break
            del sys.argv[oi]
            timeout         = int(sys.argv.pop(oi))

        while True :
            oi  = tzlib.array_find(sys.argv, "--show_info")
            if  oi < 0 :    break
            del sys.argv[oi]
            show_info      += 1

        while True :
            oi  = tzlib.array_find(sys.argv, "++show_info")
            if  oi < 0 :    break
            del sys.argv[oi]
            show_info       = max(show_info - 1, 0)

        while True :
            oi  = tzlib.array_find(sys.argv, "--no_web_hit")
            if  oi < 0 :    break
            del sys.argv[oi]
            hit_web         = False

        while True :
            oi  = tzlib.array_find(sys.argv, "--html")
            if  oi < 0 :    break
            del sys.argv[oi]
            do_html         = True

        while True :
            oi  = tzlib.array_find(sys.argv, "--csv_dir")
            if  oi < 0 :    break
            del sys.argv[oi]
            csv_dir         = sys.argv.pop(oi)

        while True :
            oi  = tzlib.array_find(sys.argv, "--sort")
            if  oi < 0 :    break
            del sys.argv[oi]
            sort_by         = sys.argv.pop(oi).upper()


        while True :
            oi  = tzlib.array_find(sys.argv, "--market_symbol")
            if  oi < 0 :    break
            del sys.argv[oi]
            market_sym       = sys.argv.pop(oi).lower()
            mktsyms = {
                        "d" : "^dji",
                        "s" : "^gspc",
                        "n" : "^ixic",
                        "r" : "^rua",
                      }
            market_sym = mktsyms.get(market_sym, market_sym);


        while True :
            oi  = tzlib.array_find(sys.argv, "--buy")
            if  oi < 0 :    break
            del sys.argv[oi]
            info            =   [ "BUY" ]
            info.append(sys.argv.pop(oi))
            info.append(sys.argv.pop(oi))
            info.append(sys.argv.pop(oi))
            orders.append(parse_order(info))

        while True :
            oi  = tzlib.array_find(sys.argv, "--sell")
            if  oi < 0 :    break
            del sys.argv[oi]
            info            =   [ "SELL" ]
            info.append(sys.argv.pop(oi))
            info.append(sys.argv.pop(oi))
            info.append(sys.argv.pop(oi))
            orders.append(parse_order(info))


        while True :
            oi  = tzlib.array_find(sys.argv, "--random_portfolio")
            if  oi < 0 :    break
            del sys.argv[oi]

            random_portfolio    = [ "@" + sys.argv.pop(oi) ]
            TZCommandLineAtFile.expand_at_sign_command_line_files(random_portfolio)

            if  len(random_portfolio) <= 1 :
                s   = "Random porfolio doesn't seem to like these symbols: " + str(random_portfolio)
                raise   IndexError(s)

            if  len(random_portfolio) < 10 :
                print "Did you mean to use these symbols for random portfolio run: " + str(random_portfolio)
                print
            pass


        while True :
            oi  = tzlib.array_find(sys.argv, "--today")
            if  oi < 0 :    break
            del sys.argv[oi]
            today   = parse_date(sys.argv.pop(oi))



        if  csv_dir         :  csv_dir   =  os.path.normpath(csv_dir)

        if  not csv_dir     :  csv_dir   =  "."

        if  not os.path.isdir(csv_dir)  :   os.makedirs(csv_dir)


        #
        #   Run through the files of commands/actions
        #
        while len(sys.argv) > 0 :

            fname = sys.argv.pop(0)

            if  fname[0:1] == '-' :
                print "Did you mean for", fname, "to be a command line parameter?"
                print

            hist    = read_portfolio_history_file(fname)
            if  len(hist) == 0 :
                s = "No actions in " + fname
                raise   IndexError(s)
            orders += hist

        pass

        if  random_portfolio :

            tz_os_priority.set_proc_to_idle_priority()

            symdata =   process_portfolio_history(market_sym, orders, csv_dir, hit_web, after_when, today, [] )
            md      =   symdata[TOTALS_SYMBOL]
            target  =   md.yr_percent - md.market.yr_percent

            results = []

            while True :
                symdata =   process_portfolio_history(market_sym, orders, csv_dir, hit_web, after_when, today, random_portfolio)

                md  = symdata[TOTALS_SYMBOL]
                r   = md.yr_percent - md.market.yr_percent

                results.append(r)
                if  TZKeyReady.tz_key_ready() :
                    break

                print len(results), "%6.2f%% %6.2f%%" % ( r, md.dy_percent - md.market.dy_percent )

            results.sort()

            midp    = results[ 50 * len(results) / 100 ]

            print   "Runs:      %u"     % ( len(results) )
            print   "Target:    %-5.1f" % ( target )
            print   "Percentile Relative"

            for i in [ 1, 2, 5, 10, 15, 20, 30, 40, 50, 60, 70, 80, 85, 90, 95, 97, 98, 99 ] :
                r   = results[i * len(results) / 100]
                print "%2u         %5.1f%% %5.1f%%" % ( i, r, r - target )

            pass

        else :
            symdata =   process_portfolio_history(market_sym, orders, csv_dir, hit_web, after_when, today)
            s       =   show_portfolio_history_str(symdata, market_sym, sort_by, show_info, do_html)
            if  do_html and os.linesep != "\r\n" :
                s   = re.sub(r"\n", "\r\n", s)

            print s

        pass

    pass


#
#
#


__all__ = [
            'TOTALS_SYMBOL',

            'parse_date',
            'parse_order',

            'process_portfolio_history',
          ]


#
#
#
# eof
