#!/usr/bin/python
# tzlib.py
# --copyright-- Copyright 2007 (C) Tranzoa, Co. All rights reserved. Warranty: You're free and on your own here. This code is not necessarily up-to-date or of public quality.
# --url-- http://www.tranzoa.net/tzpython/
# --email-- pycode is the name to send to. tranzoa.com is the place to send to.
# --bodstamps--
# July 22, 2003 bar
# October 7, 2003 bar ambiguous_file_list
# January 31, 2004 bar find_in_array
# September 28, 2004 bar define False and True
# October 25, 2004 bar hash sort routines (built in already?)
# January 12, 2005 bar array_find
# January 13, 2005 bar file_name_able and ascii and crc32 routines moved to here
# January 17, 2005 bar add <>+'" to the characters changed by file_name_able
# fix __ALL__'s file_name_able name
# January 30, 2005 bar _ascii() call -> ascii() - and make it right and import re
# February 9, 2005 bar tz_vector_cosine
# February 17, 2005 bar html character entities
# February 25, 2005 bar safe_html
# March 1, 2005 bar option to translate to space or whatever
# March 19, 2005 bar typo in ascii - StringType's
# June 27, 2005 bar collapse array_find in to find_in_array
# without_dupes
# July 12, 2005 bar check crc
# August 24, 2005 bar more fooling with the new python warnings about 32 bit ints in crc logic
# September 5, 2005 bar look up any of an array of strings in find_in_array / array_find
# March 25, 2006 bar try to avoid a latin1 to ascii fuss
# April 26, 2006 bar strrev
# June 14, 2006 bar string_pairs and flat_positional_strings
# June 16, 2006 bar de_html_str
# June 21, 2006 bar pull scripts out in de_html_str
# May 5, 2007 bar linear_regression
# May 16, 2007 bar zlib crc
# May 17, 2007 bar printable
# June 22, 2007 bar comment
# July 1, 2007 bar longitudinal parity (xor sum)
# October 12, 2007 bar distance / direction routines (here so I don't need to look 'em up again)
# November 18, 2007 bar turn on doxygen
# November 20, 2007 bar move lf_only() and no_blank_lines() from strip_files.py
# lf_only() fixes \r\r\n to be one \n
# as does safe_html() make \r\r\n one
# c_string()
# read_whole_text_file() (finally)
# read_whole_binary_file() (ditto)
# November 27, 2007 bar lf_only_with_no_trailing_white_space()
# November 27, 2007 bar insert boilerplate copyright
# December 1, 2007 bar elapsed_time()
# December 14, 2007 bar write the file in write_whole_...
# December 15, 2007 bar multiline_strip
# December 21, 2007 bar maybe_wrap_with_cdata
# January 1, 2008 bar s_except_1()
# January 18, 2008 bar unicode_byte_string()
# and auto-convert unicode to utf-16 or utf-8 when writing whole files
# January 20, 2008 bar same_object() (for doc purposes)
# January 21, 2008 bar s_except_1 takes lists and dictionarys
# January 29, 2008 bar sys_err_file_line()
# February 8, 2008 bar finally, a binary_search() i can remember
# fix blkcrc32() under python 2.2 (zlib's crc isn't right, apparently)
# add start/end indices to find_in_array and array_find
# March 9, 2008 bar temp_file_name
# March 12, 2008 bar allow arrays to de_html_str()
# March 13, 2008 bar comment
# May 17, 2008 bar email adr
# August 13, 2008 bar make_dictionary works for strings
# August 18, 2008 bar use basestring
# August 29, 2008 bar basestring instead of StringType because of unicode strings and others
# October 28, 2008 bar float_regx_str
# November 6, 2008 bar bool_to_0_or_1()
# November 15, 2008 bar egad! I've been making default params as [] and {}
# November 28, 2008 bar decode_html_entities latin1/unicode fixes
# print_exception()
# December 20, 2008 bar INITIAL_CRC32_VALUE
# December 28, 2008 bar ooops, it should be zero, not -1
# January 7, 2009 bar excel_column_name
# January 9, 2009 bar list_lstrip()
# February 20, 2009 bar golden_smaller()
# March 27, 2009 bar python 2.6
# April 1, 2009 bar printable_str()
# April 11, 2009 bar run test ok under python 2.5
# April 15, 2009 bar max_index() and min_index()
# June 2, 2009 bar find_upper_dir()
# September 2, 2009 bar fix find_upper_dir for when the dir is not found (unix needs testing)
# September 9, 2009 bar radian_angle_difference()
# September 16, 2009 bar find_upper_file_or_dir
# November 13, 2009 bar crc16
# November 28, 2009 bar file_signature()
# January 24, 2010 bar force 16 bit crc to be 16 bits
# February 10, 2010 bar get_tid() for linux
# February 11, 2010 bar value_array_for_key()
# replace_value_array_for_key()
# February 20, 2010 bar windows version of get_tid
# March 14, 2010 bar pickle file by file name routines
# July 22, 2010 bar keep replace_file inside the only place it's used (and shouldn't be here, anyway)
# September 23, 2010 bar make_index_dictionary() and update_all_case_keys()
# September 26, 2010 bar print_stack()
# September 28, 2010 bar c_ctrl_esc()
# October 1, 2010 bar C, C++ comment remover
# October 2, 2010 bar safe whole file read/write
# safe_relpath()
# October 10, 2010 bar base 36
# October 19, 2010 bar best_ascii()
# October 24, 2010 bar de_dupe_str()
# October 27, 2010 bar find_arg()
# November 2, 2010 bar allow update_all_case_keys() to accept int and other keys
# November 5, 2010 bar flatten_array()
# November 7, 2010 bar expand user and vars in ambiguous file name finder
# November 9, 2010 bar c_string changes \ to \134 rather than \\ so that there are no doubled slashes
# November 16, 2010 bar same_file()
# reroute_stdout_err()
# December 5, 2010 bar multiline_flush_left()
# invert_dictionary()
# December 27, 2010 bar safer pickling
# allow printable to zap bad chars
# February 8, 2011 bar make unpickle safer ('module' object has not attribute '---class---'
# March 21, 2011 bar line intersections
# March 27, 2011 bar allow smart proto to pickle_file
# April 14, 2011 bar under non-win32, put the .cfg file in get_ini_or_cfg_file_name() in an app directory, not at the user's home dir
# June 15, 2011 bar faster max_index and min_index
# August 9, 2011 bar let those faster _index routines work with numpy arrays
# November 5, 2011 bar can_run_program()
# November 6, 2011 bar whack_file()
# November 13, 2011 bar q_get()
# November 29, 2011 bar pyflake cleanup
# allow make_dictionary to take a dictionary (it's shallow duped, effectively)
# November 30, 2011 bar uh. test it
# December 14, 2011 bar kalman filter
# get rid of ALL
# December 23, 2011 bar cmp_str_with_ints()
# January 18, 2012 bar left_valley(), rite_valley()
# February 4, 2012 bar run_program() and in_screen_saver()
# February 7, 2012 bar add stderr to the output string in run_program
# February 11, 2012 bar wdhms_str()
# March 8, 2012 bar buples_to_dictionary()
# March 11, 2012 bar weighted_choice()
# March 15, 2012 bar make ambiguous_file_list() safer
# May 1, 2012 bar file_name_able() uses best_ascio to turn latin 1 characters in to ascii (but really should leave them for modern OS's)
# --eodstamps--
## \file
#
#
# Buncha things.
#
#
import cPickle
import glob
import htmlentitydefs
import math
import os
import Queue
import random
import re
import string
import subprocess
import sys
import time
import traceback
import unicodedata
import zlib
from types import ListType, TupleType, UnicodeType, DictionaryType
try :
import ctypes
except ImportError :
cypes = None
try :
import win32api
except ImportError :
win32api = None
## Run under older Pythons
try:
True, False
except NameError:
True = 1
False = 0
float_regx_str = r"(?:[\+\-]?(?:\d+(?:\.\d*)?|\.\d+))"
def print_exception() :
e = sys.exc_info()
traceback.print_exception(e[0], e[1], e[2])
def print_stack() :
traceback.print_stack()
def get_tid() :
tid = -2
if win32api :
tid = win32api.GetCurrentThreadId()
elif sys.platform.find('linux') >= 0 : # we could try to find the syscall.h or unistd.d file with the SYS_gettid in it
if ctypes :
tid = int(str(ctypes.CDLL('libc.so.6').syscall(224))) # does not work on 64-bit OS (that's probably the reason it returns -1 on spring)
if tid < 0 : # note: "pstree -p -H " + str(os.getpid) is ambiguous at best, and would need a system lock around it, too, if more that 1 thread were using it for this purpose
tid = int(str(ctypes.CDLL('libc.so.6').syscall(186))) # found with " #include printf("%u\n", SYS_gettid) "
pass
pass
return(tid)
def run_program(cmd, stdin = None, stdout = None, stderr = None) :
"""
Run the given command line program (full path needed for the executable's name).
Return the exit code int and the stdout/stderr output stripped and CRLFCRLF...LFLF... -> LF converted.
This is a blocking function and does not return until the program finishs. So you can't monitor the output as it runs.
"""
try :
stdin = stdin or subprocess.PIPE
stdout = stdout or subprocess.PIPE
stderr = stderr or subprocess.PIPE
if sys.platform == 'win32' :
p = subprocess.Popen(cmd, shell = True, stdin = stdin, stdout = stdout, stderr = stderr)
else :
p = subprocess.Popen(cmd, shell = True, stdin = stdin, stdout = stdout, stderr = stderr, close_fds = True)
( rs, se ) = p.communicate()
r = p.returncode
rs = (rs or "").strip() + (se or "").strip()
rs = re.sub(r"(?:\r?\n)+", "\n", rs).strip() + "\n"
except :
raise
return(r, rs)
def in_screen_saver() :
""" Is the screen saver running? (Only works under Gnome !!!! ) """
if sys.platform.find('linux') < 0 :
return(False)
( r, rs ) = run_program("gnome-screensaver-command -q")
if (not r) and (rs.find(" active") >= 0) :
return(True)
return(False)
def can_run_program(program_file_name) :
""" Return True if it appears that this program is runnable. """
return(os.path.exists(program_file_name) and os.path.isfile(program_file_name) and os.access(program_file_name, os.R_OK) and os.access(program_file_name, os.X_OK))
def ambiguous_file_list(ambiguous_name, do_sub_dirs = False) :
"""
Return an array with the names of the files that match the given ambiguous file name.
"""
abn = os.path.expanduser(os.path.expandvars(ambiguous_name))
files = glob.glob(abn)
if do_sub_dirs :
(dir_name, amb_name) = os.path.split(abn)
if not len(dir_name) :
dir_name = "./"
try :
abn = os.path.join(os.path.normpath(dir_name), os.path.normpath(amb_name)) # note: not sure why this is here. it's not what we want for listdir, but would work for glob, though they both have weirdnesses
for fn in os.listdir(dir_name) :
ffn = os.path.join(os.path.normpath(dir_name), fn)
if os.path.isdir(ffn) :
fls = ambiguous_file_list(os.path.join(ffn, amb_name), do_sub_dirs)
for fln in fls :
files.append(fln)
pass
pass
pass
except ( OSError, IOError, ValueError ) :
pass
pass
return(files)
def binary_search(a, item, cmp_rtn = None, si = None, ei = None, cmp_obj = None) :
"""
Binary search the sorted array, 'a' looking for 'item' or the first index in the array that has a value greater than 'item'.
Use the given 'cmp_rtn', which looks like the default, direct_cmp(), below.
Return the found index (or len(a) if all the array's items are above 'item').
"""
def direct_cmp(a, i, item, cmp_obj) :
return(cmp(a[i], item))
si = si or 0
if ei == None :
ei = len(a)
if cmp_rtn == None :
cmp_rtn = direct_cmp
lo = si
hi = ei
mid = (lo + hi) / 2
while lo < hi :
mid = (lo + hi) / 2
if cmp_rtn(a, mid, item, cmp_obj) < 0 :
mid += 1
lo = mid
else :
hi = mid
pass
# let the caller do mid=max(0, min(len(a), mid)) if he wants
return(mid)
def find_in_array(a, s, bi = None, ei = None) :
"""
Find an item in an array, 'a' - or the first of an array of items, 's', in the array..
Return -1 if not found.
Otherwise return the found array index.
"""
if not isinstance(a, ListType) and not isinstance(a, TupleType) :
a = [ a ]
if not isinstance(s, ListType) and not isinstance(s, TupleType) :
s = [ s ]
bi = bi or 0
if ei == None :
ei = len(a)
for ss in s :
try :
i = a.index(ss, bi, ei)
return(i)
except TypeError : # catch python 2.2 or whatever
try :
i = a[bi:ei].index(ss)
return(i)
except IndexError :
pass
except ValueError :
pass
pass
except IndexError :
pass
except ValueError :
pass
pass
return(-1)
def max_index(a) :
"""
Return the index in to the given array of the maximum value in the array.
"""
if len(a) == 0 :
return(0)
if hasattr(a, 'index') : # numpy arrays don't have this, apparently
return(a.index(max(a))) # benchmark says this is faster (than enumerate, too)
bi = 0
bv = a[bi]
for i in xrange(1, len(a)) :
v = a[i]
if bv < v :
bv =v
bi = i
pass
return(bi)
def min_index(a) :
"""
Return the index in to the given array of the minimum value in the array.
"""
if len(a) == 0 :
return(0)
if hasattr(a, 'index') : # numpy arrays don't have this, apparently
return(a.index(min(a))) # benchmark says this is faster (than enumerate, too)
bi = 0
bv = a[bi]
for i in xrange(1, len(a)) :
v = a[i]
if bv > v :
bv = v
bi = i
pass
return(bi)
def left_valley(a, i, bump = 0) :
""" Return the lowest spot in the given array to the left/west of the given index. """
bump = bump or 0
while i > 0 :
if a[i - 1] > a[i] + bump : # find the spot to the left/west of a flat valley
break
i -= 1
return(i)
west_valley = left_valley
def rite_valley(a, i, bump = 0) :
""" Return the lowest spot in the given array to the rite/east of the given index. """
bump = bump or 0
while i < len(a) - 1 :
if a[i + 1] > a[i] + bump : # find the spot to the rite/east of a flat valley
break
i += 1
return(i)
east_valley = rite_valley
right_valley = rite_valley
#
# Snagged from: http://rightfootin.blogspot.com/2006/09/more-on-python-flatten.html
#
def flatten_array(l, ltypes = (list, tuple)) :
"""
Flatten an array (of arrays).
"""
ltype = type(l)
l = list(l)
i = 0
while i < len(l) :
while isinstance(l[i], ltypes) :
if not l[i] :
l.pop(i)
i -= 1
break
else :
l[i:i + 1] = l[i]
pass
i += 1
return(ltype(l))
def invert_dictionary(d, dupable_values = {}) :
"""
Invert the given dictionary.
The result is updated with force_dict.
"""
dupable_values = dupable_values or {}
rd = dict((v, k) for k, v in d.iteritems())
if len(rd) != len(d) :
rrd = make_dictionary(rd.values())
va = [ { k : v } for k, v in d.iteritems() if (k not in rrd) and (v not in dupable_values) ]
if va :
raise IndexError("invert_dictionary(): %d duped value%s %s" % ( len(d) - len(rd), s_except_1(va), repr(va) ) )
rd.update(dupable_values)
return(rd)
def make_dictionary(a, val = True) :
"""
Make a dictionary from a list/tuple.
If python is 2.3+, then this is fromkeys().
"""
retval = {}
try :
retval.update(a)
return(retval)
except ( ValueError, AttributeError, TypeError ) :
pass
if a != None :
if not isinstance(a, ListType) and not isinstance(a, TupleType) and not isinstance(a, basestring) :
a = [ a ]
for k in a :
retval[k] = val
pass
return(retval)
def buples_to_dictionary(a) :
"""
Given an array of [ key, value ] items return a dictionary.
"""
d = {}
for i in a :
d[i[0]] = i[1]
return(d)
def make_index_dictionary(a) :
"""
Make a dictionary from a list/tuple where the keys are the list/tuple's values and the values are the list/tuple indices.
"""
retval = {}
if a != None :
if not isinstance(a, ListType) and not isinstance(a, TupleType) and not isinstance(a, basestring) :
a = [ a ]
for i, k in enumerate(a) :
retval[k] = i
pass
return(retval)
def update_all_case_keys(d) :
"""
Update a dictionary, adding key/values for all upper() or lower() versions of existing keys where there are none now.
"""
for k in d.keys() :
try :
nk = k.lower()
if nk not in d :
d[nk] = d[k]
nk = k.upper()
if nk not in d :
d[nk] = d[k]
pass
except AttributeError :
pass # the key is and int or something
pass
pass
def list_lstrip(a, va) :
"""
Return an array subjected to the logical equivalent of string.lstrip().
"""
if not isinstance(va, (ListType, TupleType)) :
va = [ va ]
va = make_dictionary(va)
for i in xrange(len(a)) :
if a[i] not in va :
return(a[i:])
pass
return([])
def fromkeys(a, val = True) :
"""
Synonym for make_dictionary.
"""
return(make_dictionary(a, val))
def without_dupes(a) :
"""
Return a copy of the given array with dupes removed.
The item order will probably be changed.
"""
return(make_dictionary(a).keys())
def de_dupe_str(s) :
"""
Return the given string without duplicate characters (after the 1st instance of each unique character).
"""
cnts = {}
so = ""
if type(s) == type(u"") :
so = u""
for c in s :
cnts[c] = cnts.get(c, 0) + 1
if cnts[c] < 2 :
so += c
pass
return(so)
def keys_sorted_by_values_keys(hash_dict) :
"""
Return an array of the keys from a dictionary, sorted by value/key.
"""
def _vkcmp(k1, k2) :
c = cmp(hash_dict[k1], hash_dict[k2])
if c != 0 :
return(c)
return(cmp(k1, k2))
v = hash_dict.keys()
v.sort(_vkcmp)
return(v)
def values_sorted_by_values_keys(hash_dict) :
"""
Return an array of the values from a dictionary, sorted by value/key.
"""
kys = keys_sorted_by_values_keys(hash_dict)
return(map(lambda k : hash_dict[k], kys))
def keys_sorted_by_keys_values(hash_dict) :
"""
Return an array of the keys from a dictionary, sorted by key/value.
"""
def _vkcmp(k1, k2) :
c = cmp(k1, k2)
if c != 0 :
return(c)
return(cmp(hash_dict[k1], hash_dict[k2]))
v = hash_dict.keys()
v.sort(_vkcmp)
return(v)
def values_sorted_by_keys_values(hash_dict) :
"""
Return an array of the values from a dictionary, sorted by key/value.
"""
kys = keys_sorted_by_keys_values(hash_dict)
return(map(lambda k : hash_dict[k], kys))
def value_array_for_key(array_of_dicts, k, dflt = None) :
"""
Return an array of values for the given key in each dict in an array of dicts.
If dflt is None :
Skip any dict that doesn't have the key, returning an array shorter than the input array.
else :
Return dflt for any missing value.
"""
if dflt == None :
return([ d[k] for d in array_of_dicts if k in d ])
return( [ d.get(k, dflt) for d in array_of_dicts ])
def replace_value_array_for_key(array_of_dicts, k, a) :
"""
Put the given array of values in each dict in an array of dicts - each value under the given key.
Raise an IndexError exception if the length of 'a' is not the same as 'array_of_dicts'.
"""
if len(array_of_dicts) != len(a) :
raise IndexError("Wrong length %u in to %u" % ( len(a), len(array_of_dicts) ))
for di in xrange(len(array_of_dicts)) :
array_of_dicts[di][k] = a[di]
pass
def array_find(a, s, si = None, ei = None) :
"""
Return the index of the item in an array or negative 1 if it's not there.
"""
return(find_in_array(a, s, si, ei))
def find_arg(args, a) :
"""
Return the index in to args of 'a' (or any string in 'a') or -1 if not found.
Underscores are ignored and/or match dashes.
"""
if not isinstance(args, ListType) and not isinstance(args, TupleType) :
args = [ args ]
adn = make_index_dictionary([ s.replace('_', '' ) for s in args ])
add = make_index_dictionary([ s.replace('_', '-') for s in args ])
if not isinstance(a, ListType) and not isinstance(a, TupleType) :
a = [ a ]
for s in a :
i = adn.get(s.replace('_', '' ), -1)
if i >= 0 :
return(i)
i = add.get(s.replace('_', '-'), -1)
if i >= 0 :
return(i)
pass
return(-1)
def strrev(s) :
"""
Reverse a string.
Or post or 2.2 (2.3?) return(s[::-1])
"""
a = list(s)
a.reverse()
return(string.join(a, ""))
def strip_c_comments(text) :
"""
Strip C and C++ comments from a string.
From: http://stackoverflow.com/questions/241327/python-snippet-to-remove-c-and-c-comments
"""
def replacer(match) :
s = match.group(0)
if s.startswith('/') :
return("")
return(s)
pattern = re.compile(r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"', re.DOTALL | re.MULTILINE)
return(re.sub(pattern, replacer, text))
def c_string(s) :
"""
Escape a string back to C string form.
Or, anyway, get the normal escaped characters back to their C string form.
This *must* be a built in function somewhere! (repr(), sort of)
"""
s = s.replace("\\", r"\134") # this way there are no doubled slashes
s = s.replace("\'", r"\'")
s = s.replace("\"", r"\"")
s = s.replace("\a", r"\a")
s = s.replace("\b", r"\b")
s = s.replace("\f", r"\f")
s = s.replace("\n", r"\n")
s = s.replace("\r", r"\r")
s = s.replace("\t", r"\t")
s = s.replace("\v", r"\v")
return(s)
def c_ctrl_esc(s) :
"""
Escape a string back to C string form with all control characters and Perl smarties escaped.
"""
def _esc(s) :
return(r"\%03o" % ord(s.group(0)))
s = c_string(s)
s = re.sub(r"[\0-\x1f$@%]", _esc, s) # escape control characters and Perl smarties
return(s)
def unicode_byte_string(s) :
"""
Return a Unicode string as a byte string.
Note: encode puts 2 bytes (converted codecs.BOM_LE), at the start of the string (under Windows or maybe x86).
"""
return(s.encode('utf-16'))
def ascii(s, ac = '_') :
"""
Return the given string after converting all characters over 127 to _ or the given character.
"""
if ac == None :
ac = '_'
astr = ""
if not isinstance(s, basestring) :
s = str(s)
for i in range(0, len(s)) :
c = s[i:i+1]
if ord(c) >= 128 :
c = ac
astr += c
return(astr)
def best_ascii(unicode_s) :
"""
Return the best guess as to ASCII characters that could be used for the given string.
"""
if not isinstance(unicode_s, unicode) :
try :
unicode_s = unicode(unicode_s.decode('utf8'))
except UnicodeDecodeError :
unicode_s = unicode(unicode_s.decode('latin1'))
pass
s = "".join([ unicodedata.normalize("NFKD", c)[0] for c in unicode_s ])
return(s)
def file_name_able(fn) :
"""
Return the given file name with the illegal file name characters stripped from it.
"""
if isinstance(fn, UnicodeType) :
fn = best_ascii(fn).encode('ascii', 'replace')
fn = ascii(fn)
fn = re.sub(r"[\"]", "", fn)
fn = re.sub(r"[\\\/\:\&\^\*\?\<\>\"\'\+]", "_", fn)
return(fn.strip())
printable_re = re.compile(r"[^" + re.escape(string.printable) + r"]")
def printable(s, tochr = "_") :
"""
Return a printable string with non-printable characters converted to underscores or whatever.
"""
tochr = "" if tochr == "" else (tochr or "_")
s = printable_re.sub(tochr, s)
return(s)
def printable_str(s) :
""" Return a string that's printable. """
try :
rs = unicode(s).encode('unicode_escape')
except UnicodeDecodeError :
try :
rs = unicode(s.decode('utf8')).encode('unicode_escape')
except UnicodeDecodeError :
try :
rs = unicode(s.decode('latin1')).encode('unicode_escape')
except UnicodeDecodeError :
rs = repr(s)
pass
pass
return(rs)
def lf_only(fs) :
""" Convert all variants of line breaks to '\n' in a string with multiple text lines. """
fs = re.sub(r"\r+\n", "\n", fs)
fs = re.sub(r"\r", "\n", fs)
return(fs)
## Get rid of leading line feeds (amounts to s.strip("\n"))
strip_first_lines_re = re.compile(r"^\n+", re.DOTALL)
## Get rid of white-space that is at the ends of text lines inside a string.
strip_eol_spaces_re = re.compile(r"\s+$", re.MULTILINE)
def lf_only_with_no_trailing_white_space(fs) :
""" Run a string through lf_only() and strip trailing white space from the lines, too. Insure last lines ends with LF. """
fs = lf_only(fs)
fs = strip_eol_spaces_re.sub("", fs)
fs += "\n"
return(fs)
def no_blank_lines(fs) :
"""
Get rid of any blank or white-space-only lines in a string containing multiple text lines.
Gets rid of white space at the ends of all text lines as a side effect.
The last line is forced to end with an LF.
"""
fs = lf_only(fs)
fs = strip_eol_spaces_re.sub("", fs)
fs += "\n"
fs = re.sub("\n(\s*\n)+", "\n", fs)
fs = strip_first_lines_re.sub("", fs)
return(fs)
def multiline_strip(fs, chrs = None) :
"""
Do a strip on a string with multiple text lines.
If no \n is at the end of 'fs', there will be none at the end of the return value.
CRLF, LF and CR all are considered to be an EOL.
"""
fs = "\n".join( [ s.strip(chrs) for s in re.split(r"\r?\n|\r", fs) ] )
return(fs)
if chrs :
rs = re.escape(chrs)
else :
rs = r"\s"
fs = lf_only(fs)
fs = re.sub(r"(?m)^[" + rs + "]+", "", fs)
fs = re.sub(r"(?m)[" + rs + "]+$", "", fs)
fs = re.sub(r"^[" + rs + "]+", "", fs)
fs = re.sub(r"[" + rs + "]+$", "", fs)
return(fs)
def multiline_flush_left(s) :
"""
Whack left-side spaces shared for all non-blank lines in the given multiline string.
Also whacks right-side white-space.
Also, converts to \n separated lines. No CRs.
"""
if isinstance(s, basestring) :
s = re.split(r"\r*\n", s)
la = [ len(ln) - len(ln.lstrip(' ')) for ln in s if ln.strip() ]
if la :
lc = min(la)
s = [ ln[lc:] if ln.strip() else ln for ln in s ]
s = [ ln.rstrip() for ln in s ]
return("\n".join(s))
_crc32_table = (
0x4dbdf21c,
0x500ae278,
0x76d3d2d4,
0x6b64c2b0,
0x3b61b38c,
0x26d6a3e8,
0x000f9344,
0x1db88320,
-1610256068, # 0xa005713c,
-1112383144, # 0xbdb26158,
-1687465484, # 0x9b6b51f4,
-2032385648, # 0x86dc4190,
-690409300, # 0xd6d930ac,
-881975096, # 0xcb6e20c8,
-306769820, # 0xedb71064,
-268435456, # 0xf0000000
)
_TEST_CRC_VALUE = -1737075662 # 0x98765432
_CRC32_MASK = 0xFFFFffffl
try :
_CRC32_MASK = int(_CRC32_MASK)
except OverflowError :
_CRC32_MASK = -1
INITIAL_CRC32_VALUE = 0 # 0xFFFFffffl
#
# Compute a 32-bit (PKZIP) crc of a string or array.
#
# REMEMBER! The value could easily be construed to be a signed int. Odd things might happen in future versions of python.
# To be "correct" about it, the value should be massaged like "crc = long(crc) & 0xFFFFffffL", yeilding a long value.
#
#
def crc32(current_crc, c) :
c = int(c)
current_crc = (((current_crc >> 4) & 0x0FFFffff) ^ _crc32_table[(current_crc ^ c ) & 0xf]) & _CRC32_MASK;
current_crc = (((current_crc >> 4) & 0x0FFFffff) ^ _crc32_table[(current_crc ^ (c >> 4)) & 0xf]) & _CRC32_MASK;
return(current_crc)
def pure_python_crc32(current_crc, xmem) :
# print "crclen", len(xmem)
for i in range(0, len(xmem)) :
cv = ord(xmem[i:i+1])
current_crc = crc32(current_crc, cv)
# print "%u %08x %02x %s\n" % ( i, current_crc, cv, chr(cv) )
return(current_crc)
def blkcrc32(current_crc, xmem) :
if sys.version < "2.4" :
return(pure_python_crc32(current_crc, xmem)) # maybe 2.3 is ok. I don't know. But 2.4 is ok and 2.2 is not.
current_crc = long(current_crc)
if current_crc & 0x80000000L :
current_crc = (~current_crc + 1) & 0xFFFFffffL
current_crc = -current_crc
return(long(zlib.crc32(xmem, int(current_crc))) & 0xFFFFffffL)
xmcrctab = (
0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50A5, 0x60C6, 0x70E7,
0x8108, 0x9129, 0xA14A, 0xB16B, 0xC18C, 0xD1AD, 0xE1CE, 0xF1EF,
0x1231, 0x0210, 0x3273, 0x2252, 0x52B5, 0x4294, 0x72F7, 0x62D6,
0x9339, 0x8318, 0xB37B, 0xA35A, 0xD3BD, 0xC39C, 0xF3FF, 0xE3DE,
0x2462, 0x3443, 0x0420, 0x1401, 0x64E6, 0x74C7, 0x44A4, 0x5485,
0xA56A, 0xB54B, 0x8528, 0x9509, 0xE5EE, 0xF5CF, 0xC5AC, 0xD58D,
0x3653, 0x2672, 0x1611, 0x0630, 0x76D7, 0x66F6, 0x5695, 0x46B4,
0xB75B, 0xA77A, 0x9719, 0x8738, 0xF7DF, 0xE7FE, 0xD79D, 0xC7BC,
0x48C4, 0x58E5, 0x6886, 0x78A7, 0x0840, 0x1861, 0x2802, 0x3823,
0xC9CC, 0xD9ED, 0xE98E, 0xF9AF, 0x8948, 0x9969, 0xA90A, 0xB92B,
0x5AF5, 0x4AD4, 0x7AB7, 0x6A96, 0x1A71, 0x0A50, 0x3A33, 0x2A12,
0xDBFD, 0xCBDC, 0xFBBF, 0xEB9E, 0x9B79, 0x8B58, 0xBB3B, 0xAB1A,
0x6CA6, 0x7C87, 0x4CE4, 0x5CC5, 0x2C22, 0x3C03, 0x0C60, 0x1C41,
0xEDAE, 0xFD8F, 0xCDEC, 0xDDCD, 0xAD2A, 0xBD0B, 0x8D68, 0x9D49,
0x7E97, 0x6EB6, 0x5ED5, 0x4EF4, 0x3E13, 0x2E32, 0x1E51, 0x0E70,
0xFF9F, 0xEFBE, 0xDFDD, 0xCFFC, 0xBF1B, 0xAF3A, 0x9F59, 0x8F78,
0x9188, 0x81A9, 0xB1CA, 0xA1EB, 0xD10C, 0xC12D, 0xF14E, 0xE16F,
0x1080, 0x00A1, 0x30C2, 0x20E3, 0x5004, 0x4025, 0x7046, 0x6067,
0x83B9, 0x9398, 0xA3FB, 0xB3DA, 0xC33D, 0xD31C, 0xE37F, 0xF35E,
0x02B1, 0x1290, 0x22F3, 0x32D2, 0x4235, 0x5214, 0x6277, 0x7256,
0xB5EA, 0xA5CB, 0x95A8, 0x8589, 0xF56E, 0xE54F, 0xD52C, 0xC50D,
0x34E2, 0x24C3, 0x14A0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405,
0xA7DB, 0xB7FA, 0x8799, 0x97B8, 0xE75F, 0xF77E, 0xC71D, 0xD73C,
0x26D3, 0x36F2, 0x0691, 0x16B0, 0x6657, 0x7676, 0x4615, 0x5634,
0xD94C, 0xC96D, 0xF90E, 0xE92F, 0x99C8, 0x89E9, 0xB98A, 0xA9AB,
0x5844, 0x4865, 0x7806, 0x6827, 0x18C0, 0x08E1, 0x3882, 0x28A3,
0xCB7D, 0xDB5C, 0xEB3F, 0xFB1E, 0x8BF9, 0x9BD8, 0xABBB, 0xBB9A,
0x4A75, 0x5A54, 0x6A37, 0x7A16, 0x0AF1, 0x1AD0, 0x2AB3, 0x3A92,
0xFD2E, 0xED0F, 0xDD6C, 0xCD4D, 0xBDAA, 0xAD8B, 0x9DE8, 0x8DC9,
0x7C26, 0x6C07, 0x5C64, 0x4C45, 0x3CA2, 0x2C83, 0x1CE0, 0x0CC1,
0xEF1F, 0xFF3E, 0xCF5D, 0xDF7C, 0xAF9B, 0xBFBA, 0x8FD9, 0x9FF8,
0x6E17, 0x7E36, 0x4E55, 0x5E74, 0x2E93, 0x3EB2, 0x0ED1, 0x1EF0
)
def crc16(current_crc, c) :
c = int(c)
# Note: Not Xmodem CRC. Xmodem has post processing. We do the Palm CRC: "dcrc /i -1 /Wp"
return((((current_crc) << 8) ^ xmcrctab[(((current_crc) >> 8) ^ c) & 0xff]) & 0xffff)
def pure_python_crc16(current_crc, xmem) :
# print "crclen", len(xmem)
for i in range(0, len(xmem)) :
cv = ord(xmem[i:i+1])
current_crc = crc16(current_crc, cv)
# print "%u %08x %02x %s\n" % ( i, current_crc, cv, chr(cv) )
return(current_crc & 0xffff)
def blkcrc16(current_crc, xmem) :
return(pure_python_crc16(current_crc, xmem))
def xorsum(s) :
"""
Compute the longitudinal parity (the XOR sum) of a string.
"""
cs = 0
for c in s :
cs ^= ord(c)
return(cs)
#
#
# Find the "cosine" of the angle between two vectors.
#
# If either vector is None, then the return value is None.
# If all elements of either of the arrays are zero (or None), then the return value is None.
#
# Undef'd values in the arrays are ignored - that "dimension" is ignored, that is.
#
# $cosine = tz_vector_cosine( [ 1, 2, 3 ], [ 4, -5, 6 ]);
#
# If the vectors are the same, then the return value is 1.
# If the vectors are exactly opposite in direction and magnitude, then the return value is -1.
# If the vectors are exactly at "right angles", then the return value is 0.
#
#
def tz_vector_cosine(v1, v2) :
"""
Return the hyperspace cosine of two vectors.
"""
if (v1 == None) or (v2 == None) : return(None)
sum = 0.0;
ss1 = 0.0;
ss2 = 0.0;
for i in range(0, min(len(v1), len(v2))) :
i1 = v1[i]
if i1 == None : continue
i2 = v2[i]
if i2 == None : continue
i1 = float(i1)
i2 = float(i2)
sum += (i1 * i2)
ss1 += (i1 * i1)
ss2 += (i2 * i2)
if (ss1 == 0.0) or (ss2 == 0.0) :
return(None)
return(sum / math.sqrt(ss1 * ss2))
#
#
# Decode HTML character entities in a string.
#
#
html_entity_re = re.compile(r"\&[^;\r\n\s]+;", re.DOTALL)
def decode_html_entities(s, nbsp_chr = None) :
"""
Decode any HTML character entities in the string to their character counterparts.
"""
if nbsp_chr == None :
nbsp_chr = htmlentitydefs.entitydefs['nbsp']
def _entity_2_chr(g) :
try :
w = g.group(0)[1:-1]
if htmlentitydefs.entitydefs.has_key(w) :
if w == 'nbsp' :
w = nbsp_chr
else :
w = htmlentitydefs.entitydefs[w]
pass
if (w[0:2] == '&#') and (w[-1:] == ';') : # handles ints from htmlentitydefs
w = w[2:-1]
try :
w = chr(int(w))
except ValueError :
w = unichr(int(w))
pass
elif w[0:1] == '#' : # handles normal int values from the passed string
w = w[1:]
try :
w = chr(int(w))
except ValueError :
w = unichr(int(w))
pass
pass
except ValueError :
w = g.group(0)
if len(w) == 1 :
if 128 <= ord(w) < 256 :
w = w.decode('latin1')
elif 256 <= ord(w) :
pass
pass
return(w)
return(html_entity_re.sub(_entity_2_chr, s))
def safe_html(txt) :
"""
Convert the given string in to text that can be put out in an HTML page without worry of it being interpreted as anything but text.
"""
def _decimalfy(s) :
return("&#" + str(ord(s.group(0))) + ";")
txt = txt.replace("&", "&")
txt = txt.replace("<", "<")
txt = txt.replace(">", ">")
txt = lf_only(txt)
txt = re.sub(r"[\r\n]", " ", txt)
txt = re.sub(r"-{6,}", "", txt)
txt = re.sub(r"[^ -\176]", _decimalfy, txt) # escape all characters outside of space-to-tilde
return(txt)
cdata_string_re = re.compile(r"^<\!\[CDATA\[.*\]\]>$")
def maybe_wrap_with_cdata(s) :
""" If needed, and if it's not already wrapped with a CDATA tag, wrap this string with <[!CDATA[...]]> """
if not cdata_string_re.match(s) :
ns = safe_html(s)
if ns != s :
s = "" % ( s.replace("]]>", "]]>") )
pass
return(s)
de_br_re = re.compile(r"]{0,200}>", re.DOTALL + re.IGNORECASE)
de_p_re = re.compile(r"]{0,200}>", re.DOTALL + re.IGNORECASE)
de_hr_re = re.compile(r"]{0,200}>", re.DOTALL + re.IGNORECASE)
de_lfs_re = re.compile(r"<(/?(?:table|tr|dir|li|ol|ul|dt|dl))\b", re.DOTALL + re.IGNORECASE)
de_tab_re = re.compile(r"\t", re.DOTALL + re.IGNORECASE)
de_space_re = re.compile(r" +", re.DOTALL + re.IGNORECASE)
de_lf_re = re.compile(r"\n *\n *(?:\n *)+", re.DOTALL + re.IGNORECASE)
de_lfsp_re = re.compile(r"\n +", re.DOTALL + re.IGNORECASE)
de_splf_re = re.compile(r" +\n", re.MULTILINE)
de_multlf_re = re.compile(r"\n+", re.DOTALL + re.IGNORECASE)
de_script_re = re.compile(r"]*>.*?]*>", re.DOTALL + re.IGNORECASE)
de_html_re = re.compile(r"<[^>]+>", re.DOTALL + re.IGNORECASE)
def de_html_str(s) :
"""
Do cheaply what something like lynx could do from the command line:
Convert a string that contains HTML in to a text string with no HTML markup, but
with the text looking kind of like the HTML would look if rendered as text.
"""
if isinstance(s, ListType) or isinstance(s, TupleType) :
s = "\n".join(s)
s = de_tab_re.sub(" ", s)
s = de_space_re.sub(" ", s)
s = s.replace("\r\n", "\n")
s = s.replace("\r", "\n")
s = de_lf_re.sub("\n\n", s)
s = de_lfsp_re.sub("\n", s)
s = de_splf_re.sub("\n", s)
s = de_multlf_re.sub("\n", s)
s = de_br_re.sub("\n", s)
s = de_p_re.sub("\n\n", s)
s = de_hr_re.sub("\n----------------------------------------\n", s)
s = de_lfs_re.sub(r"\n<\1", s)
s = de_script_re.sub("", s)
s = de_html_re.sub("", s) # remove all HTML tags
s = decode_html_entities(s, nbsp_chr = ' ')
s = s.replace("\x93", '"') # left double quote
s = s.replace("\x91", '`') # left single quote
s = s.replace("\x94", '"') # right double quote
s = s.replace("\x92", "'") # right single quote
s = s.replace("\x9c", "oe")
s = s.replace("\x96", "-") # n-dash
s = s.replace("\x8b", "<")
s = s.replace("\x9b", ">")
s = s.replace("\x8c", "OE")
s = s.replace("\xa0", ' ') # non-break space
s = s.replace("\x7f", ' ') # rubout
s = s.replace("\xad", "-") # soft hyphen
s = s.replace("\xaf", "-") # macron mark
s = s.replace("\xab", "<<")
s = s.replace("\xbb", ">>")
s = de_tab_re.sub(" ", s)
s = de_space_re.sub(" ", s)
s = de_lf_re.sub("\n\n", s)
s = de_lfsp_re.sub("\n", s)
s = de_splf_re.sub("\n", s)
return(s)
def string_pairs(a, skip = 1, connector_str = "\x80_Pr_") :
"""
Return an array of strings composed of each sequential pair in the given array of strings.
"""
if False :
t2 = [ 0 ] * (len(a) - skip)
for i in xrange(len(t2)) :
t2[i] = a[i] + connector_str + a[i + skip]
pass
else :
t2 = [ a[i] + connector_str + a[i + skip] for i in xrange(len(a) - skip) ]
return(t2)
def flat_positional_strings(strings, buckets = 3, ident_str = "\x80_Pf%u_%s") :
"""
Given an array of strings, return an array of the strings with their positions in the array concatenated with them.
For each string, there are two positions so that equal strings in similar positions in two arrays will yield at least one equal position string.
The 'buckets' refers to how many sections of the array are used. Each identical string inside a particular bucket/section will yield at least one positional string that's the same.
The first positional string will identify the bucket the string is in.
The second positional string will identify which pair of buckets the string is in and nearest.
The 'ident_str' must contain a %u and a %s in that order.
The %u will be filled in with the bucket or half-bucket number.
The %s will be filled in with the string.
"""
if len(strings) == 0 :
return( [] )
pstrs = [ 0 ] * (len(strings) * 2)
def _fill(pi, ni, ne, n) :
# print pi, ni, ne, n
for i in xrange(ni, min(ne, len(strings))) :
pstrs[pi] = ident_str % ( n, strings[i] )
pi += 1
return( ( pi, ne ) )
d = min((len(strings) + buckets - 1) / buckets, 2)
d2 = (d + 1) / 2
i = pi = 0
ni = 0
while ni < len(strings) :
(pi, ni) = _fill(pi, ni, ni + d2, i)
d2 = d
i += 1
ni = 0
while ni < len(strings) :
(pi, ni) = _fill(pi, ni, ni + d2, i)
i += 1
return(pstrs)
def golden_smaller(v) :
return(v * (1.0 / 1.61803399))
def log_positional_strings(strings, log_mult = 2.0, ident_str = "\x80_Pl_%u_%s") :
"""
Given an array of strings, return an array of the strings with their positions in the array concatenated with them.
For each string, create a string concatenated with the log of the position of the string in 'strings' multiplied by 'log_mult'.
The 'ident_str' must contain a %u and a %s in that order.
The %u will be filled in with the string's position in 'strings'
The %s will be filled in with the string.
"""
sa = [ ident_str % ( int(math.log(i + 1) * log_mult), strings[i] ) for i in xrange(len(strings)) ]
return(sa)
def linear_regression(ax = None, ay = None):
"""
Given an array of ax and ay values,
return the values that make a linear regression line,
(m * ax[i]) + a = ay[i]
"""
ay = ay or []
if not ax :
ax = range(len(ay))
if (not isinstance(ax, ListType)) and (not isinstance(ax, TupleType)) :
ax = [ x + ax for x in xrange(len(ay)) ]
if len(ax) != len(ay) :
raise ValueError, 'Different length of ax and ay array'
ln = len(ax)
sx = sy = sxx = syy = sxy = 0.0
for i in xrange(ln) :
x = ax[i]
y = ay[i]
sx = sx + x
sy = sy + y
sxx = sxx + (x * x)
syy = syy + (y * y)
sxy = sxy + (x * y)
d = (sxx * ln) - (sx * sx)
m = ((sxy * ln) - (sy * sx )) / d
a = ((sxx * sy) - (sx * sxy)) / d
return( ( m, a ) )
def y_from_distance_direction(dist, direction) :
"""
Get the Y value from a distance and a direction (direction is -pi..pi, where both pi's are at y<0:x=0 and 0 direction is y=0:x>=0)
"""
return(dist * math.sin(direction))
def x_from_distance_direction(dist, direction) :
"""
Get the X value from a distance and a direction (direction is -pi..pi, where both pi's are at y<0:x=0 and 0 direction is y=0:x>=0)
"""
return(dist * math.cos(direction))
def distance_from_x_y(x, y) :
"""
Get the distance from an X and Y value.
This routine is here for documentation purposes, folks.
"""
return(math.hypot(y, x))
def direction_from_x_y(x, y) :
"""
Get the direction from an X and Y value.
This routine is here for documentation purposes, folks.
"""
return(math.atan2(y, x))
p2 = math.pi * 2.0
def radian_angle_difference(a, b) :
"""
What's the angle between these two angles (in radians. that is, += 0..pi - to get degrees, math.degrees(a))?
"""
a = (a - b) % p2
if a > math.pi :
a -= p2
if a < -math.pi :
a += p2
return(a)
def point_on_line(xy, xyxy, fudge = None) :
""" Is the given (x, y) on infinite ( (x1,y1), (x2,y2) ) or finite line if 'fudge' is non-None? """
x = xy[0]
y = xy[1]
x1 = xyxy[0][0]
y1 = xyxy[0][1]
x2 = xyxy[1][0]
y2 = xyxy[1][1]
if fudge != None :
if ( False
or (not (min(x1, x2) - fudge <= x <= max(x1, x2) + fudge))
or (not (min(y1, y2) - fudge <= y <= max(y1, y2) + fudge))
) :
return(False)
pass
if x2 == x1 : # special case vertical line
return(abs(x - x1) <= (fudge or 0.0))
sl = float(y2 - y1) / float(x2 - x1)
zy = y1 - (sl * x1)
return(abs((sl * x) + zy - y) <= (fudge or 0.0))
#
# From: http://www.topcoder.com/tc?module=Static&d1=tutorials&d2=geometry2
# See also: http://stackoverflow.com/questions/563198/how-do-you-detect-where-two-line-segments-intersect
# And: http://en.wikipedia.org/wiki/Bentley%E2%80%93Ottmann_algorithm
#
def convert_two_points_to_ABC(x1, y1, x2, y2) :
""" Find the values of ABC in "A*x1 + B*y1 = C" given two points. """
A = float(y2 - y1)
B = float(x1 - x2)
C = float((A * x1) + (B * y1))
return(A, B, C)
def get_line_intersection(xy1, xy2, fudge = None) :
"""
Find the intersection point between two lines. Return (None,None) if lines are parallel.
If 'fudge' is non-None, then do strict comparison (subject to 'fudge' slop).
That is, the intersection must be between the points
on the lines that go from point to point, not-infinite length.
And, if the lines are parallel find an intersection point
if the lines overlap, 'fudge' logic accounted for.
"""
( A1, B1, C1 ) = convert_two_points_to_ABC(xy1[0][0], xy1[0][1], xy1[1][0], xy1[1][1])
( A2, B2, C2 ) = convert_two_points_to_ABC(xy2[0][0], xy2[0][1], xy2[1][0], xy2[1][1])
det = (A1 * B2) - (A2 * B1)
if not det :
if point_on_line(xy1[0], xy2, fudge) : return(float(xy1[0][0]), float(xy1[0][1]))
if point_on_line(xy1[1], xy2, fudge) : return(float(xy1[1][0]), float(xy1[1][1]))
if point_on_line(xy2[0], xy1, fudge) : return(float(xy2[0][0]), float(xy2[0][1]))
if point_on_line(xy2[1], xy1, fudge) : return(float(xy2[1][0]), float(xy2[1][1]))
return(None, None)
x = ((B2 * C1) - (B1 * C2)) / det
y = ((A1 * C2) - (A2 * C1)) / det
if fudge != None :
if ( False
or (not (min(xy1[0][0], xy1[1][0]) - fudge <= x <= max(xy1[0][0], xy1[1][0]) + fudge))
or (not (min(xy1[0][1], xy1[1][1]) - fudge <= y <= max(xy1[0][1], xy1[1][1]) + fudge))
or (not (min(xy2[0][0], xy2[1][0]) - fudge <= x <= max(xy2[0][0], xy2[1][0]) + fudge))
or (not (min(xy2[0][1], xy2[1][1]) - fudge <= y <= max(xy2[0][1], xy2[1][1]) + fudge))
) :
return(None, None)
pass
return(x, y)
def safe_relpath(fn) :
"""
os.path.relpath(), returning the given file name if there is a problem with it (if it's "" or None).
"""
try :
fn = os.path.relpath(fn)
except ValueError : # happens if the file name is "" or None
pass
return(fn)
def same_file(f1, f2) :
"""
Are these the same files?
"""
try :
return(os.path.samefile(f1, f2))
except ( AttributeError, OSError, IOError ) : # attrib? samefile() does not exist under Unix. OS/IO? File doesn't exist.
f1 = os.path.abspath(f1)
f2 = os.path.abspath(f2)
if f1 == f2 :
return(True)
pass
return(False)
def _read_whole_file(fname, how = "t") :
""" Read a whole file. """
fi = open(fname, "r" + how)
fs = fi.read()
fi.close()
return(fs)
def read_whole_text_file(fname) :
""" Read a whole text file. """
return(_read_whole_file(fname, "t"))
def read_whole_binary_file(fname) :
""" Read a whole binary file. """
return(_read_whole_file(fname, "b"))
def _safe_read_whole_file(fn, rtn) :
try :
return(rtn(fn))
except ( OSError, IOError ) :
pass
return(None)
def safe_read_whole_text_file(fn) :
return(_safe_read_whole_file(fn, read_whole_text_file))
def safe_read_whole_binary_file(fn) :
return(_safe_read_whole_file(fn, read_whole_binary_file))
def _write_whole_file(fname, how, s) :
""" Write a whole file. """
if isinstance(s, UnicodeType) :
s = unicode_byte_string(s)
fi = open(fname, "w" + how)
fi.write(s)
fi.close()
del(fi)
def write_whole_text_file(fname, s) :
""" Write a whole text file. """
if isinstance(s, UnicodeType) :
s = s.encode('utf-8')
return(_write_whole_file(fname, "t", s))
def write_whole_binary_file(fname, s) :
""" Write a whole binary file. """
return(_write_whole_file(fname, "b", s))
def _safe_write_whole_file(fn, fd, rtn) :
try :
import replace_file
except ImportError :
replace_file = None
tfn = fn + ".tmp"
try :
rtn(tfn, fd)
except ( OSError, IOError ) :
return(False)
bfn = fn + ".bak"
if replace_file and None :
try :
replace_file.replace_file(fn, tfn, bfn)
except ( OSError, IOError ) :
return(False)
pass
else :
try :
if os.path.exists(bfn) :
os.unlink(bfn)
if os.path.exists(fn) :
os.rename(fn, bfn)
os.rename(tfn, fn)
except ( OSError, IOError ) :
return(False)
pass
return(True)
def safe_write_whole_text_file(fn, fd) :
return(_safe_write_whole_file(fn, fd, write_whole_text_file))
def safe_write_whole_binary_file(fn, fd) :
return(_safe_write_whole_file(fn, fd, write_whole_binary_file))
def whack_file(fn) :
""" Safely make sure the given file is no longer with us. """
try :
os.unlink(fn)
except ( OSError, IOError ) :
pass
pass
def elapsed_time() :
""" Return a running clock value based on some arbitrary "zero". """
if sys.platform == 'win32' :
return(time.clock()) # as an alternate: win32api.GetTickCount()
# Mac could use MacOS.GetTicks() / 60.0, I suppose.
return(os.times()[4]) # under windows this value stays at zero
def wdhms_str(t, weeks = False) :
""" Return a string that represents the shortest weeks:days:hours:minutes:seconds.fraction for the given time. """
def _nxt(s, t, d, fm = '%02u:') :
tt = t / d
if tt or s :
s += (fm % tt)
return(s, t - (int(tt) * d))
t = float(t)
s = ""
if weeks :
( s, t ) = _nxt(s, t, 60 * 60 * 24 * 7, fm = "%u:" )
( s, t ) = _nxt(s, t, 60 * 60 * 24 , fm = "%1u:" )
( s, t ) = _nxt(s, t, 60 * 60 )
( s, t ) = _nxt(s, t, 60 )
if t == int(t) :
( s, t ) = _nxt(s, t, 1 , fm = "%02u" )
else :
( s, t ) = _nxt(s, t, 1 , fm = "%09.6f")
s = re.sub(r"^[0\:]+", "", s)
s = re.sub(r"[0\.]+$", "", s)
if s.endswith(':') :
s += "00"
if re.search(r":\d$", s) :
s += "0"
return(s)
def get_ini_or_cfg_file_name(base_name, app_name = None) :
""" Return a file name of config_directory/base_name.ini or .cfg, appropriate to the OS. And make the config directory for the app. """
base_name = base_name or "tzlibpy"
app_name = os.path.splitext(os.path.basename(app_name or base_name))[0]
if sys.platform == 'win32' :
app_path = os.path.normpath(os.path.join(os.path.expandvars("${appdata}"), app_name))
if not os.path.isdir(app_path) :
os.makedirs(app_path)
return(os.path.join(app_path, base_name + ".ini"))
app_path = os.path.join(os.path.expanduser("~"), "." + app_name)
if not os.path.isdir(app_path) :
os.makedirs(app_path)
return(os.path.join(app_path, base_name + ".cfg"))
def temp_file_name(base_name = None, app_name = None, ext = None) :
ext = ext or ".tmp"
if (not app_name) or (not base_name) :
td = os.environ.get("TEMP", None)
td = td or os.environ.get("TMP", None)
td = td or "."
fn = os.path.join(td, "tmp_tzlibpy" + ext)
else :
fn = get_ini_or_cfg_file_name(base_name, app_name)
while True :
rs = "_%08x" % ( random.randint(0, 2000000000) )
rfn = os.path.splitext(fn)[0] + rs + ext
if not os.path.isfile(rfn) :
break
pass
return(rfn)
def s_except_1(v) :
""" Return an "s" if the 'v' is not 1. """
if isinstance(v, (TupleType, ListType, DictionaryType ) ) :
v = len(v)
if v != 1 :
return("s")
return("")
def same_object(o1, o2) :
""" Return whether these two things are the same, exact thing (in memory). """
return(id(o1) == id(o2))
def sys_err_file_line(sys_exc_info_2 = None) :
""" Return a simple, one-line string with the bottom line about where the latest except failure has been. """
sys_exc_info_2 = sys_exc_info_2 or sys.exc_info()[2]
lns = traceback.format_tb(sys_exc_info_2)[-1].strip()
lns = re.sub(r"[\r\n]+", " ---- ", lns)
return(lns)
def bool_to_0_or_1(b) :
""" Return 0 or 1 depending upon the given bool's value. """
return((b or 0) and 1)
def excel_column_name(c) :
if c < 0 :
raise ValueError("%d<0" % c)
if c >= 26 * 27 :
raise ValueError("%d>max" % c)
cc = chr(ord('a') + c % 26)
if c >= 26 :
cc = chr(ord('a') + (c / 26) - 1) + cc
return(cc)
def find_upper_dir(to_find_dir, our_dir = None) :
our_dir = our_dir or "."
our_dir = os.path.abspath(our_dir)
sd = our_dir
while sd and (not sd.endswith(os.path.normpath("/"))) :
( p, dn ) = os.path.split(sd)
if dn == to_find_dir :
return(sd)
if not dn :
sd = ""
break
sd = p
sd = our_dir
while sd and (not sd.endswith(os.path.normpath("/"))) :
p = os.path.join(sd, to_find_dir)
if os.path.isdir(p) :
return(p)
sd = os.path.split(sd)[0]
return("")
def find_upper_file_or_dir(to_find, our_dir = None) :
our_dir = our_dir or "."
our_dir = os.path.abspath(our_dir)
to_find = os.path.normpath(to_find)
sd = our_dir
while sd :
p = os.path.join(sd, to_find)
if os.path.exists(p) :
return(p)
if sd.endswith(os.path.normpath("/")) :
break
sd = os.path.split(sd)[0]
return("")
def file_signature(file_name) :
""" Return the CRC of the file's name, size and date/time. """
return(blkcrc32(INITIAL_CRC32_VALUE, file_name + str(os.path.getsize(file_name)) + str(os.path.getmtime(file_name))))
def unpickle_file(fn) :
fd = safe_read_whole_binary_file(fn)
if fd != None :
try :
return(cPickle.loads(fd))
except ( cPickle.PickleError, TypeError, AttributeError ) :
pass
pass
return(None)
def pickle_file(fn, o, proto = 0) :
try :
fd = cPickle.dumps(o, protocol = min(cPickle.HIGHEST_PROTOCOL, max(0, proto or 0)))
except ( cPickle.PickleError, TypeError, AttributeError ) :
return(False)
return(safe_write_whole_binary_file(fn, fd))
def base_36_encode(number, alphabet = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ') :
'''
Convert positive integer to a base36 string.
Modified: http://en.wikipedia.org/wiki/Base_36
'''
if not isinstance(number, (int, long)) :
number = int(number) # let the chips fall where they may in the case of floats and others
if number == 0 :
return('0')
base36 = ''
sign = ''
if number < 0:
sign = '-'
number = -number
while number != 0 :
number, i = divmod(number, len(alphabet))
base36 = alphabet[i] + base36
return(sign + base36)
def base_36_decode(number) :
return(int(number,36))
def q_get(q, timeout = None) :
"""
Get from a Queue.Queue(), but with a single parameter.
timeout == None wait forever
timeout == 0 non-blocking
otherwise wait for timeout time.
Return None if timeout or nothing in queue.
"""
try :
if timeout == None :
r = q.get()
elif not timeout :
r = q.get(False)
else :
r = q.get(True, timeout)
pass
except Queue.Empty :
r = None
return(r)
def reroute_stdout_err(argv) :
"""
According to command line parameters, reroute stdout and stderr (for use at the top of .exe files made with py2exe).
"""
try :
import replace_file
except ImportError :
replace_file = None
stdout_name = None
stderr_name = None
ai = 0
while ai < len(argv) :
arg = argv[ai]
if arg in [ "--std_out", "--std-out", "--stdout", "--so", "-so", ] :
del(argv[ai])
stdout_name = argv.pop(ai)
elif arg in [ "--std_err", "--std-err", "--stderr", "--se", "-se", ] :
del(argv[ai])
stderr_name = argv.pop(ai)
elif arg in [ "--std_out_err", "--std-err-out", "--stdouterr", "--std_err_out", "--std-out-err", "--stderrout", "--soe", "-soe", "--seo", "-seo", "--eo", "-eo", "--oe", "-oe", ] :
del(argv[ai])
stderr_name = argv.pop(ai)
stdout_name = stderr_name
else :
ai += 1
pass
f_out = None
f_err = None
if stdout_name and stderr_name :
if replace_file :
replace_file.replace_file(stdout_name, None, stdout_name + ".bak")
f_out = open(stdout_name, "w")
if same_file(stdout_name, stderr_name) :
f_err = f_out
else :
if replace_file :
replace_file.replace_file(stderr_name, None, stderr_name + ".bak")
f_err = open(stderr_name, "w")
pass
elif stdout_name :
if replace_file :
replace_file.replace_file(stdout_name, None, stdout_name + ".bak")
f_out = open(stdout_name, "w")
elif stderr_name :
if replace_file :
replace_file.replace_file(stderr_name, None, stderr_name + ".bak")
f_err = open(stderr_name, "w")
if f_out : sys.stdout = f_out
if f_err : sys.stderr = f_err
return(f_out, f_err)
def close_rerouted_stdout_err(f_out, f_err) :
"""
Close the files (or None's) returned by reroute_stdout_err().
"""
if f_out : f_out.close()
if f_err and (not same_object(f_out, f_err)) : f_err.close()
class a_kalman_filter(object) :
def __init__(me, process_variance = 0.01, measurement_variance = 0.04, initial_value = 0.0) :
me.pv = process_variance or 0.01
me.mv = measurement_variance or 0.04
me.guess = initial_value or 0.0 # the current best guess of the output value
me.offset = 1.0 # our tracker, so to speak. How much we juice the process_variance the next measurement.
def next(me, measurement) :
pm = me.offset + me.pv # make the current process variance, sort of
mult = pm / (pm + me.mv) # make the current tracking multiplier
me.guess += (mult * (measurement - me.guess)) # track the measurement
me.offset = (1 - mult) * pm # update our tracker (notice that if we think the measurement is perfect, we go back to the initial state)
return(me.guess)
# a_kalman_filter
def weighted_choice(wates) :
""" Given a list of (positive) weights, yield a randomly chosen index (like random.choice()), likely to be of a higher weight. """
wates = [ max(0, w) for w in wates ]
if not max(wates) :
wates = [ 1 ] * len(wates)
mxw = max(wates) * 2.0
ii = int(random.random() * len(wates))
w = 0.0
while True :
w += (random.random() * mxw)
while w > wates[ii]:
w -= wates[ii]
ii = (ii + 1) % len(wates)
yield(ii)
pass
def cmp_str_with_ints(a, b) :
"""
For use in numbered file name sorting, compare two strings with integer values in them.
E.g. [ 'fn_1_1.txt', 'fn_1_9.txt', 'fn_1_10.txt', 'fn_1_100.txt', 'fn_1_20.txt', ].sort(tzlib.cmp_str_with_ints).
"""
na = re.findall(r'\d+', a)
nb = re.findall(r'\d+', b)
if len(na) and (len(na) == len(nb)) :
a = [ int(d) for d in na ]
b = [ int(d) for d in nb ]
return(cmp(a, b))
def cmp_lower_str_with_ints(a, b) :
"""
For use in numbered file name sorting, compare two strings with integer values in them.
E.g. [ 'fn_1_1.txt', 'fn_1_9.txt', 'fn_1_10.txt', 'fn_1_100.txt', 'fn_1_20.txt', ].sort(tzlib.cmp_str_with_ints).
Lower case version.
"""
a = a.lower()
b = b.lower()
na = re.findall(r'\d+', a)
nb = re.findall(r'\d+', b)
if len(na) and (len(na) == len(nb)) :
a = [ int(d) for d in na ]
b = [ int(d) for d in nb ]
return(cmp(a, b))
#
#
# Test
#
#
if __name__ == '__main__' :
ctm = elapsed_time()
print "%20.10f" % ( ctm )
a = [ 1, 2, 3 ]
d = make_dictionary(a)
print d[1], d[2], d[3]
d = make_dictionary(a, "17")
b = d.keys()
a.sort()
b.sort()
for i in range(0, max(len(a), len(b))) :
if a[i] != b[i] :
s = "make_dictionary a != b %d" % i
raise ValueError(s)
pass
a = [ 10, 11, 12, 13, 14 ]
d = make_index_dictionary(a)
for i in range(max(len(a), len(d))) :
if i != d[a[i]] :
s = "make_index_dictionary i != d[a[i]] %d" % i
raise ValueError(s)
pass
d = { "xyzzy" : "y", "a" : "baby" }
rd = invert_dictionary(d)
if rd != { "y" : "xyzzy", "baby" : "a" } :
s = "invert_dictionary(%s) is %s" % ( repr(d), repr(rd) )
raise ValueError(s)
d = { "x" : "y", "a" : "y" }
rd = invert_dictionary(d, dupable_values = { 'y' : 'x' })
if rd != { "y" : "x" } :
s = "invert_dictionary(%s) is %s" % ( repr(d), repr(rd) )
raise ValueError(s)
d = { "x" : "y", "a" : "y" }
rd = invert_dictionary(d, dupable_values = { 'y' : 'a' })
if rd != { "y" : "a" } :
s = "invert_dictionary(%s) is %s" % ( repr(d), repr(rd) )
raise ValueError(s)
try :
d = { "x" : "y", "a" : "y" }
rd = invert_dictionary(d)
raise ValueError("invert_dictionary(%s) happy" % repr(d))
except IndexError :
pass
a = [ [ 'a', 1 ], [ 'b', 2 ], ]
rd = buples_to_dictionary(a)
if len(rd) != 2 :
s = "buples_to_dictionary not 2 long! %s" % rd
raise ValueError(s)
if (rd['a'] != 1) or (rd['b'] != 2) :
s = "buples_to_dictionary values wrong! %s" % rd
raise ValueError(s)
d = { "xYz" : 5, "zyZ" : 10, "yyz" : 15, }
update_all_case_keys(d)
if (
(d['xyz'] != 5)
or (d['XYZ'] != 5)
or (d['xYz'] != 5)
or (d['zyz'] != 10)
or (d['ZYZ'] != 10)
or (d['zyZ'] != 10)
or (d['yyz'] != 15)
or (d['YYZ'] != 15)
or (len(d) != 8)
) :
s = "update_all_case_keys " + str(d)
raise ValueError(s)
crc = blkcrc32(_TEST_CRC_VALUE, "now is the time")
crc = long(crc) & 0xFFFFffffL
print "crc=0x%08lx %lu" % ( crc, crc )
if crc != 0xa458b82eL :
s = "crc32 [%s] is wrong!" % ( str(crc) )
raise ValueError(s)
crc = blkcrc32(0x98765431L, "now is the time")
crc = crc & 0xFFFFffffL
print "crc=0x%08lx %lu" % ( crc, crc )
if crc != 0xb525d257L :
s = "crc32 [%s] is wrong!" % ( str(crc) )
raise ValueError(s)
crc = 0
bts = 0
for i in range(1000) :
s = "%u %f" % ( i, random.random() )
pc = pure_python_crc32(crc, s)
bts |= crc
nc = blkcrc32(crc, s)
if pc != nc :
s = "crc mismatch 'tween pure and zlib (%s (%08lx:%u) %08lx:%u != %08lx:%u)!" % ( s, crc, crc, pc, pc, nc, nc )
raise ValueError(s)
if random.random() >= 0.5 :
crc = pc
else :
crc = nc
if (i > 100) and ((bts & 0xFFFFffffL) == 0xFFFFffffL) :
break
pass
print "bcrc = %08x" % ( crc )
crc = blkcrc16(_TEST_CRC_VALUE, "now is the time")
print "crc=0x%04x %u" % ( crc, crc )
if crc != 0xe6eb :
s = "crc16 [0x%04x %s] is wrong!" % ( crc, str(crc) )
raise ValueError(s)
crc = blkcrc16(0x1234L, "now is the time")
print "crc=0x%04x %u" % ( crc, crc )
if crc != 0x11ee :
s = "crc16 [0x%04x %s] is wrong!" % ( crc, str(crc) )
raise ValueError(s)
vls = [
[ 0, '0', ],
[ 1, '1', ],
[ -1, '-1', ],
[ 1234, 'YA', ],
[ -1234, '-YA', ],
[ 123456787901234, '17RF9JYFQQ', ],
[ -123456787901234, '-17RF9JYFQQ', ],
[ 1234.9, 'YA', ],
[ -1234.9, '-YA', ],
]
for v, vs in vls :
s = base_36_encode(v)
if s != vs :
raise ValueError("base_36 of %s is %s" % ( str(v), str(s), ) )
s = base_36_decode(s)
if s != int(v) :
raise ValueError("un-base_36 of %s is %s" % ( str(v), str(s), ) )
pass
if bool_to_0_or_1(False) != 0 :
raise ValueError("bool_to_0_or_1(False) == %s" % bool_to_0_or_1(False))
if bool_to_0_or_1(True) != 1 :
raise ValueError("bool_to_0_or_1(True) == %s" % bool_to_0_or_1(True))
if array_find( [ 1, 2, 3 ], 2) != 1 :
s = "1st array_find != 1"
raise ValueError(s)
if array_find( [ 1, 2, 3 ], [ 2, 1 ] ) != 1 :
s = "2nd array_find != 1"
raise ValueError(s)
if array_find( [ 1, 2, 3 ], [ 1, 2 ] ) != 0 :
s = "array_find != 0"
raise ValueError(s)
if array_find( [ 1, 2, 3 ], [ 4, 5, 6, 7 ] ) >= 0 :
s = "array_find >= 0"
raise ValueError(s)
v = [ [], [ 1, 2, 3 ], "4", [ 5, ], "", [ 6, [ 7, [ [], [ 8, 9, ], [] ], [10], [], [ 11, 12, 13, ] ], '14', ], 15, [], ]
if flatten_array(v) != [ 1, 2, 3, "4", 5, "", 6, 7, 8, 9, 10, 11, 12, 13, "14", 15 ] :
s = "flatten_array %s" % flatten_array(v)
raise ValueError(s)
if find_arg( "x" , "y") >= 0 :
raise ValueError("find_arg found y in x")
if find_arg([ "x", ], [ "y", ]) >= 0 :
raise ValueError("find_arg found [y] in [x]")
if find_arg( "x" , "_x_" ) < 0 :
raise ValueError("find_arg not found _x_ in x ")
if find_arg([ "x", ], [ "_x_", ]) < 0 :
raise ValueError("find_arg not found [_x_] in [x]")
if find_arg([ "-x-", ], [ "_x_", ]) < 0 :
raise ValueError("find_arg not found [_x_] in [-x-]")
if find_arg([ "b", "--x_y", ], [ "-a", "--xy", ]) < 0 :
raise ValueError("find_arg not found [--xy] in [--x_y]")
if find_arg([ "b", "--x-y", ], [ "-a", "--x_y", ]) < 0 :
raise ValueError("find_arg not found [--x_y] in [--x-y]")
if find_arg([ "b", "--xy", ], [ "-a", "--x_y", ]) < 0 :
raise ValueError("find_arg not found [--x_y] in [--xy]")
if strrev("x") != "x" :
s = "strrev of 'x' is " + strrev("x")
raise ValueError(s)
if strrev("xy") != "yx" :
s = "strrev of 'xy' is " + strrev("xy")
raise ValueError(s)
if strrev("xyz") != "zyx" :
s = "strrev of 'xyz' is ", strrev("xyz")
raise ValueError(s)
s = file_name_able("abc:^%&\"\';?*blah")
if s != "abc__%__;__blah" :
raise ValueError("file_name_able not right! [%s]" % s)
s = file_name_able(u"\xe0bc:^%&\"\';?*blah")
if s != u"abc__%__;__blah" :
raise ValueError("file_name_able not right! [%s]" % s)
s = safe_relpath(None)
if s != None :
raise ValueError("safe_relpath of None! [%s]" % str(s))
s = safe_relpath("")
if s != "" :
raise ValueError("safe_relpath of ''! [%s]" % str(s))
s = safe_relpath("./x.y")
if s != "x.y".replace('/', os.path.sep) :
raise ValueError("safe_relpath of ./x.y! [%s]" % str(s))
s = safe_relpath("../x.y")
if s != "../x.y".replace('/', os.path.sep) :
raise ValueError("safe_relpath of ../x.y! [%s]" % str(s))
if same_file("tzlib.py", "tz_lib_test.py") :
raise ValueError('same_file("tzlib.py", "tz_lib_test.py")' is True)
if not same_file("tzlib.py", "tzlib.py") :
raise ValueError('same_file("tzlib.py", "tzlib.py")' is False)
if not same_file("../tzpython/tzlib.py", "tzlib.py") :
raise ValueError('same_file("../tzpython/tzlib.py", "tzlib.py")' is False)
if not can_run_program('shipit.py') :
s = "Cannot run ship.py, apparently!"
raise ValueError(s)
if can_run_program('xxx.yyy') :
s = "Can run xxx.yyy, apparently!"
raise ValueError(s)
if sys.platform != 'win32' :
if can_run_program('tzlib.py') :
s = "Can run tzlib.py, apparently!"
raise ValueError(s)
pass
( r, rs ) = run_program("python factor.py %u" % ( 2 * 3 * 5 * 7 * 11 * 13 ) )
if r :
raise ValueError("Failure to run_program('python factory.py') : %s", str(r))
if rs.find("30030") < 0 :
raise ValueError("run_program('python factory.py') failed to print result")
print rs
print "Dir of *.py files in this directory:"
files = ambiguous_file_list("*.py")
print files
print "Dir of *.py files in this directory and sub-dirs:"
files = ambiguous_file_list("*.py", True)
print files
if tz_vector_cosine( [ 1, 2, 3 ], [ 4, -5, 6 ]) != 0.365486942323903610 :
s = "Cosine problem %2.50f" % ( tz_vector_cosine( [ 1, 2, 3 ], [ 4, -5, 6 ]) )
raise ValueError(s)
s = " ".join(string_pairs( [ "a", "bb", "c", "d" ], 1, "x"))
if s != "axbb bbxc cxd" :
s = "string_pairs problem: " + s
raise ValueError(s)
s = " ".join(flat_positional_strings( [ "a", "bb", "c", "d" ], 3, "_%u_%s_"))
if s != "_0_a_ _1_bb_ _1_c_ _2_d_ _3_a_ _3_bb_ _4_c_ _4_d_" :
s = "flat_positional_strings problem: " + s
raise ValueError(s)
s = " ".join(log_positional_strings( [ "a", "bb", "c", "d", "e", "f", "g", "h", "i", "j" ], 2.0, "_%u_%s_"))
if s != "_0_a_ _1_bb_ _2_c_ _2_d_ _3_e_ _3_f_ _3_g_ _4_h_ _4_i_ _4_j_" :
s = "log_positional_strings problem: " + s
raise ValueError(s)
s = " ".join(log_positional_strings( [ "a", "bb", "c", "d", "e", "f", "g", "h", "i", "j" ], 3.0, "_%u_%s_"))
if s != "_0_a_ _2_bb_ _3_c_ _4_d_ _4_e_ _5_f_ _5_g_ _6_h_ _6_i_ _6_j_" :
s = "log_positional_strings problem: " + s
raise ValueError(s)
( m, a ) = linear_regression( ( -2, -1, 0, 1, 2, 3 ), ( -1, 0, 1, 2, 3, 4 ) )
if (m != 1.0) or (a != 1.0) :
s = "Line: m=" + str(m) + " a=" + str(a)
raise ValueError(s)
( m, a ) = linear_regression( None, ( -1, 1, 3, 5, 7 ) )
if (m != 2.0) or (a != -1.0) :
s = "NoneX: m=" + str(m) + " a=" + str(a)
raise ValueError(s)
( m, a ) = linear_regression( 1, ( -1, 1, 3, 5, 7 ) )
if (m != 2.0) or (a != -3.0) :
s = "OneX: m=" + str(m) + " a=" + str(a)
raise ValueError(s)
( m, a ) = linear_regression( ( -6, -5, -4, -3, -2, -1, 0, 1, 2, 3, 4, 5, 6 ), ( -4, -5, -2, -3, 0, -1, 1, 3, 2, 5, 4, 7, 6 ) )
if (int(round(m * 1000.0)) != 967) or (a != 1.0) :
s = "Jag: m=" + str(m) + " a=" + str(a)
raise ValueError(s)
s = strip_c_comments("""/* now is the time
*/ for all "/* */" fun "//" blah // testing
// more test
and finally/**/ this end
""")
if s != ' for all "/* */" fun "//" blah \n\nand finally this end\n' :
s = repr(s)
raise ValueError(s)
s = decode_html_entities("hearts:♥ sigma:σ oacute=ó gt=> lt=<").encode('utf8')
ss = ("hearts:" + unichr(0x2665) + " sigma:" + unichr(0x3c3) + " oacute=" + "\xf3".decode('latin1') + " gt=> lt=<").encode('utf8')
if s != ss :
s = "decode_html_entities problem: " + s
raise ValueError(s)
s = safe_html("\x00&<><>&~\x7f\x80\xff#\r\n-----\r\n------\r\n-------\r\n-\r-\n-\r\n")
if s != "�&<><>&~€ÿ# ----- - - - " :
s = "safe_html problem: " + s
raise ValueError(s)
s = printable(" !@#$%^&*()_-|\\{}[];:'\x22<>,.?/`~abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890 \x00\x01\x02\x1f\x7f\x80\xff z", "~+~")
if s != " !@#$%^&*()_-|\\{}[];:'\x22<>,.?/`~abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890 ~+~~+~~+~~+~~+~~+~~+~ z" :
s = "printable problem: " + s
raise ValueError(s)
s = printable(" !@#$%^&*()_-|\\{}[];:'\x22<>,.?/`~abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890 \x00\x01\x02\x1f\x7f\x80\xff z")
if s != " !@#$%^&*()_-|\\{}[];:'\x22<>,.?/`~abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890 _______ z" :
s = "printable problem: " + s
raise ValueError(s)
s = printable_str(" !@#$%^&*()_-|{}[];:'\x22<>,.?/`~abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890 \x00\x01\x02\x1f\x7f\x80\xff z")
ss = r""" !@#$%^&*()_-|{}[];:'"<>,.?/`~abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890 \x00\x01\x02\x1f\x7f\x80\xff z"""
if s != ss : # note: the backslash, "\\" is interpreted or converted differently in Python 2.4 and 2.5. Arrgh.
for ci in range(min(len(ss), len(s))) :
if ss[ci] != s[ci] :
print "char mismatch %u [%s]!=[%s]" % ( ci, s[ci], ss[ci] )
break
pass
s = "printable_str problem: %s %u %u" % ( s, len(s), len(ss) )
raise ValueError(s)
s = c_string("[\\ \' \" \a \b \f \n \r \t \v \\]")
if s != r"[\134 \' \" \a \b \f \n \r \t \v \134]" :
s = "c_string: " + repr(s)
raise ValueError(s)
s = c_ctrl_esc("[\\ [null\0null] [1f\x1f1f] [at@at] [dollar$dollar] [percent%percent] \' \" \a \b \f \n \r \t \v \\]")
if s != r"[\134 [null\000null] [1f\0371f] [at\100at] [dollar\044dollar] [percent\045percent] \' \" \a \b \f \n \r \t \v \134]" :
s = "c_ctrl_esc: " + repr(s)
raise ValueError(s)
s = lf_only("\r\r\r\n\n\tnow\t\r\nis\r the \n\r\ntime \r\r\nfor")
if s != "\n\n\tnow\t\nis\n the \n\ntime \nfor" :
s = "lf_only: " + c_string(s)
raise ValueError(s)
s = lf_only_with_no_trailing_white_space(" \n blah \n lkjsd \f \t \n lkjsdf \t")
if s != "\n blah\n lkjsd\n lkjsdf\n" :
s = "lf_only_with_no_trailing_white_space: " + c_string(s)
raise ValueError(s)
s = lf_only_with_no_trailing_white_space(" \n blah \n lkjsd \f \t \n lkjsdf \t\n")
if s != "\n blah\n lkjsd\n lkjsdf\n" :
s = "lf_only_with_no_trailing_white_space: " + c_string(s)
raise ValueError(s)
s = no_blank_lines("\r\r\r\n\n\tnow\t\r\nis\r the \n\r\ntime \r\r\nfor")
if s != "\tnow\nis\n the\ntime\nfor\n" :
s = "no_blank_lines: " + c_string(s)
raise ValueError(s)
s = multiline_strip(" x \r\n\t \tx \r \t x\nbla \t x \n test \n")
if s != "x\nx\nx\nbla \t x\ntest\n" :
s = "multiline_strip: " + c_string(s)
raise ValueError(s)
s = multiline_strip("x \r\n\t \tx \r \t x\nbla \t x \n test ")
if s != "x\nx\nx\nbla \t x\ntest" :
s = "multiline_strip: " + c_string(s)
raise ValueError(s)
s = multiline_strip("\r x \r\n\t \tx \r \t x\nbla \t x \n test ")
if s != "\nx\nx\nx\nbla \t x\ntest" :
s = "multiline_strip: " + c_string(s)
raise ValueError(s)
s = multiline_strip(" \r x \r\n\t \tx \r \t x\nbla \t x \n test ")
if s != "\nx\nx\nx\nbla \t x\ntest" :
s = "multiline_strip: " + c_string(s)
raise ValueError(s)
s = multiline_strip(" \r xZ \r\n\tZ \tx \r \t x\nbla \t x \n test ", " \tZ")
if s != "\nx\nx\nx\nbla \t x\ntest" :
s = "multiline_strip: " + c_string(s)
raise ValueError(s)
s = multiline_flush_left(" \r xZ \r\n \tZ \tx \r \t x \r\n bla \t x \r\n\r\n test ")
if s != " \r xZ\n\tZ \tx \r \t x\n bla \t x\n\n test" :
s = "multiline_flush_left: [%s]" % c_string(s)
raise ValueError(s)
s = " now is the time "
cs = maybe_wrap_with_cdata(s)
if s != cs :
raise ValueError("gratuitous cdata")
s = " now is the