#!/usr/bin/python # MSclient.py # --copyright-- Copyright 2007 (C) Tranzoa, Co. All rights reserved. Warranty: You're free and on your own here. This code is not necessarily up-to-date or of public quality. # --url-- http://www.tranzoa.net/tzpython/ # --email-- pycode is the name to send to. tranzoa.com is the place to send to. # --bodstamps-- # August 4, 2003 bar # August 6, 2003 bar finally get the SloppyType type logic correct # then toss it # August 7, 2003 bar put in full bayesian analysis # August 8, 2003 bar read the file in the background at startup time # August 11, 2003 bar move toward being able to get the XML file from the server # August 12, 2003 bar fix typo in random quality # write separate quality_list.xml file and get the downloading working # SPAM_OVERKILL_COUNT # put the search_list.xml file in the \pj directory # option to take new songs from the middle of the song list # pass by_human to next/prev # August 13, 2003 bar tickle the evaluator when the learn mode changes # August 15, 2003 bar some test code # September 3, 2003 bar show_info a printout that wasn't protected by the flag # November 18, 2007 bar turn on doxygen # November 21, 2007 bar PJids doesn't exist, so ... # November 27, 2007 bar insert boilerplate copyright # May 12, 2008 bar don't use id as a variable name # May 17, 2008 bar email adr # November 29, 2011 bar pyflake cleanup # May 27, 2012 bar doxygen namespace # August 12, 2012 bar put name in thread # May 28, 2014 bar put thread id in threads # July 7, 2018 bar pyflakes # --eodstamps-- ## \file # \namespace tzpython.MSclient # # # This module is the operational guts of the Music Searcher program (which has been thrown in as a menu item on the PJP program). # # It maintains a "tape" of what's been played. # If implements whatever search methods it wants to. # # In particular, it has a custom tokenizing routine that deals with a search_list.xml file containing tags that we have to work with. # # It creates the tokens to feed to a Bayesian spam filter. # Then it sorts all the clips (music) in order of "spamness". # And plays the least-spammy of the music that has not been heard before. # # It contains a routine to reset the logic - starting anew, fresh. # # It is able to get the search_list.xml file off the server, gigachu. # And it is able to notice whether the clip files are actually on the disk drive in the \pj\clips\ directory. # If a clip file is not on-disk, then the clip file will be played from a server URL directly without separate download. # # # TODO: # # When the PJP program is stopped before the search_list.xml file has been read, there is a Python exception. # Danged if I know what's going on with it. # I tried various things to try to fix it. But making an infinite join() in the close() routine was the only thing to "fix" it. # # Pick 'em from the songs so that the user is mixing love and hates well. # # import copy import os import os.path import random import re import string import threading import time from types import ListType, TupleType import attrmap import numeric_tokens from replace_file import replace_file import tzlib import url_getter import MusicMetaData try : import PJids except : pass import bayes.classifier __ALL__ = [ 'a_ms_client' ] HOST_PORT = "gigachu.office.aol.com/pj/" QXML_BASE_FILE_NAME = "quality_list" XML_FILE_NAME = "search_list.xml" TAG_NAME = "matching_ids" TAG_NAME = "matching_mfcc_ids" MIN_LOVE = 1 MIN_HATE = 1 RANDOM_QUALITY = 0.01 SPAM_OVERKILL_COUNT = 3 TAKE_FROM_MIDDLE = 0 TAKE_RANDOM = 0 LEARN_AMBIGUOUS = False def _convert_music_info_to_tokens(md, show_info = False) : """ Convert an array of MusicMetaData music data to a map, keyed by ID, containing a text token array. """ ids_tokens = {} nt = numeric_tokens.a_numeric_token_translator(16, 1) text_tags = tzlib.make_dictionary([ 'album', 'artist', 'genre', 'title', ]) numeric_tags = tzlib.make_dictionary([ 'TRM_AFD', 'TRM_AZX', 'TRM_BPM', 'TRM_HAR', 'TRM_MSR', 'TRM_NRD', 'TRM_NZX', 'TRM_SPC', 'TRM_SPS', 'date', 'duration', 'file_size', 'year', ]) include_id_tags = tzlib.make_dictionary([ 'matching_mfcc_ids', 'matching_savage_beast_ids', ]) for song in md : sid = song['id'] ttokens = [] ntokens = [] for k in song.keys() : if text_tags.has_key(k) or numeric_tags.has_key(k) or include_id_tags.has_key(k) : v = song[k] if isinstance(v, ListType) or isinstance(v, TupleType) : v = string.join(v) v = str(v).strip() if include_id_tags.has_key(k) : v = v + " " + song['id'] + " " + song['id'] t = re.split(r"\s+", v) t = map(lambda w : str(k + "~" + w), t) t = map(string.lower, t) if numeric_tags.has_key(k) : ntokens[len(ntokens):] = t else : ttokens[len(ttokens):] = t pass nt.learn(ntokens) ids_tokens[sid] = [ ttokens, ntokens ] for song in md : sid = song['id'] ids_tokens[sid] = ids_tokens[sid][0] + nt.translate(ids_tokens[sid][1]) # fix up numeric tokens so that there are only N different tokens by each "name". The "name" is the non-numeric, left part of the token. if show_info : print ids_tokens.get('1234', None) if show_info : print ids_tokens.get('4003', None) return(ids_tokens) class a_ms_client : class _a_dj(threading.Thread) : """ An object of this class re-sorts the music list according to how the user has evaluated it. """ def __init__(me, thang) : threading.Thread.__init__(me, name = "a_ms_client") me.thang = thang me.tid = None me.stop = False; me.trigger = threading.Event() me.trigger.set() def run(me) : me.tid = PJids.get_tid() if me.thang._show_info : print "eval running" try : while not me.stop : me.trigger.wait() # wait for a trigger set so that we can do our thing me.trigger.clear() if me.stop : break if me.thang._show_info : print "eval doing" me.thang._learn_what_we_know() if me.thang._show_info : print "eval done" pass pass except : open("msexcept.txt", "w").write("msclient except " + time.asctime()) raise pass if me.thang._show_info : print "eval ended" pass def __init__(me, config_file_dir = ".", host_port = None, music_info_file = None, new_user = False, show_info = False ) : me.take_from_middle = TAKE_FROM_MIDDLE me.take_random = TAKE_RANDOM me.learn_mode = LEARN_AMBIGUOUS def __fix_parm(p, dflt = None) : if (p != None) and not len(p) : p = None if p == None : p = dflt return(p) me.host_port = "http://" + __fix_parm(host_port, HOST_PORT) music_info_file = __fix_parm(music_info_file, XML_FILE_NAME) config_file_dir = __fix_parm(config_file_dir, ".") if not os.path.isdir(config_file_dir) : try : os.makedirs(config_file_dir) except OSError : config_file_dir = "." except IOError : config_file_dir = "." pass me.music_info_file = os.path.normpath(os.path.join(config_file_dir, music_info_file)) # not exactly, we'll later move up a dir, probably (h, t) = os.path.split(me.music_info_file) (b, x) = os.path.splitext(t) me.music_quality_file = os.path.join(h, QXML_BASE_FILE_NAME + x) if show_info : print "mqf:", me.music_quality_file if config_file_dir != "." : config_file_dir = os.path.join(config_file_dir, "..") me.music_info_file = os.path.normpath(os.path.join(config_file_dir, music_info_file)) me.config_file_dir = config_file_dir # this is NOT the user's directory, but is rather it's parent directory, unless the config dir is the current directory, "." me.new_user = new_user me._show_info = show_info me._ready = False me.md = [] me.id_to_song = {} me.ids_tokens = {} me._forget_stuff() me.lock = threading.RLock() me.dj = a_ms_client._a_dj(me) me.dj.setDaemon(True) # so that we can kick out of the program while the thread is running me.dj.start() def _forget_stuff(me) : me.cursor = 0 me.playlist = [] me.playlist_by_id = {} me.playing = False me.love_count = 0 me.hate_count = 0 me.play_count = 0 def start_new_user(me) : me.lock.acquire() me.stop_playing() me.new_user = True me._forget_stuff() if me._ready and me.new_user : random.shuffle(me.md) for song in me.md : song['_quality'] = 0.5 if song.has_key('_reaction') : del song['_reaction'] pass pass me.lock.release() def _fix_urls(me, my_md, new_user) : clip_url_to = me.host_port + "/clips/" md = [] for song in my_md : if song.has_key(TAG_NAME) and len(song[TAG_NAME].strip()) : def _valid_num(n, dflt) : try : n = int(str(n)) except : n = dflt return(n) song[TAG_NAME] = re.split(r"\s+", song[TAG_NAME]) song['audio_url'] = clip_url_to + str(song['id']) + "_clip.mp3" song['_bayes'] = "" if song.has_key('_quality') and new_user : del song['_reaction'] if not song.has_key('_quality') or new_user : song['_quality'] = 0.5 if not song.has_key('_file_name') : song['_file_name'] = "" if not song.has_key('artist') : song['artist'] = "" if not song.has_key('title') : song['title'] = "" song['_reaction'] = _valid_num(song.get('_reaction', None), None) if song['_reaction'] == None : del song['_reaction'] song['_quality'] = _valid_num(song['_quality'], 0.5) md.append(song) pass return(md) def close(me) : me.dj.stop = True me.dj.trigger.set() if me._ready and me.love_count and me.hate_count and me.play_count and len(me.md) : me.lock.acquire() mds = [] ok = False for song in me.md : qsong = {} qsong['id'] = song['id'] if me._show_info : if song.has_key('_bayes') : qsong['_bayes'] = song['_bayes'] ok = True pass if song.has_key('_quality') : qsong['_quality'] = song['_quality'] ok = True if song.has_key('_reaction') : qsong['_reaction'] = song['_reaction'] ok = True mds.append(qsong) me.lock.release() if ok : tf = me.music_quality_file + ".tmp" MusicMetaData.print_file(tf, mds) replace_file(me.music_quality_file, tf, me.music_quality_file + ".bak") pass me.dj.join(0.75) # !!!! if this is an infinite join, then there is no exception at close time if the file is being read def show_info(me, how) : retval = me._show_info if how != None : me._show_info = how return(retval) def do_it(me) : pass def _do_reaction_id(me, sid, ids, idm, depth) : id_depth = idm.get(sid, -1) if (id_depth <= depth) : song = me.id_to_song.get(sid, None) if song != None : id_depth = max(0, id_depth) for i in range(0, depth + 1 - id_depth) : ids.append(sid) # replicate higher order ids so that they have more effect on the music list order idm[sid] = depth + 1 if depth : depth = depth - 1 near_ids = song[TAG_NAME] for sid in near_ids : me._do_reaction_id(sid, ids, idm, depth) pass pass pass pass def reaction(me, sid = None, value = None, change = 0) : me.stop_playing() if sid == None : sid = me.current_selection_id() if change and (sid != None) : sid = PJids.make_full_id(PJids.real_id(sid)) song = me.id_to_song.get(sid, None) v = 0 if song != None : v = song.get('_reaction', 0) v = max(-2, v) v = min( 2, v) nv = v if value != None : nv = value nv = nv + change nv = max(-2, nv) nv = min( 2, nv) if song != None : song['_reaction'] = nv if nv > 0 : me.love_count = me.love_count + 1 elif nv < 0 : me.hate_count = me.hate_count + 1 if nv : ids = [] me._do_reaction_id(sid, ids, {}, 2) if me._show_info : print ids random.shuffle(ids) me.lock.acquire() if me._show_info : print "Learning", len(ids) for sid in ids : j = i = MusicMetaData.id_idx(me.md, sid) if nv > 0 : j = int(i / 2) elif nv < 0 : j = (i + len(me.md)) / 2 me.md.insert(j, me.md.pop(i)) if me._show_info : print me.dj.trigger.set() me.lock.release() me.next_selection() pass def love(me) : me.reaction(change = 1) def hate(me) : me.reaction(change = -1) def so_so(me) : me.reaction(change = 0) def add_event(me, sid, event, args = "") : pass def id_file_name(me, sid) : if sid != None : fn = os.path.normpath(os.path.join(me.config_file_dir, "clips", sid + "_clip.mp3")) print "idfn", sid, fn if os.path.isfile(fn) : return(fn) pass return(None) def id_url(me, sid) : if sid != None : song = me.id_to_song.get(sid, None) if song != None : return(song['audio_url']) pass return(None) def current_selection_id(me) : if me.cursor < len(me.playlist) : return(me.playlist[me.cursor]) return(None) def current_playing_id(me) : if me.playing : return(me.current_selection_id()) return(None) def stop_playing(me) : me.playing = False def play_current_selection(me) : if not me.playing : sid = me.current_selection_id() if sid == None : me.next_selection() sid = me.current_selection_id() if sid != None : # don't check for whether there is a file, because he can play from a URL me.play_count = me.play_count + 1 me.playing = True if me._show_info : print "cursor", me.cursor, len(me.playlist) pass return(me.current_playing_id()) def next_selection(me, by_human = False) : if by_human : me.so_so() me.stop_playing() if me.cursor < len(me.playlist) : me.cursor = me.cursor + 1 if me.cursor >= len(me.playlist) : me.lock.acquire() v = 0 if me.take_from_middle : v = len(me.md) / 2 v = range(v, len(me.md)) for idx in v : song = me.md[idx] sid = song['id'] if not me.playlist_by_id.has_key(sid) : me.cursor = me.playlist_by_id[sid] = len(me.playlist) me.playlist.append(sid) break pass me.lock.release() return(me.current_selection_id()) def previous_selection(me, by_human = False) : me.stop_playing() if me.cursor : me.cursor = me.cursor - 1 return(me.current_selection_id()) def get_current_selection_info(me) : sid = me.current_selection_id() song = None if sid != None : song = copy.copy(me.id_to_song[sid]) song['_reaction'] = song.get('_reaction', 0) song = attrmap.convert_map_to_attributes(song) return(song) def get_msg_to_user(me) : return(None) def get_url_to_user(me) : return(None) def get_new_program_url(me) : return(None) def play_mode(me, how = None) : retval = "normal" if me.take_random : retval = "random" elif me.take_from_middle : retval = "middle" if how == None : pass elif how == "normal" : me.take_from_middle = False me.take_random = False elif how == "random" : me.take_from_middle = True me.take_random = True me.lock.acquire() random.shuffle(me.md) me.lock.release() elif how == "middle" : me.take_from_middle = True me.take_random = False else : print "MSclient: Bad mode" raise "MSclient: Bad mode" return(retval) def learn_ambiguous(me, how = None) : retval = me.learn_mode if (how != None) and (how != retval) : me.learn_mode = how me.dj.trigger.set() return(retval) def geek_numbers(me) : retval = {} retval[' id '] = me.current_selection_id() c = lc = hc = 0 for sid in me.id_to_song.keys() : song = me.id_to_song[sid] r = song.get('_reaction', None) if r != None : if r > 0 : lc = lc + 1 elif r < 0 : hc = hc + 1 else : c = c + 1 pass pass retval[' love count'] = str(lc) retval[' hate count'] = str(hc) retval[' ???? count'] = str( c) me.lock.acquire() c = lc = hc = 0 for idx in range(0, len(me.md) / 20) : song = me.md[idx] r = song.get('_reaction', None) if r != None : if r > 0 : lc = lc + 1 elif r < 0 : hc = hc + 1 else : c = c + 1 pass pass me.lock.release() retval['top love count'] = str(lc) retval['top hate count'] = str(hc) retval['top ???? count'] = str( c) me.lock.acquire() c = lc = hc = 0 for idx in range(int(len(me.md) * 19) / 20, len(me.md)) : song = me.md[idx] r = song.get('_reaction', None) if r != None : if r > 0 : lc = lc + 1 elif r < 0 : hc = hc + 1 else : c = c + 1 pass pass me.lock.release() retval['bottom love count'] = str(lc) retval['bottom hate count'] = str(hc) retval['bottom ???? count'] = str( c) retval[' play count'] = str(me.play_count) retval[' mode '] = me.play_mode() retval[' mode-learn amb'] = str(me.learn_ambiguous()) return(retval) # # # Callbacks to get info for downloading the XML file from the server # # def _get_url_and_file_name(me) : if me.got_xml_file != None : return( ( None, None ) ) url = me.host_port + "/" + "pj/" + XML_FILE_NAME if me._show_info : print "telling url getter to get", url return( ( url, me.music_info_file ) ) def _got_file_from_url(me, file_name, ok) : if me._show_info : print "Maybe got", file_name, "==", ok me.got_xml_file = ok def _learn_what_we_know(me) : if not me._ready : md = [] mq = [] if os.path.isfile(me.music_info_file) : try : md = MusicMetaData.parse_file(me.music_info_file) except : me = [] pass # this try/except does not fix the exception problem when the program is stopped while reading the file pass else : me.got_xml_file = None getter = url_getter.a_url_getter(me, show_info = me._show_info) getter.setDaemon(True) # so that we can kick out of the program while the thread is running getter.start() getter.trigger.set() if me._show_info : print "getting", me.music_info_file, "from server" while (not me.dj.stop) and (me.got_xml_file == None) : time.sleep(0.127) # wait for the file to be downloaded from the server if me._show_info : print "got", me.music_info_file, "from server ==", me.got_xml_file getter.lock.acquire() getter.stop = True getter.trigger.set() getter.lock.release() getter.trigger.set() getter.join(0.17) if (not me.dj.stop) and os.path.isfile(me.music_info_file) : md = MusicMetaData.parse_file(me.music_info_file) pass if me._show_info : print "clip mode search file read with", len(md), "entries:", me.music_info_file if not me.dj.stop : if os.path.isfile(me.music_quality_file) : mq = MusicMetaData.parse_file(me.music_quality_file) if len(mq) : id_to_song = {} for song in md : id_to_song[song['id']] = song for song in mq : sid = song['id'] if id_to_song.has_key(sid) : if song.has_key( '_bayes') : id_to_song[sid]['_bayes'] = song['_bayes'] if song.has_key( '_quality') : id_to_song[sid]['_quality'] = song['_quality'] if song.has_key( '_reaction') : id_to_song[sid]['_reaction'] = song['_reaction'] pass pass pass pass md = me._fix_urls(md, me.new_user) id_to_song = {} for song in md : id_to_song[song['id']] = song if me._show_info : print "clip mode quality file read" me.ids_tokens = {} if not me.dj.stop : me.ids_tokens = _convert_music_info_to_tokens(md, me._show_info) me.lock.acquire() me.id_to_song = id_to_song me.md = md if me.new_user and not me.dj.stop : random.shuffle(me.md) me._ready = True me.lock.release() if me._show_info : print "-------------- clip mode ready:", len(me.md), "clips ----------------" if (not me.dj.stop) and (me.love_count >= MIN_LOVE) and (me.hate_count >= MIN_HATE) : # # Learn what the person thinks of what he's heard # c = bayes.classifier.Classifier() for song in me.id_to_song.values() : if song.get('_reaction', None) != None : sid = song['id'] is_spam = song['_reaction'] < 0 tokes = me.ids_tokens[sid] for i in range(0, SPAM_OVERKILL_COUNT) : c._add_msg(iter(tokes), is_spam) if me.learn_mode : c._add_msg(iter(tokes), True) c._add_msg(iter(tokes), False) c._add_msg(iter(tokes), is_spam) pass # # Evaluate all of the music # for sid in me.id_to_song.keys() : song = me.id_to_song[sid] r = c.spamprob(iter(me.ids_tokens[sid]), 1) song['_bayes'] = str(r) r = r[0] # use just the bottom line number of spam-ness if RANDOM_QUALITY != 0.0 : r = r + (random.random() * RANDOM_QUALITY) - (RANDOM_QUALITY / 2) song['_quality'] = r # # Sort the music by taste # me.lock.acquire() if me.take_random : random.shuffle(me.md) else : def __spam_cmp(s1, s2) : return(cmp(s1['_quality'], s2['_quality'])) me.md.sort(__spam_cmp) me.lock.release() pass def ready(me) : return(me._ready) pass # end of class # # # Options: # # if __name__ == '__main__' : import sys sid = sys.argv[2] t = time.time() md = MusicMetaData.parse_file(sys.argv[1]) nt = time.time() print "file read", nt - t t = nt idx = MusicMetaData.id_idx(md, sid) if idx < 0 : print "Can't find ID:", sid raise song = md[idx] ids_tokens = _convert_music_info_to_tokens(md) c = bayes.classifier.Classifier() nt = time.time() print "ID", sid, nt - t t = nt tokes = ids_tokens[sid] c._add_msg(iter(tokes), False) for song in md : sid = song['id'] r = c.spamprob(iter(ids_tokens[sid]), 1) song['_bayes'] = str(r) song['_quality'] = r[0] nt = time.time() print "eval'ed", nt - t t = nt def __spam_cmp(s1, s2) : return(cmp(s1['_quality'], s2['_quality'])) md.sort(__spam_cmp) nt = time.time() print "sorted", nt - t t = nt try : MusicMetaData.print_file(sys.argv[3], md) except : MusicMetaData.print_file("x.xml", md) nt = time.time() print "done", nt - t t = nt # # # # eof