#!/usr/bin/python # tz_geonames_cache.py # --copyright-- Copyright 2007 (C) Tranzoa, Co. All rights reserved. Warranty: You're free and on your own here. This code is not necessarily up-to-date or of public quality. # --url-- http://www.tranzoa.net/tzpython/ # --email-- pycode is the name to send to. tranzoa.com is the place to send to. # --bodstamps-- # January 1, 2008 bar spun off from trod_track # January 3, 2008 bar forgotten import # January 5, 2008 bar get_point_name() # May 17, 2008 bar email adr # May 31, 2008 bar set ostr if no loc found - and protect both of the "" ostr sets if the point changed while we were hitting the server # label_cache # July 2, 2008 bar clean up some stuff and prepare for outsiders to get the point that's close to us # July 5, 2008 bar hash_str() # September 4, 2008 bar fool with timing (turns out the big time is taken writing a file, not looking up names that might be in the file) # name our thread # September 9, 2008 bar try to consistenly resolve locations that are the same place # November 29, 2011 bar pyflake cleanup # May 27, 2012 bar doxygen namespace # May 28, 2014 bar put thread id in threads # --eodstamps-- ## \file # \namespace tzpython.tz_geonames_cache # # # Keep a cache of geonames locations. # # TODO: # # do something more rational about points that the server didn't give names for than just hitting the server again # use tz_gps's 9-way lookup for the nearest point in the cache # keep track of points that are not looked up, but which have been passed to set_point # do them in the background when nothing else is going on # better: have outsider give us tracks to do in the background # we would look for points that are unknown to the cache and/or are furthest from known, cached points # handle non-unicode names (and callers must do so, too) # cache all of the acceptable returned locations # mark locations "hit" if they are used # background: refresh all the "hit" locations, learning their nearbys # after a new location is learned, always refresh the current point's location # # import cPickle import time import threading import latlon import replace_file import tz_geonames import tz_gps import tzlib class a_nearby_point(object) : def __init__(me, p) : me.lat = p.lat me.lon = p.lon try : me.altitude = p.altitude except AttributeError : me.altitude = None me.name = p.name def distance_from(me, p) : return(latlon.calcDistance(me.lat, me.lon, p.lat, p.lon) ) def __str__(me) : s = "" s += "lat=%.6f " % ( me.lat ) s += "lon=%.6f " % ( me.lon ) s += "alt=%i " % ( me.altitude or 0 ) return(s) def hash_str(me) : return(str(me) + me.name) pass class a_cache(threading.Thread) : def __init__(me, lag = None, max_distance = None, pickle_file_name = None, query = True, *args, **kwargs) : kwargs['name'] = kwargs.get('name', "tz_geonames_cache.a_cache") super(a_cache, me).__init__(args = args, kwargs = kwargs) me.lag = lag or 0.3 me.max_distance = max_distance or 2.5 me.query = query me.pickle_file = pickle_file_name me.changed_when = 0.0 me.p = None me.ps = "" me.tid = None me.trkcache = {} # key: tz_geonames.a_location.__str__ value: tz_geonames.a_location Stops us from adding redundant location/points to 'tracks' me.tracks = [ tz_gps.a_track() ] # an array of 1 track with its .points being an array of a_nearby_point, the locations/points we've found from geonames me.cache = {} # key: a_nearby_point.__str__ value: a_nearby_point of the geonames point whose name was looked up already - whose __str__ is the key me.cached = False me.weak = True me.label_cache = {} me.label_points = [] me.blahcnt = 0 me.okcnt = 0 me.ostr = None # output string - information, "", or None me.lock = threading.RLock() me._stop = False me.setDaemon(True) me.lock.acquire() if me.pickle_file : fi = None try : fi = open(me.pickle_file, "rb") ( me.cache, me.tracks ) = cPickle.load(fi) fi.close() fi = None for ps in me.cache.keys() : try : str(me.cache[ps].name) except UnicodeEncodeError : del(me.cache[ps]) pass me.trkcache = {} t = me.tracks[0] for pi in xrange(len(t.points) - 1, -1, -1) : p = t.points[pi] ps = str(p) if me.trkcache.get(ps, "") : del(t.points[pi]) else : me.trkcache[ps] = p pass # print "read", len(me.cache), "points and", len(t.points), "named locations" pass except AttributeError : me.cache = {} me.tracks = [ tz_gps.a_track() ] except IndexError : me.cache = {} me.tracks = [ tz_gps.a_track() ] except IOError : me.cache = {} me.tracks = [ tz_gps.a_track() ] except ValueError : me.cache = {} me.tracks = [ tz_gps.a_track() ] except cPickle.PickleError : me.cache = {} me.tracks = [ tz_gps.a_track() ] if fi : fi.close() pass me.lock.release() @staticmethod def try_loc(loc, l) : try : str(l) # !!!! for the moment, we'll simply filter out unicode names by triggering the exception if (not loc) or ((loc.lat == l.lat) and (loc.lon == l.lon) and (loc.name.lower() < l.name.lower())) : # we ignore loc.elevation because the callers will be dealing in flat distances - we take the higher name so longer same-starting names and non-zero length names take precedence loc = l pass except UnicodeEncodeError : pass return(loc) def run(me) : me.tid = tzlib.get_tid() if me.query : while not me._stop : me.lock.acquire() if tzlib.elapsed_time() - me.changed_when < me.lag : me.lock.release() time.sleep(0.17) # hang out until he's idled his cursor over a point for a while elif me.p and (me.weak or (me.ostr == None)) : p = me.p ps = me.ps me.lock.release() r = tz_geonames.get_nearby(p.lat, p.lon, me.max_distance) # go out to the server to get nearby names if me._stop : break me.lock.acquire() if not r : if ps == me.ps : me.ostr = "" # mark us as having found nothing me.weak = False # don't record the lack of location, though, in our cache - hmmmm pass else : loc = None for l in r : if ('lat' in l.tags) and ('lon' in l.tags) and ('name' in l.tags) : if (l.tags.get("fcl", "") != 'P') or (l.tags.get("fcode", "") != 'PPL') or l.tags.get("population", 0) : loc = me.try_loc(loc, l) pass pass if not loc : for l in r : if ('lat' in l.tags) and ('lon' in l.tags) and ('name' in l.tags) : # print "non-zeropop", r[0] loc = me.try_loc(loc, l) pass pass if loc : ls = str(loc) loc_p = a_nearby_point(loc) me.cache[ps] = loc_p # one way or the other, remember the stripped location in a cache that's looked up by the caller's points if not me.trkcache.get(ls, None) : # don't store this location if we already know it in 'tracks' me.trkcache[ls] = loc me.tracks[0].points.append(loc_p) # note: to keep the data store down in size, just store the data we need: lat/lon/alt/name me.okcnt += 1 else : me.blahcnt += 1 # for metrics purposes, keep a count of how many times we've hit geonames and just found what we already knew # print "ok:", me.okcnt, "blah:", me.blahcnt me.cached = True if ps == me.ps : me.ostr = loc.name me.weak = False else : pp = me.p me.set_point(p) me.set_point(pp) pass elif ps == me.ps : me.ostr = "" # mark us as having found nothing me.weak = False # don't record the lack of location, though, in our cache - hmmmm pass me.lock.release() else : me.lock.release() time.sleep(0.17) # there's nothing to do - we've already got the point's name in 'ostr' pass me.ostr = None pass def pickle(me) : me.lock.acquire() if me.pickle_file and me.cache and me.cached : fn = me.pickle_file tfn = fn + ".tmp" me.pickle_file = None fo = open(tfn, "wb") try : cPickle.dump( ( me.cache, me.tracks ), fo, protocol = cPickle.HIGHEST_PROTOCOL) except cPickle.PickleError : fn = None fo.close() del(fo) if fn : replace_file.replace_file(fn, tfn, fn + ".bak") pass me.lock.release() def set_point(me, p) : if p : p = a_nearby_point(p) ps = str(p) me.lock.acquire() if me.ps != ps : me.ps = ps me.ostr = None # mark us as not having queried the server yet me.changed_when = tzlib.elapsed_time() me.p = p if not p : me.weak = False me.ostr = "" # mark us as having found nothing elif ps in me.label_cache : me.weak = False me.ostr = me.label_cache[ps].name else : lp = me._find_point_in_label_points(p) if lp : me.label_cache[ps] = lp me.weak = False me.ostr = lp.name elif ps in me.cache : me.weak = False me.ostr = me.cache[ps].name else : npts = tz_gps.get_nearby_tracks_points(me.tracks, p, me.max_distance) # look for quick-guess-approximate, named points that we got from the server if npts : me.weak = True # tell us to look up the cursor point anyway me.ostr = npts[0].point.name or "" # but meanwhile, show the user the best guess pass pass pass me.lock.release() def get_a_nearby_point(me, p) : """ Return a tz_gps.a_nearby_point that's closest to the given point. The '.point' in the tz_gps.a_nearby_point is probably one of our a_nearby_point objects. """ if p : p = a_nearby_point(p) ps = str(p) if ps in me.label_cache : p.name = me.label_cache[ps].name return(tz_gps.a_nearby_point(p, 0.0)) lp = me._find_point_in_label_points(p) if lp : me.label_cache[ps] = lp p.name = lp.name return(tz_gps.a_nearby_point(p, 0.0)) if ps in me.cache : cp = me.cache[ps] p = tz_gps.a_nearby_point(cp, p.distance_from(cp)) return(p) npts = tz_gps.get_nearby_tracks_points(me.tracks, p, me.max_distance) # look for quick-guess-approximate, named points from the server if npts : return(npts[0]) # return a tz_gps.a_nearby_point containing the name and the geonames a_nearby_point pass return(None) def get_point_name(me, p) : p = me.get_a_nearby_point(p) if p : return(p.point.name) return("") def stop(me) : me._stop = True me.pickle() def _find_point_in_label_points(me, p) : for lp in me.label_points : if lp.is_inside(p) : return(lp) pass return(None) def append_known_names(me, label_points) : me.label_points += label_points # remember labeled points - they have .name and .is_inside() attributes pass # a_cache if __name__ == '__main__' : t = a_cache() # # # # eof