#!/usr/bin/env python """ @file common.py @author Paul Hubbard @date 8/20/09 @brief Common code and data for the OOI DX module. @package ooidx.common Shared code and routines """ import re import string import urlparse import logging import os.path from twisted.internet.defer import inlineCallbacks from magnet.protocol import ClientCreator as MCC from ooici.protocol import Message, ApplicationProtocol from twisted.internet import defer, reactor # being able to drop into ipython is mighty handy #from IPython.Shell import IPShellEmbed #ipshell = IPShellEmbed('') """ @brief A rough pass at name->address relationships. """ routing_keys = { 'cache' : 'dxCache', 'controller' : 'dxCacheController', 'distributor' : 'dxDistributor', 'fetcher' : 'dxFetcher', 'mgmt' : 'dxMgmt', 'persister' : 'dxPersister', 'proxy' : 'dxProxy', 'DAP_data' : 'dxDapData', 'pub_sub' : 'dxPubSub', 'attribute_store' : 'msAttributeStore', 'notification' : 'dxNotification', 'user_notifications' : 'dxUserNotifications', } def rewrite_url(dsUrl, newHostname='localhost'): """ @brief Given a DAP URL, presume that we host it locally and rewrite it to reflect same. Changes hostname to localhost, removes any port, rewrites path to be just root. Used by the cache front end, to change canonical URLs into local-only URLs. @param dsUrl Original URL to rewrite @param newHostname Default is localhost, TCP name of server @retval String with rewritten URL. @see ooidx.test.test_rewrite_url for the unit tests for this. """ ml = urlparse.urlsplit(dsUrl) # Replace the hostname with localhost chg_host = ml._replace(netloc=newHostname) # Remove all path components chg_path = chg_host._replace(path=os.path.basename(chg_host.path)) return chg_path.geturl() def generate_local_filename(dataset_url): """ @brief Given a URL, generate a local filesystem name for same. Used by persister for write and cache for purge operations. It's a stinky hack, really. @param dataset_url Original URL @retval Local filename @todo Complete refactor - directories, etc """ basicName = os.path.basename(dataset_url) #file name safe characters datasetid = ''.join([char for char in basicName if char in (string.letters + string.digits + "_-.")]) # Relative paths work on both mac and EC2, duh. return '../../dap_server/data/' + datasetid def baseDapUrl(srcUrl): """ Given a DAP URL with optional parameters, return the base URL. The DAP URL itself is used as a key for many indices in the OOIDX (is a dataset cached, for example) so this is quite important. Current implementation uses regular expressions. @see test_baseUrl.py @param srcUrl Source URL, with DAP suffixes (DDS, DAS, DODS, etc) @retval Base URL, or None if invalid @bug Needs to recognize Ferret functions and return URL plus expression. """ # This covers normal DAP URLs, but fails on base URLs such as http://127.0.0.1:8001/etopo120.cdf mset = re.search('(http|https)://([^/]+)(.+)(\.d(a|d)s|\.dods|\.asc(ii)*)(\?.+)*', srcUrl) if mset == None: # logging.debug('Checking for match with base URL') # This regex works on just plain DAP URLs - dds/das/dods optional mset = re.search('(http|https)://([^/]+)(.+)(\.dds|\.das|\.dods|\.asc(i)*)*(\?.+)*', srcUrl) if mset == None: logging.warning('No URL match in "%s" % srcUrl') return None try: return mset.group(1) + '://' + mset.group(2) + mset.group(3) except: logging.error('DAP URL does not match expected pattern!') return None try: return mset.group(1) + '://' + mset.group(2) + mset.group(3) except: logging.error('DAP URL does not match expected pattern!') return None def dumpMessage(msg): """ @param msg Message to dump @retval None @brief Debug code to dump headers and to-to-80 bytes of payload using logging.debug. Surprisingly useful. Disable by setting log level to INFO or higher. """ if isinstance(msg, Message): for field in msg.getHeaderKeys(): logging.debug('%s: %s' % (field, msg.getHeader(field))) logging.debug('Payload: ' + str(msg.payload[:80])) def parseMsg(msg): """ Parses a message into numeric return code and list of chunks, as done by split(). Convenience function. @param msg String message, with http code as the first element @retval return code and message list, None if parse error """ try: mlist = msg.split() rc = int(mlist[0]) contents = mlist[1:] return rc, contents except: logging.exception('Error parsing message') return None, None @inlineCallbacks def simpleSend(preactor, msg, send_to, method='producer'): """ Given an ooici.protocol.Message and destination, create and destroy the objects required to send it off. This is a one-way, one-time send. @param preactor Preactor instance @param msg Message object @param method producer or ms, depending on one-way or bidirectional @param send_to Destination address in broker """ try: mclient = MCC(reactor, preactor, ApplicationProtocol) if method == 'producer': mc = yield mclient.connectSimpleProducer(send_to) elif method == 'ms': mc = yield mclient.connectMS(send_to) yield mc.sendMessage(msg) # mclient.transport.loseConnection() except: logging.exception('Error on send!') class RequestResponseAP(ApplicationProtocol): """ Request-response base class for ApplicationProtocol. Just saves the deferred, really. Code from Dorian. @see ooidx.pub_sub.PubSubClient for an example """ def __init__(self): ApplicationProtocol.__init__(self) self.deferred = None def makeRequest(self, request): self.sendMessage(request) self.deferred = defer.Deferred() return self.deferred def messageReceived(self, msg): self.deferred.callback(msg)