From a583f311a025ffa8c689f134bdec70564250ac75 Mon Sep 17 00:00:00 2001 From: Bryan Date: Thu, 20 Feb 2014 05:39:13 -0500 Subject: [PATCH] making fabric better for VM deployment and btw #335 --- dicthelpers.py | 126 +++++++++++++++++++++++++++++++++++++++++++++ dicthelperstest.py | 40 ++++++++++++++ fabfile.py | 34 ++++++++---- 3 files changed, 190 insertions(+), 10 deletions(-) create mode 100644 dicthelpers.py create mode 100644 dicthelperstest.py diff --git a/dicthelpers.py b/dicthelpers.py new file mode 100644 index 0000000..8ee08c6 --- /dev/null +++ b/dicthelpers.py @@ -0,0 +1,126 @@ +""" +Karmaworld Dictionary helpers +Finals Club (c) 2014 +Author: Bryan Bonvallet +""" + +# I'd wager none of these are thread safe. + +import re +from collections import MutableMapping + + +needs_mapping = re.compile('{[a-zA-Z_]\w*}') + + +class attrdict(object): + """ Access dictionary by object attributes. """ + def __getattr__(self, attr): + # create call stack, if not already there, to avoid loops + try: + callstack = object.__getattribute__(self, '__callstack') + except AttributeError: + object.__setattr__(self, '__callstack', []) + callstack = object.__getattribute__(self, '__callstack') + + try: + # don't call something already on the stack + if attr in callstack: + raise KeyError() + # track this attribute before possibly recursing + callstack.append(attr) + retattr = self.__getitem__(attr) + # success, remove attr from stack + callstack.remove(attr) + return retattr + except KeyError: + # if code is here, attr is not in dict + try: + # try to grab attr from the object + retattr = super(attrdict, self).__getattribute__(attr) + return retattr + finally: + # remove stack now that the attribute succeeded or failed + callstack.remove(attr) + + def __setattr__(self, attr, value): + self.__setitem__(attr, value) + + def __delattr__(self, attr): + self.__delitem__(attr) + + +class fallbackdict(MutableMapping, attrdict): + """ + Retrieve default values from a fallback dictionary and dynamically .format + """ + def __init__(self, fallback, *args, **kwargs): + """ + Supply dictionary to fall back to when keys are missing. + Other arguments are handled as normal dictionary. + """ + # dodge attrdict problems by using object.__setattr__ + classname = type(self).__name__ + object.__setattr__(self, '_{0}__internaldict'.format(classname), {}) + object.__setattr__(self, '_{0}__fallback'.format(classname), fallback) + object.__setattr__(self, '_{0}__fetching'.format(classname), []) + super(fallbackdict, self).__init__(*args, **kwargs) + + def __needs_format__(self, value): + """ Helper to determine when a string needs formatting. """ + return hasattr(value, 'format') and needs_mapping.search(value) + + def __getitem__(self, key): + """ + Use this dict's value if it has one otherwise grab value from parent + and populate any strings with format keyword arguments. + """ + indict = self.__internaldict + fetching = self.__fetching + + local = (key in indict) + # this will throw a key error if the key isn't in either place + # which is desirable + value = indict[key] if local else self.__fallback[key] + + if (key in fetching) or (not self.__needs_format__(value)): + # already seeking this key in a recursed call + # or it doesn't need any formatting + # so return as-is + return value + + # if the code is this far, strings needs formatting + # **self will call this function recursively. + # prevent infinite recursion from e.g. d['recurse'] = '{recurse}' + fetching.append(key) + value = value.format(**self) + # undo infinite recursion prevention + fetching.remove(key) + return value + + def __setitem__(self, key, value): + """ + Set the internal dict with key/value. + """ + self.__internaldict[key] = value + + def __delitem__(self, key): + """ Delete the key from the internal dict. """ + del self.__internaldict[key] + + def __uniquekeys__(self): + """ Returns unique keys between this dictionary and the fallback. """ + mykeys = set(self.__internaldict.keys()) + fbkeys = set(self.__fallback.keys()) + # set union: all keys from both, no repeats. + return (mykeys | fbkeys) + + def __iter__(self): + """ Returns the keys in this dictionary and the fallback. """ + return iter(self.__uniquekeys__()) + + def __len__(self): + """ + Returns the number of keys in this dictionary and the fallback. + """ + return len(self.__uniquekeys__()) diff --git a/dicthelperstest.py b/dicthelperstest.py new file mode 100644 index 0000000..eb64e19 --- /dev/null +++ b/dicthelperstest.py @@ -0,0 +1,40 @@ +# I need to turn this into a proper unit test. +# uhm, this is horrible. sorry everybody. +# -Bryan + +import time +from dicthelpers import fallbackdict + +start = time.time() + +fallback = {1: '1'} +poop = fallbackdict(fallback) +print 'expect 1 (fallback): ' + poop[1] +fallback[2] = '2' +print 'expect 2 (dynamic fallback): ' + poop[2] +poop[2] = '3' +print 'expect 3 (overridden): ' + poop[2] +del poop[2] +print 'expect 2 (fallback after delete): ' + poop[2] +fallback['three'] = 3 +fallback[3] = '3{three}' +print 'expect 33 (formatting): ' + poop[3] +print 'expect 4 (len): ' + str(len(poop)) +fallback['infinite'] = '{infinite}' +print 'simple infinite recursion test: ' + poop['infinite'] +fallback['infone'] = '{inftwo}' +fallback['inftwo'] = '{infone}' +print 'double infinite recursion test: ' + poop['infone'] +fallback['2infone'] = '{2inftwo}' +fallback['2inftwo'] = '{2infone}' +print 'double infinite recursion test: ' + poop['2inftwo'] +poop['four'] = 4 +poop[4] = '4{four}' +print 'expect 34 (self as string mapping): {three}{four}'.format(**poop) +print 'expect 44 (self formatting): ' + poop[4] +# silent iter length processing. +# this used to take 1 or 2 seconds prior to optimizations +for x in poop: poop[x] + +stop = time.time() +print "execution " + str(stop - start) diff --git a/fabfile.py b/fabfile.py index d76384d..88dc5bc 100644 --- a/fabfile.py +++ b/fabfile.py @@ -3,21 +3,32 @@ import os import ConfigParser +from cStringIO import StringIO -from fabric.api import cd, env, lcd, prefix, run, sudo, task, local, settings +from fabric.api import cd, lcd, prefix, run, sudo, task, local, settings +from fabric.state import env as fabenv from fabric.contrib import files -######### GLOBAL +from dicthelpers import fallbackdict + +# Use local SSH config for connections if available. +fabenv['use_ssh_config'] = True + +######## env wrapper +# global environment variables fallback to fabric env variables +# (also getting vars will do format mapping on strings with env vars) +env = fallbackdict(fabenv) + +######### GLOBALS +env.django_user = '{user}' # this will be different when sudo/django users are env.group = 'www-data' env.proj_repo = 'git@github.com:FinalsClub/karmaworld.git' -env.repo_root = '~/karmaworld' # transient setting for VMs only +env.repo_root = '/home/{django_user}/karmaworld' env.proj_root = '/var/www/karmaworld' env.branch = 'prod' # only used for supervisor conf two lines below. cleanup? env.code_root = env.proj_root -env.supervisor_conf = '{0}/confs/{1}/supervisord.conf'.format(env.code_root, env.branch) -env.usde_csv = '{0}/confs/accreditation.csv'.format(env.code_root) - -env.use_ssh_config = True +env.supervisor_conf = '{code_root}/confs/{branch}/supervisord.conf' +env.usde_csv = '{code_root}/confs/accreditation.csv' ######## Run Commands in Virtual Environment def virtenv_path(): @@ -211,19 +222,22 @@ def file_setup(): Deploy expected files and directories from non-apt system services. """ ini_parser = ConfigParser.SafeConfigParser() - if not ini_parser.read(env.supervisor_conf): - raise Exception("Could not parse INI file {0}".format(env.supervisor_conf)) + # read remote data into a file like object + data_flo = StringIO(run('cat {supervisor_conf}'.format(**env))) + ini_parser.readfp(data_flo) for section, option in (('supervisord','logfile'), ('supervisord','pidfile'), ('unix_http_server','file'), ('program:celeryd','stdout_logfile')): + if not ini_parser.has_section(section): + raise Exception("Could not parse INI file {supervisor_conf}".format(**env)) filepath = ini_parser.get(section, option) # generate file's directory structure if needed run('mkdir -p {0}'.format(os.path.split(filepath)[0])) # touch a file and change ownership if needed if 'log' in option and not files.exists(filepath): sudo('touch {0}'.format(filepath)) - sudo('chown {0}:{1} {2}'.format(env.user, env.group, filepath)) + sudo('chown {0}:{1} {2}'.format(env.django_user, env.group, filepath)) @task def check_secrets(): -- 2.25.1