From: Bryan <btbonval@gmail.com>
Date: Thu, 20 Feb 2014 10:39:13 +0000 (-0500)
Subject: making fabric better for VM deployment and btw #335
X-Git-Tag: release-20150131~163^2~13
X-Git-Url: https://git.librecmc.org/?a=commitdiff_plain;h=a583f311a025ffa8c689f134bdec70564250ac75;p=oweals%2Fkarmaworld.git

making fabric better for VM deployment and btw #335
---

diff --git a/dicthelpers.py b/dicthelpers.py
new file mode 100644
index 0000000..8ee08c6
--- /dev/null
+++ b/dicthelpers.py
@@ -0,0 +1,126 @@
+"""
+Karmaworld Dictionary helpers
+Finals Club (c) 2014
+Author: Bryan Bonvallet
+"""
+
+# I'd wager none of these are thread safe.
+
+import re
+from collections import MutableMapping
+
+
+needs_mapping = re.compile('{[a-zA-Z_]\w*}')
+
+
+class attrdict(object):
+    """ Access dictionary by object attributes. """
+    def __getattr__(self, attr):
+        # create call stack, if not already there, to avoid loops
+        try:
+            callstack = object.__getattribute__(self, '__callstack')
+        except AttributeError:
+            object.__setattr__(self, '__callstack', [])
+            callstack = object.__getattribute__(self, '__callstack')
+
+        try:
+            # don't call something already on the stack
+            if attr in callstack:
+                raise KeyError()
+            # track this attribute before possibly recursing
+            callstack.append(attr)
+            retattr = self.__getitem__(attr)
+            # success, remove attr from stack
+            callstack.remove(attr)
+            return retattr
+        except KeyError:
+            # if code is here, attr is not in dict
+            try:
+                # try to grab attr from the object
+                retattr = super(attrdict, self).__getattribute__(attr)
+                return retattr
+            finally:
+                # remove stack now that the attribute succeeded or failed
+                callstack.remove(attr)
+
+    def __setattr__(self, attr, value):
+        self.__setitem__(attr, value)
+
+    def __delattr__(self, attr):
+        self.__delitem__(attr)
+
+
+class fallbackdict(MutableMapping, attrdict):
+    """
+    Retrieve default values from a fallback dictionary and dynamically .format
+    """
+    def __init__(self, fallback, *args, **kwargs):
+        """
+        Supply dictionary to fall back to when keys are missing.
+        Other arguments are handled as normal dictionary.
+        """
+        # dodge attrdict problems by using object.__setattr__
+        classname = type(self).__name__
+        object.__setattr__(self, '_{0}__internaldict'.format(classname), {})
+        object.__setattr__(self, '_{0}__fallback'.format(classname), fallback)
+        object.__setattr__(self, '_{0}__fetching'.format(classname), [])
+        super(fallbackdict, self).__init__(*args, **kwargs)
+
+    def __needs_format__(self, value):
+        """ Helper to determine when a string needs formatting. """
+        return hasattr(value, 'format') and needs_mapping.search(value)
+
+    def __getitem__(self, key):
+        """
+        Use this dict's value if it has one otherwise grab value from parent
+        and populate any strings with format keyword arguments.
+        """
+        indict = self.__internaldict
+        fetching = self.__fetching
+
+        local = (key in indict)
+        # this will throw a key error if the key isn't in either place
+        # which is desirable
+        value = indict[key] if local else self.__fallback[key]
+
+        if (key in fetching) or (not self.__needs_format__(value)):
+            # already seeking this key in a recursed call
+            # or it doesn't need any formatting
+            # so return as-is
+            return value
+
+        # if the code is this far, strings needs formatting
+        # **self will call this function recursively.
+        # prevent infinite recursion from e.g. d['recurse'] = '{recurse}'
+        fetching.append(key)
+        value = value.format(**self)
+        # undo infinite recursion prevention
+        fetching.remove(key)
+        return value
+
+    def __setitem__(self, key, value):
+        """
+        Set the internal dict with key/value.
+        """
+        self.__internaldict[key] = value
+
+    def __delitem__(self, key):
+        """ Delete the key from the internal dict. """
+        del self.__internaldict[key]
+
+    def __uniquekeys__(self):
+        """ Returns unique keys between this dictionary and the fallback. """
+        mykeys = set(self.__internaldict.keys())
+        fbkeys = set(self.__fallback.keys())
+        # set union: all keys from both, no repeats.
+        return (mykeys | fbkeys)
+
+    def __iter__(self):
+        """ Returns the keys in this dictionary and the fallback. """
+        return iter(self.__uniquekeys__())
+
+    def __len__(self):
+        """
+        Returns the number of keys in this dictionary and the fallback.
+        """
+        return len(self.__uniquekeys__())
diff --git a/dicthelperstest.py b/dicthelperstest.py
new file mode 100644
index 0000000..eb64e19
--- /dev/null
+++ b/dicthelperstest.py
@@ -0,0 +1,40 @@
+# I need to turn this into a proper unit test.
+# uhm, this is horrible. sorry everybody.
+# -Bryan
+
+import time
+from dicthelpers import fallbackdict
+
+start = time.time()
+
+fallback = {1: '1'}
+poop = fallbackdict(fallback)
+print 'expect 1 (fallback): ' + poop[1]
+fallback[2] = '2'
+print 'expect 2 (dynamic fallback): ' + poop[2]
+poop[2] = '3'
+print 'expect 3 (overridden): ' + poop[2]
+del poop[2]
+print 'expect 2 (fallback after delete): ' + poop[2]
+fallback['three'] = 3
+fallback[3] = '3{three}'
+print 'expect 33 (formatting): ' + poop[3]
+print 'expect 4 (len): ' + str(len(poop))
+fallback['infinite'] = '{infinite}'
+print 'simple infinite recursion test: ' + poop['infinite']
+fallback['infone'] = '{inftwo}'
+fallback['inftwo'] = '{infone}'
+print 'double infinite recursion test: ' + poop['infone']
+fallback['2infone'] = '{2inftwo}'
+fallback['2inftwo'] = '{2infone}'
+print 'double infinite recursion test: ' + poop['2inftwo']
+poop['four'] = 4
+poop[4] = '4{four}'
+print 'expect 34 (self as string mapping): {three}{four}'.format(**poop)
+print 'expect 44 (self formatting): ' + poop[4]
+# silent iter length processing.
+# this used to take 1 or 2 seconds prior to optimizations
+for x in poop: poop[x]
+
+stop = time.time()
+print "execution " + str(stop - start)
diff --git a/fabfile.py b/fabfile.py
index d76384d..88dc5bc 100644
--- a/fabfile.py
+++ b/fabfile.py
@@ -3,21 +3,32 @@
 
 import os
 import ConfigParser
+from cStringIO import StringIO
 
-from fabric.api import cd, env, lcd, prefix, run, sudo, task, local, settings
+from fabric.api import cd, lcd, prefix, run, sudo, task, local, settings
+from fabric.state import env as fabenv
 from fabric.contrib import files
 
-######### GLOBAL
+from dicthelpers import fallbackdict
+
+# Use local SSH config for connections if available.
+fabenv['use_ssh_config'] = True
+
+######## env wrapper
+# global environment variables fallback to fabric env variables
+# (also getting vars will do format mapping on strings with env vars)
+env = fallbackdict(fabenv)
+
+######### GLOBALS
+env.django_user = '{user}' # this will be different when sudo/django users are
 env.group = 'www-data'
 env.proj_repo = 'git@github.com:FinalsClub/karmaworld.git'
-env.repo_root = '~/karmaworld' # transient setting for VMs only
+env.repo_root = '/home/{django_user}/karmaworld'
 env.proj_root = '/var/www/karmaworld'
 env.branch = 'prod' # only used for supervisor conf two lines below. cleanup?
 env.code_root = env.proj_root
-env.supervisor_conf = '{0}/confs/{1}/supervisord.conf'.format(env.code_root, env.branch)
-env.usde_csv = '{0}/confs/accreditation.csv'.format(env.code_root)
-
-env.use_ssh_config = True
+env.supervisor_conf = '{code_root}/confs/{branch}/supervisord.conf'
+env.usde_csv = '{code_root}/confs/accreditation.csv'
 
 ######## Run Commands in Virtual Environment
 def virtenv_path():
@@ -211,19 +222,22 @@ def file_setup():
     Deploy expected files and directories from non-apt system services.
     """
     ini_parser = ConfigParser.SafeConfigParser()
-    if not ini_parser.read(env.supervisor_conf):
-      raise Exception("Could not parse INI file {0}".format(env.supervisor_conf))
+    # read remote data into a file like object
+    data_flo = StringIO(run('cat {supervisor_conf}'.format(**env)))
+    ini_parser.readfp(data_flo)
     for section, option in (('supervisord','logfile'),
                             ('supervisord','pidfile'),
                             ('unix_http_server','file'),
                             ('program:celeryd','stdout_logfile')):
+      if not ini_parser.has_section(section):
+          raise Exception("Could not parse INI file {supervisor_conf}".format(**env))
       filepath = ini_parser.get(section, option)
       # generate file's directory structure if needed
       run('mkdir -p {0}'.format(os.path.split(filepath)[0]))
       # touch a file and change ownership if needed
       if 'log' in option and not files.exists(filepath):
           sudo('touch {0}'.format(filepath))
-          sudo('chown {0}:{1} {2}'.format(env.user, env.group, filepath))
+          sudo('chown {0}:{1} {2}'.format(env.django_user, env.group, filepath))
 
 @task
 def check_secrets():