1 """ Karmaworld Fabric management script
2 Finals Club (c) 2013"""
6 from cStringIO import StringIO
8 from fabric.api import cd, lcd, prefix, run, sudo, task, local, settings
9 from fabric.state import env as fabenv
10 from fabric.contrib import files
12 from dicthelpers import fallbackdict
14 # Use local SSH config for connections if available.
15 fabenv['use_ssh_config'] = True
18 # global environment variables fallback to fabric env variables
19 # (also getting vars will do format mapping on strings with env vars)
20 env = fallbackdict(fabenv)
23 env.django_user = '{user}' # this will be different when sudo/django users are
24 env.group = 'www-data'
25 env.proj_repo = 'git@github.com:FinalsClub/karmaworld.git'
26 env.repo_root = '/home/{django_user}/karmaworld'
27 env.proj_root = '/var/www/karmaworld'
28 env.branch = 'prod' # only used for supervisor conf two lines below. cleanup?
29 env.code_root = env.proj_root
30 env.supervisor_conf = '{code_root}/confs/{branch}/supervisord.conf'
31 env.usde_csv = '{code_root}/confs/accreditation.csv'
33 ######## Run Commands in Virtual Environment
36 Find and memoize the virtualenv for use internally.
38 default_venv = env.proj_root + '/venv/bin/activate'
40 # Return environment root if its been memoized
41 if 'env_root' in env and env['env_root']:
42 return env['env_root']
44 # Not memoized. Try to find a single unique virtual environment.
45 with settings(warn_only=True):
46 outp = run("find -L {0} -path '*/bin/activate' | grep -v '/local/'".format(env.proj_root))
47 if not len(outp) or len(outp.splitlines()) != 1:
48 # Cannot find any virtualenv or found multiple virtualenvs.
49 if len(outp) and default_venv not in outp:
50 # Multiple venvs and the default is not present.
51 raise Exception('Cannot determine the appropriate virtualenv.')
52 # If there are no virtualenvs, then use the default (this will create
53 # one if being called by make_virtualenv, otherwise it will cause an
55 # If there are multiple virtualenvs and the default is in their midst,
58 # Pop off the /bin/activate from /venv/bin/activate
59 outp = os.path.sep.join(outp.split(os.path.sep)[:-2])
60 env['env_root'] = outp
63 def virtenv_exec(command):
65 Execute command in Virtualenv
67 with prefix('source {0}/bin/activate'.format(virtenv_path())):
70 ######## Sync database
76 virtenv_exec('{0}/manage.py syncdb --migrate --noinput'.format(env.code_root))
79 ####### Collect Static Files
83 Collect static files (if AWS config. present, push to S3)
86 virtenv_exec('{0}/manage.py collectstatic --noinput'.format(env.code_root))
88 ####### Compress Static Files
90 def compress_static():
95 virtenv_exec('{0}/manage.py compress'.format(env.code_root))
98 ####### Run Dev Server
102 Runs the built-in django webserver
105 virtenv_exec('{0}/manage.py runserver'.format(env.code_root))
107 ####### Create Virtual Environment
112 Link the karmaworld repo into the appropriate production location
114 if not files.exists(env.code_root):
115 run('ln -s {0} {1}'.format(env.repo_root, env.code_root))
118 def make_virtualenv():
120 Create our Virtualenv
122 run('virtualenv {0}'.format(virtenv_path()))
125 def start_supervisord():
129 virtenv_exec('supervisord -c {0}'.format(env.supervisor_conf))
133 def stop_supervisord():
137 virtenv_exec('supervisorctl -c {0} shutdown'.format(env.supervisor_conf))
141 def restart_supervisord():
143 Restarts supervisord, also making sure to load in new config data.
145 virtenv_exec('supervisorctl -c {0} update; supervisorctl -c {0} restart all'.format(env.supervisor_conf))
148 def supervisorctl(action, process):
150 Takes as arguments the name of the process as is
151 defined in supervisord.conf and the action that should
152 be performed on it: start|stop|restart.
154 virtenv_exec('supervisorctl -c {0} {1} {2}'.format(env.supervisor_conf, action, process))
160 Starts the celeryd process
162 supervisorctl('start', 'celeryd')
168 Stops the celeryd process
170 supervisorctl('stop', 'celeryd')
174 def restart_celery():
176 Restarts the celeryd process
178 supervisorctl('restart', 'celeryd')
182 def start_gunicorn():
184 Starts the gunicorn process
186 supervisorctl('start', 'gunicorn')
192 Stops the gunicorn process
194 supervisorctl('stop', 'gunicorn')
198 def restart_gunicorn():
200 Restarts the gunicorn process
202 supervisorctl('restart', 'gunicorn')
205 ####### Update Requirements
208 # first install must be done without --upgrade for a few packages that break
209 # due to a pip problem.
210 virtenv_exec('pip install -r {0}/reqs/prod.txt'.format(env.code_root))
214 # this should generally work to install reqs too, save for a pip problem
215 # with a few packages.
216 virtenv_exec('pip install --upgrade -r {0}/reqs/prod.txt'.format(env.code_root))
218 ####### Pull new code
221 virtenv_exec('cd {0}; git pull'.format(env.code_root))
225 Create backup using bup
232 Deploy expected files and directories from non-apt system services.
234 ini_parser = ConfigParser.SafeConfigParser()
235 # read remote data into a file like object
236 data_flo = StringIO(run('cat {supervisor_conf}'.format(**env)))
237 ini_parser.readfp(data_flo)
238 for section, option in (('supervisord','logfile'),
239 ('supervisord','pidfile'),
240 ('unix_http_server','file'),
241 ('program:celeryd','stdout_logfile')):
242 if not ini_parser.has_section(section):
243 raise Exception("Could not parse INI file {supervisor_conf}".format(**env))
244 filepath = ini_parser.get(section, option)
245 # generate file's directory structure if needed
246 run('mkdir -p {0}'.format(os.path.split(filepath)[0]))
247 # touch a file and change ownership if needed
248 if 'log' in option and not files.exists(filepath):
249 sudo('touch {0}'.format(filepath))
250 sudo('chown {0}:{1} {2}'.format(env.django_user, env.group, filepath))
255 Ensure secret files exist for syncdb to run.
258 secrets_path = env.code_root + '/karmaworld/secret'
259 secrets_files = ('filepicker.py', 'static_s3.py', 'db_settings.py', 'drive.py', 'client_secrets.json', 'drive.p12')
262 for sfile in secrets_files:
263 ffile = os.path.sep.join((secrets_path,sfile))
264 if not files.exists(ffile):
265 errors.append('{0} missing. Please add and try again.'.format(ffile))
267 raise Exception('\n'.join(errors))
272 Download USDE accreditation school CSV.
274 virtenv_exec('{0}/manage.py fetch_usde_csv {1}'.format(env.code_root, env.usde_csv))
279 Import accreditation school CSV into the database and scrub it.
281 virtenv_exec('{0}/manage.py import_usde_csv {1}'.format(env.code_root, env.usde_csv))
282 virtenv_exec('{0}/manage.py sanitize_usde_schools'.format(env.code_root))
287 Sets up and deploys the project for the first time.
299 print "You should run `manage.py createsuperuser` in the virtual environment"
305 Deploys the latest changes
312 restart_supervisord()
313 ########## END COMMANDS