2 """ Karmaworld Fabric management script
3 Finals Club (c) 2013"""
8 from fabric.api import cd, env, lcd, prefix, run, sudo, task, local, settings
9 from fabric.contrib import files
12 env.group = 'www-data'
13 env.proj_repo = 'git@github.com:FinalsClub/karmaworld.git'
14 env.repo_root = '~/karmaworld' # transient setting for VMs only
15 env.proj_root = '/var/www/karmaworld'
16 env.branch = 'prod' # only used for supervisor conf two lines below. cleanup?
17 env.code_root = env.proj_root
18 env.supervisor_conf = '{0}/confs/{1}/supervisord.conf'.format(env.code_root, env.branch)
19 env.usde_csv = '{0}/confs/acceditation.csv'.format(env.code_root)
21 env.use_ssh_config = True
23 ######## Run Commands in Virtual Environment
26 Find and memoize the virtualenv for use internally.
28 default_venv = env.proj_root + '/venv/bin/activate'
30 # Return environment root if its been memoized
31 if 'env_root' in env and env['env_root']:
32 return env['env_root']
34 # Not memoized. Try to find a single unique virtual environment.
35 outp = run("find -L {0} -path '*/bin/activate' | grep -v '/local/'".format(env.proj_root))
36 if not len(outp) or len(outp.splitlines()) != 1:
37 # Cannot find any virtualenv or found multiple virtualenvs.
38 if len(outp) and default_venv not in outp:
39 # Multiple venvs and the default is not present.
40 raise Exception('Cannot determine the appropriate virtualenv.')
41 # If there are no virtualenvs, then use the default (this will create
42 # one if being called by make_virtualenv, otherwise it will cause an
44 # If there are multiple virtualenvs and the default is in their midst,
47 # Pop off the /bin/activate from /venv/bin/activate
48 outp = os.path.sep.join(outp.split(os.path.sep)[:-2])
49 env['env_root'] = outp
52 def virtenv_exec(command):
54 Execute command in Virtualenv
56 with prefix('source {0}/bin/activate'.format(virtenv_path())):
59 ######## Sync database
65 virtenv_exec('{0}/manage.py syncdb --migrate'.format(env.code_root))
68 ####### Collect Static Files
72 Collect static files (if AWS config. present, push to S3)
75 virtenv_exec('{0}/manage.py collectstatic --noinput'.format(env.code_root))
77 ####### Run Dev Server
81 Runs the built-in django webserver
84 virtenv_exec('{0}/manage.py runserver'.format(env.code_root))
86 ####### Create Virtual Environment
91 Link the karmaworld repo into the appropriate production location
93 if not files.exists(env.code_root):
94 run('ln -s {0} {1}'.format(env.repo_root, env.code_root))
97 def make_virtualenv():
101 run('virtualenv {0}'.format(virtenv_path()))
104 def start_supervisord():
108 virtenv_exec('supervisord -c {0}'.format(env.supervisor_conf))
112 def stop_supervisord():
116 virtenv_exec('supervisorctl -c {0} shutdown'.format(env.supervisor_conf))
120 def restart_supervisord():
122 Restarts supervisord, also making sure to load in new config data.
124 virtenv_exec('supervisorctl -c {0} update; supervisorctl -c {0} restart all'.format(env.supervisor_conf))
127 def supervisorctl(action, process):
129 Takes as arguments the name of the process as is
130 defined in supervisord.conf and the action that should
131 be performed on it: start|stop|restart.
133 virtenv_exec('supervisorctl -c {0} {1} {2}'.format(env.supervisor_conf, action, process))
139 Starts the celeryd process
141 supervisorctl('start', 'celeryd')
147 Stops the celeryd process
149 supervisorctl('stop', 'celeryd')
153 def restart_celery():
155 Restarts the celeryd process
157 supervisorctl('restart', 'celeryd')
161 def start_gunicorn():
163 Starts the gunicorn process
165 supervisorctl('start', 'gunicorn')
171 Stops the gunicorn process
173 supervisorctl('stop', 'gunicorn')
177 def restart_gunicorn():
179 Restarts the gunicorn process
181 supervisorctl('restart', 'gunicorn')
184 ####### Update Requirements
187 virtenv_exec('pip install -r {0}/reqs/prod.txt'.format(env.code_root))
189 ####### Pull new code
192 virtenv_exec('cd {0}; git pull'.format(env.code_root))
196 Create backup using bup
203 Deploy expected files and directories from non-apt system services.
205 ini_parser = ConfigParser.SafeConfigParser()
206 if not ini_parser.read(env.supervisor_conf):
207 raise Exception("Could not parse INI file {0}".format(env.supervisor_conf))
208 for section, option in (('supervisord','logfile'),
209 ('supervisord','pidfile'),
210 ('unix_http_server','file'),
211 ('program:celeryd','stdout_logfile')):
212 filepath = ini_parser.get(section, option)
213 # generate file's directory structure if needed
214 run('mkdir -p {0}'.format(os.path.split(filepath)[0]))
215 # touch a file and change ownership if needed
216 if 'log' in option and not files.exists(filepath):
217 sudo('touch {0}'.format(filepath))
218 sudo('chown {0}:{1} {2}'.format(env.local_user, env.group, filepath))
223 Ensure secret files exist for syncdb to run.
226 secrets_path = env.code_root + '/karmaworld/secret'
227 secrets_files = ('filepicker.py', 'static_s3.py', 'db_settings.py', 'drive.py', 'client_secrets.json', 'drive.p12')
230 for sfile in secrets_files:
231 ffile = os.path.sep.join((secrets_path,sfile))
232 if not files.exists(ffile):
233 errors.append('{0} missing. Please add and try again.'.format(ffile))
235 raise Exception('\n'.join(errors))
240 Download USDE accreditation school CSV.
242 virtenv_exec('{0}/manage.py fetch_usde_csv {1}'.format(env.code_root, env.usde_csv))
247 Import accreditation school CSV into the database and scrub it.
249 virtenv_exec('{0}/manage.py import_usde_csv {1}'.format(env.code_root, env.usde_csv))
250 virtenv_exec('{0}/manage.py sanitize_usde_schools'.format(env.code_root))
253 def install_pdf2htmlEX():
255 # Some things we need:
256 sudo apt-get install cmake libpng-dev libjpeg-dev libgtk2.0-dev pkg-config libfontconfig1-dev autoconf libtool
258 # Ubuntu 12.04 comes with a version of poppler that is too
259 # old, so compile our own
260 wget http://poppler.freedesktop.org/poppler-0.24.4.tar.xz
261 tar xf poppler-0.24.4.tar.gz
262 ./configure --prefix=/usr --enable-xpdf-headers
266 # Ubuntu 12.04 comes with a version of fontforge that is too
267 # old, so compile our own
268 git clone https://github.com/fontforge/fontforge.git
270 ./configure --prefix=/usr
275 wget https://github.com/coolwanglu/pdf2htmlEX/archive/v0.10.tar.gz
282 print "not implemented yet!"
287 Sets up and deploys the project for the first time.
304 Deploys the latest changes
310 restart_supervisord()
311 ########## END COMMANDS