2 """ Karmaworld Fabric management script
3 Finals Club (c) 2013"""
8 from fabric.api import cd, env, lcd, prefix, run, sudo, task, local, settings
9 from fabric.contrib import files
12 env.group = 'www-data'
13 env.proj_repo = 'git@github.com:FinalsClub/karmaworld.git'
14 env.repo_root = '~/karmaworld' # transient setting for VMs only
15 env.proj_root = '/var/www/karmaworld'
16 env.branch = 'prod' # only used for supervisor conf two lines below. cleanup?
17 env.code_root = env.proj_root
18 env.env_root = env.proj_root
19 env.supervisor_conf = '{0}/confs/{1}/supervisord.conf'.format(env.code_root, env.branch)
20 env.usde_csv = '{0}/confs/acceditation.csv'.format(env.code_root)
22 env.use_ssh_config = True
24 ######## Define host(s)
27 Connection information for the local machine
29 def _custom_local(command):
30 prefixed_command = '/bin/bash -l -i -c "%s"' % command
31 return local(prefixed_command)
33 # This is required for the same reason as above
34 env.proj_root = '/var/www/karmaworld'
36 env.reqs = 'reqs/dev.txt'
37 env.confs = 'confs/beta'
42 ####### Define production host
46 Connection Information for production machine
50 env.hosts = ['karmanotes.org']
51 env.proj_root = '/var/www/karmaworld'
52 env.reqs = 'reqs/prod.txt'
53 env.confs = 'confs/prod/'
55 env.gunicorn_addr = '127.0.0.1:8000'
57 ####### Define beta host
61 Connection Information for beta machine
65 env.hosts = ['beta.karmanotes.org']
66 env.proj_root = '/var/www/karmaworld'
67 env.reqs = 'reqs/prod.txt'
68 env.confs = 'confs/prod/'
71 ######## Run Commands in Virutal Environment
74 Builds the virtualenv path.
76 # not much here now, but maybe useful later.
79 def virtenv_exec(command):
81 Execute command in Virtualenv
85 with prefix('source {0}/bin/activate'.format(path)):
88 ######## Sync database
94 virtenv_exec('{0}/manage.py syncdb --migrate'.format(env.code_root))
97 ####### Collect Static Files
101 Collect static files (if AWS config. present, push to S3)
104 virtenv_exec('%s/manage.py collectstatic --noinput' % env.code_root )
106 ####### Run Dev Server
110 Runs the built-in django webserver
113 virtenv_exec('%s/manage.py runserver' % env.code_root)
115 ####### Create Virtual Environment
120 Link the karmaworld repo into the appropriate production location
122 if not files.exists(env.code_root):
123 run('ln -s {0} {1}'.format(env.repo_root, env.code_root))
126 def make_virtualenv():
128 Create our Virtualenv
130 run('virtualenv {0}'.format(virtenv_path()))
133 def start_supervisord():
137 virtenv_exec('supervisord -c {0}'.format(env.supervisor_conf))
141 def stop_supervisord():
145 virtenv_exec('supervisorctl -c {0} shutdown'.format(env.supervisor_conf))
149 def restart_supervisord():
151 Restarts supervisord, also making sure to load in new config data.
153 virtenv_exec('supervisorctl -c {0} update'.format(env.supervisor_conf))
154 virtenv_exec('supervisorctl -c {0} restart all'.format(env.supervisor_conf))
157 def supervisorctl(action, process):
159 Takes as arguments the name of the process as is
160 defined in supervisord.conf and the action that should
161 be performed on it: start|stop|restart.
163 virtenv_exec('supervisorctl -c {0} {1} {2}'.format(env.supervisor_conf, action, process))
169 Starts the celeryd process
171 supervisorctl('start', 'celeryd')
177 Stops the celeryd process
179 supervisorctl('stop', 'celeryd')
183 def restart_celery():
185 Restarts the celeryd process
187 supervisorctl('restart', 'celeryd')
191 def start_gunicorn():
193 Starts the gunicorn process
195 supervisorctl('start', 'gunicorn')
201 Stops the gunicorn process
203 supervisorctl('stop', 'gunicorn')
207 def restart_gunicorn():
209 Restarts the gunicorn process
211 supervisorctl('restart', 'gunicorn')
214 ####### Update Requirements
217 virtenv_exec('pip install -r {0}/reqs/{1}.txt'.format(env.repo_root, env.branch))
219 ####### Pull new code
222 virtenv_exec('cd %s; git pull' % env.proj_root )
226 Create backup using bup
233 Deploy expected files and directories from non-apt system services.
235 ini_parser = ConfigParser.SafeConfigParser()
236 if not ini_parser.read(env.supervisor_conf):
237 raise Exception("Could not parse INI file {0}".format(env.supervisor_conf))
238 for section, option in (('supervisord','logfile'),
239 ('supervisord','pidfile'),
240 ('unix_http_server','file'),
241 ('program:celeryd','stdout_logfile')):
242 filepath = ini_parser.get(section, option)
243 # generate file's directory structure if needed
244 run('mkdir -p {0}'.format(os.path.split(filepath)[0]))
245 # touch a file and change ownership if needed
246 if 'log' in option and not files.exists(filepath):
247 sudo('touch {0}'.format(filepath))
248 sudo('chown {0}:{1} {2}'.format(env.local_user, env.group, filepath))
253 Ensure secret files exist for syncdb to run.
256 secrets_path = env.code_root + '/karmaworld/secret'
257 secrets_files = ('filepicker.py', 'static_s3.py', 'db_settings.py', 'drive.py', 'client_secrets.json', 'drive.p12')
260 for sfile in secrets_files:
261 ffile = os.path.sep.join((secrets_path,sfile))
262 if not files.exists(ffile):
263 errors.append('{0} missing. Please add and try again.'.format(ffile))
265 raise Exception('\n'.join(errors))
270 Download USDE accreditation school CSV.
272 virtenv_exec('{0}/manage.py fetch_usde_csv {1}'.format(env.code_root, env.usde_csv))
277 Import accreditation school CSV into the database and scrub it.
279 virtenv_exec('{0}/manage.py import_usde_csv {1}'.format(env.code_root, env.usde_csv))
280 virtenv_exec('{0}/manage.py sanitize_usde_schools'.format(env.code_root))
283 def install_pdf2htmlEX():
285 # Some things we need:
286 sudo apt-get install cmake libpng-dev libjpeg-dev libgtk2.0-dev pkg-config libfontconfig1-dev autoconf libtool
288 # Ubuntu 12.04 comes with a version of poppler that is too
289 # old, so compile our own
290 wget http://poppler.freedesktop.org/poppler-0.24.4.tar.xz
291 tar xf poppler-0.24.4.tar.gz
292 ./configure --prefix=/usr --enable-xpdf-headers
296 # Ubuntu 12.04 comes with a version of fontforge that is too
297 # old, so compile our own
298 git clone https://github.com/fontforge/fontforge.git
300 ./configure --prefix=/usr
305 wget https://github.com/coolwanglu/pdf2htmlEX/archive/v0.10.tar.gz
312 print "not implemented yet!"
317 Sets up and deploys the project for the first time.
334 Deploys the latest changes
340 restart_supervisord()
341 ########## END COMMANDS