-
""" Karmaworld Fabric management script
Finals Club (c) 2013"""
import os
-import requests
import ConfigParser
+from cStringIO import StringIO
-from bs4 import BeautifulSoup as BS
-from fabric.api import cd, env, lcd, prefix, run, sudo, task, local, settings
+from fabric.api import cd, lcd, prefix, run, sudo, task, local, settings
+from fabric.state import env as fabenv
from fabric.contrib import files
-######### GLOBAL
-env.user = 'vagrant'
-env.group = 'vagrant'
-env.proj_repo = 'git@github.com:FinalsClub/karmaworld.git'
-env.repo_root = '~/karmaworld'
-env.proj_root = '/var/www/karmaworld'
-env.branch = 'prod'
-env.code_root = env.proj_root
-env.env_root = env.proj_root
-env.supervisor_conf = '{0}/confs/{1}/supervisord.conf'.format(env.code_root, env.branch)
-
-env.use_ssh_config = True
-
-USDE_LINK = "http://ope.ed.gov/accreditation/GetDownloadFile.aspx"
-
-######## Define host(s)
-def here():
- """
- Connection information for the local machine
- """
- def _custom_local(command):
- prefixed_command = '/bin/bash -l -i -c "%s"' % command
- return local(prefixed_command)
-
- # This is required for the same reason as above
- env.proj_root = '/var/www/karmaworld'
- env.cd = lcd
- env.reqs = 'reqs/dev.txt'
- env.confs = 'confs/beta'
- env.branch = 'beta'
-
+from dicthelpers import fallbackdict
+# Use local SSH config for connections if available.
+fabenv['use_ssh_config'] = True
-####### Define production host
-@task
-def prod():
- """
- Connection Information for production machine
- """
+######## env wrapper
+# global environment variables fallback to fabric env variables
+# (also getting vars will do format mapping on strings with env vars)
+env = fallbackdict(fabenv)
- env.user = 'djkarma'
- env.hosts = ['karmanotes.org']
- env.proj_root = '/var/www/karmaworld'
- env.reqs = 'reqs/prod.txt'
- env.confs = 'confs/prod/'
- env.branch = 'beta'
- env.gunicorn_addr = '127.0.0.1:8000'
-
-####### Define beta host
-@task
-def beta():
- """
- Connection Information for beta machine
- """
-
- env.user = 'djkarma'
- env.hosts = ['beta.karmanotes.org']
- env.proj_root = '/var/www/karmaworld'
- env.reqs = 'reqs/prod.txt'
- env.confs = 'confs/prod/'
- env.branch = 'beta'
+######### GLOBALS
+env.django_user = '{user}' # this will be different when sudo/django users are
+env.group = 'www-data'
+env.proj_repo = 'git@github.com:FinalsClub/karmaworld.git'
+env.repo_root = '/home/{django_user}/karmaworld'
+env.proj_root = '/var/www/karmaworld'
+env.branch = 'prod' # only used for supervisor conf two lines below. cleanup?
+env.code_root = env.proj_root
+env.supervisor_conf = '{code_root}/confs/{branch}/supervisord.conf'
+env.usde_csv = '{code_root}/confs/accreditation.csv'
-######## Run Commands in Virutal Environment
+######## Run Commands in Virtual Environment
def virtenv_path():
"""
- Builds the virtualenv path.
- """
- # not much here now, but maybe useful later.
- return env.env_root
+ Find and memoize the virtualenv for use internally.
+ """
+ default_venv = env.proj_root + '/venv/bin/activate'
+
+ # Return environment root if its been memoized
+ if 'env_root' in env and env['env_root']:
+ return env['env_root']
+
+ # Not memoized. Try to find a single unique virtual environment.
+ with settings(warn_only=True):
+ outp = run("find -L {0} -path '*/bin/activate' | grep -v '/local/'".format(env.proj_root))
+ if not len(outp) or len(outp.splitlines()) != 1:
+ # Cannot find any virtualenv or found multiple virtualenvs.
+ if len(outp) and default_venv not in outp:
+ # Multiple venvs and the default is not present.
+ raise Exception('Cannot determine the appropriate virtualenv.')
+ # If there are no virtualenvs, then use the default (this will create
+ # one if being called by make_virtualenv, otherwise it will cause an
+ # error).
+ # If there are multiple virtualenvs and the default is in their midst,
+ # use the default.
+ outp = default_venv
+ # Pop off the /bin/activate from /venv/bin/activate
+ outp = os.path.sep.join(outp.split(os.path.sep)[:-2])
+ env['env_root'] = outp
+ return outp
def virtenv_exec(command):
"""
Execute command in Virtualenv
"""
-
- path = virtenv_path()
- with prefix('source {0}/bin/activate'.format(path)):
+ with prefix('source {0}/bin/activate'.format(virtenv_path())):
run(command)
######## Sync database
"""
Sync Database
"""
- virtenv_exec('{0}/manage.py syncdb --migrate'.format(env.code_root))
+ virtenv_exec('{0}/manage.py syncdb --migrate --noinput'.format(env.code_root))
####### Collect Static Files
Collect static files (if AWS config. present, push to S3)
"""
- virtenv_exec('%s/manage.py collectstatic --noinput' % env.code_root )
+ virtenv_exec('{0}/manage.py collectstatic --noinput'.format(env.code_root))
+
+####### Compress Static Files
+@task
+def compress_static():
+ """
+ Compress static files
+ """
+
+ virtenv_exec('{0}/manage.py compress'.format(env.code_root))
+
####### Run Dev Server
@task
Runs the built-in django webserver
"""
- virtenv_exec('%s/manage.py runserver' % env.code_root)
+ virtenv_exec('{0}/manage.py runserver'.format(env.code_root))
####### Create Virtual Environment
@task
def restart_supervisord():
"""
- Restarts supervisord
+ Restarts supervisord, also making sure to load in new config data.
"""
- stop_supervisord()
- start_supervisord()
+ virtenv_exec('supervisorctl -c {0} update; supervisorctl -c {0} restart all'.format(env.supervisor_conf))
def supervisorctl(action, process):
@task
-def start_celeryd():
+def start_celery():
"""
Starts the celeryd process
"""
@task
-def stop_celeryd():
+def stop_celery():
"""
Stops the celeryd process
"""
"""
supervisorctl('restart', 'gunicorn')
+@task
+def flush_memcache():
+ """
+ Clear everything cached in memcached
+ """
+ virtenv_exec('echo "flush_all" | nc localhost 11211')
####### Update Requirements
+@task
+def install_reqs():
+ # first install must be done without --upgrade for a few packages that break
+ # due to a pip problem.
+ virtenv_exec('pip install -r {0}/reqs/prod.txt'.format(env.code_root))
+
@task
def update_reqs():
- virtenv_exec('pip install -r {0}/reqs/{1}.txt'.format(env.repo_root, env.branch))
+ # this should generally work to install reqs too, save for a pip problem
+ # with a few packages.
+ virtenv_exec('pip install --upgrade -r {0}/reqs/prod.txt'.format(env.code_root))
####### Pull new code
@task
def update_code():
- virtenv_exec('cd %s; git pull' % env.proj_root )
+ virtenv_exec('cd {0}; git pull'.format(env.code_root))
def backup():
"""
Deploy expected files and directories from non-apt system services.
"""
ini_parser = ConfigParser.SafeConfigParser()
- if not ini_parser.read(env.supervisor_conf):
- raise Exception("Could not parse INI file {0}".format(env.supervisor_conf))
+ # read remote data into a file like object
+ data_flo = StringIO(run('cat {supervisor_conf}'.format(**env)))
+ ini_parser.readfp(data_flo)
for section, option in (('supervisord','logfile'),
('supervisord','pidfile'),
('unix_http_server','file'),
('program:celeryd','stdout_logfile')):
+ if not ini_parser.has_section(section):
+ raise Exception("Could not parse INI file {supervisor_conf}".format(**env))
filepath = ini_parser.get(section, option)
# generate file's directory structure if needed
run('mkdir -p {0}'.format(os.path.split(filepath)[0]))
# touch a file and change ownership if needed
if 'log' in option and not files.exists(filepath):
sudo('touch {0}'.format(filepath))
- sudo('chown {0}:{1} {2}'.format(env.user, env.group, filepath))
+ sudo('chown {0}:{1} {2}'.format(env.django_user, env.group, filepath))
@task
def check_secrets():
"""
secrets_path = env.code_root + '/karmaworld/secret'
- secrets_files = ('filepicker.py', 'static_s3.py', 'db_settings.py', 'drive.py', 'client_secrets.json')
+ secrets_files = ('filepicker.py', 'static_s3.py', 'drive.py', 'client_secrets.json', 'drive.p12')
errors = []
for sfile in secrets_files:
raise Exception('\n'.join(errors))
@task
-def fetch_accreditation():
- """
- Connects to USDE accreditation and drops a CSV into confs.
- """
- r = requests.get(USDE_LINK)
- # Ensure the page was retrieved with 200
- if not r.ok:
- r.raise_for_status()
-
- # Process the HTML with BeautifulSoup
- soup = BS(r.text)
- # Extract all the anchor links.
- a = soup.find_all('a')
- # TODO maybe hit up itertools for speed? Probably.
- # Extract the HREFs from anchors.
- def get_href(anchor):
- return anchor.get('href')
- a = map(get_href, a)
- # Filter out all but the Accreditation links.
- def contains_accreditation(link):
- return 'Accreditation' in link and 'zip' in link
- a = filter(contains_accreditation, a)
- # Find the most recent. (Accreditation_YYYY_MM.zip means alphanumeric sort)
- link = sorted(a)[-1]
-
- # Download the linked file to the FS and extract the CSV
- tempfile = '/tmp/accreditation.zip'
- csvfile = env.proj_root + '/confs/accreditation.csv'
- run('wget -B {0} -O {1} {2}'.format(USDE_LINK, tempfile, link))
- run("7z e -i'!*.csv' -so {0} >> {1}".format(tempfile, csvfile))
+def fetch_usde():
+ """
+ Download USDE accreditation school CSV.
+ """
+ virtenv_exec('{0}/manage.py fetch_usde_csv {1}'.format(env.code_root, env.usde_csv))
+
+@task
+def import_usde():
+ """
+ Import accreditation school CSV into the database and scrub it.
+ """
+ virtenv_exec('{0}/manage.py import_usde_csv {1}'.format(env.code_root, env.usde_csv))
+ virtenv_exec('{0}/manage.py sanitize_usde_schools'.format(env.code_root))
@task
def first_deploy():
make_virtualenv()
file_setup()
check_secrets()
- update_reqs()
+ install_reqs()
syncdb()
+ compress_static()
collect_static()
+ fetch_usde()
+ import_usde()
start_supervisord()
+ print "You should run `manage.py createsuperuser` in the virtual environment"
@task
update_code()
update_reqs()
syncdb()
+ compress_static()
collect_static()
restart_supervisord()
########## END COMMANDS
projects / oweals / karmaworld.git / blobdiff