X-Git-Url: https://git.librecmc.org/?a=blobdiff_plain;f=fabfile.py;h=ef39c4ae159ee516793ca83a523ec6159741419a;hb=1e9fdbf37e6c258927e53ba48702aed04ebe983d;hp=90fa97732dfbb8a4518e3d40b52008b52ca36ad7;hpb=3eae33636cc75bf73cd279306d0cbadb5146b236;p=oweals%2Fkarmaworld.git diff --git a/fabfile.py b/fabfile.py old mode 100755 new mode 100644 index 90fa977..ef39c4a --- a/fabfile.py +++ b/fabfile.py @@ -1,155 +1,311 @@ -"""Management utilities.""" -import os -from contextlib import contextmanager as _contextmanager +""" Karmaworld Fabric management script + Finals Club (c) 2013""" -from fabric.contrib.console import confirm -from fabric.api import abort, cd, env, local, prefix, run, settings, task +import os +import ConfigParser +from fabric.api import cd, env, lcd, prefix, run, sudo, task, local, settings +from fabric.contrib import files -########## GLOBALS +######### GLOBAL +env.group = 'www-data' env.proj_repo = 'git@github.com:FinalsClub/karmaworld.git' -env.virtualenv = 'venv-kw' -env.activate = 'workon %s' % env.virtualenv -env.run = './manage.py' -########## END GLOBALS +env.repo_root = '~/karmaworld' # transient setting for VMs only +env.proj_root = '/var/www/karmaworld' +env.branch = 'prod' # only used for supervisor conf two lines below. cleanup? +env.code_root = env.proj_root +env.supervisor_conf = '{0}/confs/{1}/supervisord.conf'.format(env.code_root, env.branch) +env.usde_csv = '{0}/confs/acceditation.csv'.format(env.code_root) + +env.use_ssh_config = True + +######## Run Commands in Virtual Environment +def virtenv_path(): + """ + Find and memoize the virtualenv for use internally. + """ + default_venv = env.proj_root + '/venv/bin/activate' + + # Return environment root if its been memoized + if 'env_root' in env and env['env_root']: + return env['env_root'] + + # Not memoized. Try to find a single unique virtual environment. + outp = run("find -L {0} -path '*/bin/activate' | grep -v '/local/'".format(env.proj_root)) + if not len(outp) or len(outp.splitlines()) != 1: + # Cannot find any virtualenv or found multiple virtualenvs. + if len(outp) and default_venv not in outp: + # Multiple venvs and the default is not present. + raise Exception('Cannot determine the appropriate virtualenv.') + # If there are no virtualenvs, then use the default (this will create + # one if being called by make_virtualenv, otherwise it will cause an + # error). + # If there are multiple virtualenvs and the default is in their midst, + # use the default. + outp = default_venv + # Pop off the /bin/activate from /venv/bin/activate + outp = os.path.sep.join(outp.split(os.path.sep)[:-2]) + env['env_root'] = outp + return outp + +def virtenv_exec(command): + """ + Execute command in Virtualenv + """ + with prefix('source {0}/bin/activate'.format(virtenv_path())): + run(command) + +######## Sync database +@task +def syncdb(): + """ + Sync Database + """ + virtenv_exec('{0}/manage.py syncdb --migrate'.format(env.code_root)) -########## HELPERS -def cont(cmd, message): - """Given a command, ``cmd``, and a message, ``message``, allow a user to - either continue or break execution if errors occur while executing ``cmd``. +####### Collect Static Files +@task +def collect_static(): + """ + Collect static files (if AWS config. present, push to S3) + """ - :param str cmd: The command to execute on the local system. - :param str message: The message to display to the user on failure. + virtenv_exec('{0}/manage.py collectstatic --noinput'.format(env.code_root)) - .. note:: - ``message`` should be phrased in the form of a question, as if ``cmd``'s - execution fails, we'll ask the user to press 'y' or 'n' to continue or - cancel exeuction, respectively. +####### Run Dev Server +@task +def dev_server(): + """ + Runs the built-in django webserver + """ + + virtenv_exec('{0}/manage.py runserver'.format(env.code_root)) - Usage:: +####### Create Virtual Environment + +@task +def link_code(): + """ + Link the karmaworld repo into the appropriate production location + """ + if not files.exists(env.code_root): + run('ln -s {0} {1}'.format(env.repo_root, env.code_root)) - cont('heroku run ...', "Couldn't complete %s. Continue anyway?" % cmd) +@task +def make_virtualenv(): + """ + Create our Virtualenv """ - with settings(warn_only=True): - result = local(cmd, capture=True) + run('virtualenv {0}'.format(virtenv_path())) - if message and result.failed and not confirm(message): - abort('Stopped execution per user request.') +@task +def start_supervisord(): + """ + Starts supervisord + """ + virtenv_exec('supervisord -c {0}'.format(env.supervisor_conf)) -@_contextmanager -def _virtualenv(): +@task +def stop_supervisord(): """ - Changes to the proj_dir and activates the virtualenv + Restarts supervisord """ - with cd(env.proj_dir): - with prefix(env.activate): - yield + virtenv_exec('supervisorctl -c {0} shutdown'.format(env.supervisor_conf)) -########## END HELPERS -########## ENVIRONMENTS -def beta(): +@task +def restart_supervisord(): """ - Beta connection information + Restarts supervisord, also making sure to load in new config data. """ - env.user = 'djkarma' - env.hosts = ['beta.karmanotes.org'] - env.proj_root = '/var/www/karmaworld' - env.proj_dir = os.path.join(env.proj_root, 'karmaworld') + virtenv_exec('supervisorctl -c {0} update; supervisorctl -c {0} restart all'.format(env.supervisor_conf)) -def prod(): +def supervisorctl(action, process): """ - Production connection information + Takes as arguments the name of the process as is + defined in supervisord.conf and the action that should + be performed on it: start|stop|restart. """ - env.user = 'djkarma' - env.hosts = ['karmanotes.org'] - env.proj_root = '/var/www/karmaworld' - env.proj_dir = os.path.join(env.proj_root, 'karmaworld') -########## END ENVIRONMENTS + virtenv_exec('supervisorctl -c {0} {1} {2}'.format(env.supervisor_conf, action, process)) -########## DATABASE MANAGEMENT @task -def syncdb(): - """Run a syncdb.""" - local('%(run)s syncdb --noinput' % env) +def start_celeryd(): + """ + Starts the celeryd process + """ + supervisorctl('start', 'celeryd') @task -def migrate(app=None): - """Apply one (or more) migrations. If no app is specified, fabric will - attempt to run a site-wide migration. +def stop_celeryd(): + """ + Stops the celeryd process + """ + supervisorctl('stop', 'celeryd') + - :param str app: Django app name to migrate. +@task +def restart_celery(): + """ + Restarts the celeryd process """ - if app: - local('%s migrate %s --noinput' % (env.run, app)) - else: - local('%(run)s migrate --noinput' % env) -########## END DATABASE MANAGEMENT + supervisorctl('restart', 'celeryd') -########## FILE MANAGEMENT @task -def collectstatic(): - """Collect all static files, and copy them to S3 for production usage.""" - local('%(run)s collectstatic --noinput' % env) -########## END FILE MANAGEMENT +def start_gunicorn(): + """ + Starts the gunicorn process + """ + supervisorctl('start', 'gunicorn') -########## COMMANDS +@task +def stop_gunicorn(): + """ + Stops the gunicorn process + """ + supervisorctl('stop', 'gunicorn') -def make_virtualenv(): + +@task +def restart_gunicorn(): """ - Creates a virtualenv on the remote host + Restarts the gunicorn process """ - run('mkvirtualenv %s' % env.virtualenv) + supervisorctl('restart', 'gunicorn') +####### Update Requirements +@task def update_reqs(): + virtenv_exec('pip install -r {0}/reqs/prod.txt'.format(env.code_root)) + +####### Pull new code +@task +def update_code(): + virtenv_exec('cd {0}; git pull'.format(env.code_root)) + +def backup(): """ - Makes sure all packages listed in requirements are installed + Create backup using bup """ - with _virtualenv(): - with cd(env.proj_dir): - run('pip install -r requirements/production.pip') + pass +@task +def file_setup(): + """ + Deploy expected files and directories from non-apt system services. + """ + ini_parser = ConfigParser.SafeConfigParser() + if not ini_parser.read(env.supervisor_conf): + raise Exception("Could not parse INI file {0}".format(env.supervisor_conf)) + for section, option in (('supervisord','logfile'), + ('supervisord','pidfile'), + ('unix_http_server','file'), + ('program:celeryd','stdout_logfile')): + filepath = ini_parser.get(section, option) + # generate file's directory structure if needed + run('mkdir -p {0}'.format(os.path.split(filepath)[0])) + # touch a file and change ownership if needed + if 'log' in option and not files.exists(filepath): + sudo('touch {0}'.format(filepath)) + sudo('chown {0}:{1} {2}'.format(env.local_user, env.group, filepath)) -def clone(): +@task +def check_secrets(): """ - Clones the project from the central repository + Ensure secret files exist for syncdb to run. """ - run('git clone %s %s' % (env.proj_repo, env.proj_dir)) + secrets_path = env.code_root + '/karmaworld/secret' + secrets_files = ('filepicker.py', 'static_s3.py', 'db_settings.py', 'drive.py', 'client_secrets.json', 'drive.p12') -def update_code(): + errors = [] + for sfile in secrets_files: + ffile = os.path.sep.join((secrets_path,sfile)) + if not files.exists(ffile): + errors.append('{0} missing. Please add and try again.'.format(ffile)) + if errors: + raise Exception('\n'.join(errors)) + +@task +def fetch_usde(): """ - Pulls the latest changes from the central repository + Download USDE accreditation school CSV. """ - with cd(env.proj_dir): - run('git pull') + virtenv_exec('{0}/manage.py fetch_usde_csv {1}'.format(env.code_root, env.usde_csv)) +@task +def import_usde(): + """ + Import accreditation school CSV into the database and scrub it. + """ + virtenv_exec('{0}/manage.py import_usde_csv {1}'.format(env.code_root, env.usde_csv)) + virtenv_exec('{0}/manage.py sanitize_usde_schools'.format(env.code_root)) -def deploy(): +@task +def install_pdf2htmlEX(): + """ + # Some things we need: + sudo apt-get install cmake libpng-dev libjpeg-dev libgtk2.0-dev pkg-config libfontconfig1-dev autoconf libtool + + # Ubuntu 12.04 comes with a version of poppler that is too + # old, so compile our own + wget http://poppler.freedesktop.org/poppler-0.24.4.tar.xz + tar xf poppler-0.24.4.tar.gz + ./configure --prefix=/usr --enable-xpdf-headers + make + sudo make install + + # Ubuntu 12.04 comes with a version of fontforge that is too + # old, so compile our own + git clone https://github.com/fontforge/fontforge.git + ./autogen.sh + ./configure --prefix=/usr + make + sudo make install + + # Compile pdf2htmlEX + wget https://github.com/coolwanglu/pdf2htmlEX/archive/v0.10.tar.gz + tar xf x0.10.tar.gz + cd pdf2htmlEX + cmake . + make + sudo make install + """ + print "not implemented yet!" + +@task +def first_deploy(): """ - Creates or updates the project, runs migrations, installs dependencies. + Sets up and deploys the project for the first time. """ - first_deploy = False - with settings(warn_only=True): - if run('test -d %s' % env.proj_dir).failed: - # first_deploy var is for initial deploy information - first_deploy = True - clone() - if run('test -d $WORKON_HOME/%s' % env.virtualenv).failed: - make_virtualenv() + link_code() + make_virtualenv() + file_setup() + check_secrets() + update_reqs() + syncdb() + collect_static() + fetch_usde() + import_usde() + start_supervisord() + +@task +def deploy(): + """ + Deploys the latest changes + """ update_code() update_reqs() syncdb() - #TODO: run gunicorn - #restart_uwsgi() + collect_static() + restart_supervisord() ########## END COMMANDS