From eb06dd4d68f181508b164b05b1cd9981a1d21329 Mon Sep 17 00:00:00 2001 From: Bryan Date: Sat, 28 Dec 2013 00:58:41 -0500 Subject: [PATCH] moving some docs into README and others under docs/, removing redundant or out of date docs for #179. --- README.md | 42 +++++- docs/Makefile | 153 -------------------- docs/{source => }/dbimport.rst | 0 docs/{source => }/secrets.rst | 0 docs/source/TODO.rst | 14 -- docs/source/database.rst | 22 --- docs/source/deployment.rst | 5 - docs/source/external-css-and-js.rst | 11 -- docs/source/gdrive.rst | 50 ------- docs/source/layout.rst | 90 ------------ docs/source/os-install.rst | 207 ---------------------------- docs/source/virtualenv.rst | 20 --- 12 files changed, 39 insertions(+), 575 deletions(-) delete mode 100644 docs/Makefile rename docs/{source => }/dbimport.rst (100%) rename docs/{source => }/secrets.rst (100%) delete mode 100644 docs/source/TODO.rst delete mode 100644 docs/source/database.rst delete mode 100644 docs/source/deployment.rst delete mode 100644 docs/source/external-css-and-js.rst delete mode 100644 docs/source/gdrive.rst delete mode 100644 docs/source/layout.rst delete mode 100644 docs/source/os-install.rst delete mode 100644 docs/source/virtualenv.rst diff --git a/README.md b/README.md index 7e6dd62..c13c9cc 100644 --- a/README.md +++ b/README.md @@ -179,12 +179,19 @@ not generally be needed. * `nginx` * `libxslt` and `libxml2` (used by some Python libraries) * `RabbitMQ` (server) - * `memcached` (might not be used) + * `memcached` * `Python` * `PIP` * `virtualenv` * `virtualenvwrapper` (might not be needed anymore) + On a Debian system supporting Apt, this can be done with: + + sudo apt-get install python-pip postgresql python-virtualenv \ + virtualenvwrapper git nginx p7zip-full \ + postgresql-server-dev-9.1 libxslt1-dev libxml2-dev \ + libmemcached-dev python-dev rabbitmq-server + 1. Generate a PostgreSQL database and a role with read/write permissions. * For Debian, these instructions are helpful: https://wiki.debian.org/PostgreSql @@ -304,8 +311,37 @@ Please see [vagrant documentation](http://docs.vagrantup.com/v2/cli/index.html) for more information on how to use the vagrant CLI to manage your development VM. -Thanks -====== +# Django Database management + +## South + +We have setup Django to use +[south](http://south.aeracode.org/wiki/QuickStartGuide) for migrations. When +changing models, it is important to run +`python {project_root}/manage.py schemamigration` which will create a migration + to reflect the model changes into the database. These changes can be pulled +into the database with `python {project_root}/manage.py migrate`. + +Sometimes the database already has a migration performed on it, but that +information wasn't told to south. There are subtleties to the process which +require looking at the south docs. As a tip, start by looking at the `--fake` +flag. + +# Assets from Third Parties + +A number of assets have been added to the repository which come from external +sources. It would be difficult to keep a complete list in this README and keep +it up to date. Software which originally came from outside parties can +generally be found in `{project_root}/karmaworld/assets`. + +Additionally, all third party Python projects (downloaded and installed with +pip) are listed in these files: + +* `{project_root}/reqs/common.txt` +* `{project_root}/reqs/dev.txt` +* `{project_root}/reqs/prod.txt` + +# Thanks * KarmaNotes.org is a project of the FinalsClub Foundation with generous funding from the William and Flora Hewlett Foundation diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index ffe2347..0000000 --- a/docs/Makefile +++ /dev/null @@ -1,153 +0,0 @@ -# Makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = -BUILDDIR = build - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source - -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext - -help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " texinfo to make Texinfo files" - @echo " info to make Texinfo files and run them through makeinfo" - @echo " gettext to make PO message catalogs" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - -clean: - -rm -rf $(BUILDDIR)/* - -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/django-skel.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/django-skel.qhc" - -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/django-skel" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/django-skel" - @echo "# devhelp" - -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -texinfo: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo - @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." - @echo "Run \`make' in that directory to run these through makeinfo" \ - "(use \`make info' here to do that automatically)." - -info: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo "Running Texinfo files through makeinfo..." - make -C $(BUILDDIR)/texinfo info - @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." - -gettext: - $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale - @echo - @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." diff --git a/docs/source/dbimport.rst b/docs/dbimport.rst similarity index 100% rename from docs/source/dbimport.rst rename to docs/dbimport.rst diff --git a/docs/source/secrets.rst b/docs/secrets.rst similarity index 100% rename from docs/source/secrets.rst rename to docs/secrets.rst diff --git a/docs/source/TODO.rst b/docs/source/TODO.rst deleted file mode 100644 index da4b10d..0000000 --- a/docs/source/TODO.rst +++ /dev/null @@ -1,14 +0,0 @@ -TODO: -================== - - -1. Fix minor issues in documentation ------------------- - - - -2. No-Image.gif issue (resulting from Wordpress issues) ------------------- - -http://stackoverflow.com/questions/92720/jquery-javascript-to-replace-broken-images - diff --git a/docs/source/database.rst b/docs/source/database.rst deleted file mode 100644 index 88188d6..0000000 --- a/docs/source/database.rst +++ /dev/null @@ -1,22 +0,0 @@ -Database -======== - -So you've checked out this git repo, and you've set up your virtualenv. -Next you will want to setup your database. - - -Local development db --------------------- - -We use `south `_ to manage our -database migrations. - -First run a standard django syncdb - - ./manage.py syncdb - -And when asked, create a superuser to log into the django-admin. - -TODO: Do we need to convert_to_south, or do an initial ``schemamigration --fake`` - - diff --git a/docs/source/deployment.rst b/docs/source/deployment.rst deleted file mode 100644 index 218a4dc..0000000 --- a/docs/source/deployment.rst +++ /dev/null @@ -1,5 +0,0 @@ -Deployment -========== - -Required packages: -memcached diff --git a/docs/source/external-css-and-js.rst b/docs/source/external-css-and-js.rst deleted file mode 100644 index 6dd9db6..0000000 --- a/docs/source/external-css-and-js.rst +++ /dev/null @@ -1,11 +0,0 @@ -External CSS and Javascript -=========================== - -We use the following css/javascript frameworks: - -+ foundation.css by zurb -+ responsive tables (for foundation) by zurb - -Imported from cdns: -+ jquery -+ jquery-ui diff --git a/docs/source/gdrive.rst b/docs/source/gdrive.rst deleted file mode 100644 index bdc1540..0000000 --- a/docs/source/gdrive.rst +++ /dev/null @@ -1,50 +0,0 @@ -Google Drive Authentication -=========================== - -Authorizing a new installation of Karmaworld with Google Drive is a convoluted -process. -But it happens infrequently enough that we haven't simplified the process. -This should probably happen in the future, -but in the meantime we can document the process. - -* Start a django interactive shell -* Import the `karmaworld.apps.notes.gdrive` module. -* build a flow object called `flow` -* open a private browsing instance in a browser -* Log into the google drive account of the GOOGLE_USER set in `karmaworld.secrets.drive` -* get the authorization url - - flow.step1_get_authorize_url() - -* go to that url in your browser -* it will redirecto you to a url in the format of: - - http://localhost:8000/oauth2callback?code=EXCHANGE_CODE - -The url of the page you are re-directed to will contain the EXCHANGE_CODE required to complete the authentication - -take the EXCHANGE_CODE from the url, and feed it as a string to - - credentials = flow.step2_exchange('EXCHANGE_CODE') - -This gives you a credentials object, keep this value - -Create an instance of DriveAuth() - - from karmaworld.apps.notes.models import DriveAuth - auth = DriveAuth() - -and feed the credentials object to it with store() - - auth.store(credentials) - -This will save the DriveAuth() - -You are now authenticated! - -If you want to run the document_upload tests, then you need to save the contents of - - DriveAuth.objects.all()[0].credentials - -to secret/oauth_token.json. Obviously, don't commit this. - diff --git a/docs/source/layout.rst b/docs/source/layout.rst deleted file mode 100644 index 2ef88bf..0000000 --- a/docs/source/layout.rst +++ /dev/null @@ -1,90 +0,0 @@ -Layout -====== - -Here is the rough karmaworld repo layout:: - - . - ├── fabfile.py - ├── gunicorn.py.ini - ├── manage.py - ├── Procfile - ├── reqs - │   ├── common.txt - │   ├── dev.txt - │   └── prod.txt - ├── requirements.txt - ├── karmaworld - │   ├── apps - │   │   ├── __init__.py - │   │   ├── courses - │   │   │   ├── models.py - │   │   │   └── … - │   │   ├── notes - │   │   │   ├── models.py - │   │   │   └── … - │   ├── __init__.py - │   ├── libs - │   │   └── __init__.py - │   ├── settings - │   │   ├── common.py - │   │   ├── dev.py - │   │   ├── __init__.py - │   │   └── prod.py - │   ├── templates - │   │   ├── 404.html - │   │   └── 500.html - │   └── urls.py - └── wsgi.py - - -``fabfile.py`` is a utility script (written using `Fabric -`_) that adds some helpful -shortcut commands. It can automatically bootstrap a Heroku app for you, and a -number of other useful things. You can run ``fab --list`` from the command line -to see its usage. - -``gunicorn.py.ini`` is our `gunicorn `_ web server -configuration file. It is optimized for large scale sites, and should work well -in any environment. - -``manage.py`` is our default Django management script. It uses the `dev.py` settings file, so we will have to override this in production - -``Procfile`` is our Heroku process file--which tells Heroku what our three -types of services are: ``web``, ``scheduler``, and ``worker``. To learn more -about this, see `Heroku's Procfile documentation -`_. - -``reqs`` is a directory which contains all of our pip requirement files, broken -into categories by the environment in which they're used. The ``common.txt`` -file contains all of our 'shared' requirements, the ``dev.txt`` file contains -all of our local development requirements, and the ``prod.txt`` file contains -our production requirements. This modular approach is taken to make development -as flexible (and intuitive) as possible. - -``requirements.txt`` is a Heroku specific file which tells Heroku to install -our production requirements *only*. - -``karmaworld`` is the Django project site. Everything inside this directory is -considered the actual Django code. - -``karmaworld/apps`` is the directory meant to hold all of our local Django -applications. - -``karmaworld/libs`` is a directory meant to hold all of your local Django -libraries--code which doesn't really fit into 'applications'. This usually -includes stuff like templatetags that are used in various place, or other -helpful utility functions. - -``karmaworld/settings`` is a directory which holds all of your Django settings files! -Much like our pip requirements, there is a settings file for each environment: -``dev.py``, ``prod.py``, and ``common.py`` (shared settings). - -``karmaworld/templates`` is a directory that holds all your Django templates. - -``karmaworld/urls.py`` is your standard Django urlconf. - -``wsgi.py`` is your standard Django wsgi configuration file. Our webserver -uses this to figure things out :) - - -.. image:: _static/yeah.png diff --git a/docs/source/os-install.rst b/docs/source/os-install.rst deleted file mode 100644 index 3740bf8..0000000 --- a/docs/source/os-install.rst +++ /dev/null @@ -1,207 +0,0 @@ -os-install.rst -============== - -This document defines the deployment of the KarmaNotes / karmaworld platform on Ubuntu server. - -Required packages: - -+ django-1.4.x -+ virtualenv -+ python-pip -+ memcached -+ (See $SRC_ROOT)/reqs/common.py) - -Production Requirements: - -+ rabbitmq-server -+ postgresql-server 9.1.x -+ ($SRC_ROOT)/reqs/prod.py) - -Before we begin, we are going to need a few commonly installed tools: - - sudo apt-get install git-core make gcc libmemcached-dev python-pip libxml2-dev python-dev libxslt-dev - - sudo pip install pip --upgrade # upgrade pip to the latest pip for our version of python - - sudo pip install virtualenv - -If we are in a production environment, we need: - - sudo apt-get install rabbitmq-server postgresql-9.1 postgresql-server-dev-9.1 - - -0. Check out code ------------------ - - git clone https://github.com/finalsclub/karmaworld.git - -Generally, it is advised to have a common $WEB_ROOT. -Ours is in: `/var/www` -So, for our use case, our $SRC_ROOT is: - - /var/www/karmaworld - -Also note that /var/www needs to have proper permissions and creating a separate -user to interact with the app is advised (with basic user permissions). - -1. Environment Setup --------------------- - -First we need to setup our environment to run the app. This includes installing -necessary dependencies, setting proper config files and creating a new databases -(if in production). - -In a production environment we use the following: - -+ ubuntu server 12.04 LTS -+ postgresql-server 9.1.x -+ rabbitmq-server (our broker) -+ python-pip -+ virutalenv -+ django-1.4.3+ -+ libmemcached-dev -+ (see $SRC_ROOT/reqs/prod.txt) - -In all of our deployments, we use virtualenv to provide a clean way of -installing dependencies and providing a solid environment from which the app can -be run from. It is advised that virtualenv be used for all deployments. - - sudo pip install virtualenv - -After installing virtualenv, we need to configure our new environment. Note that -installing packages within the environment does not need superuser permissions. - - cd $SRC_ROOT - - virtualenv beta - - source beta/bin/activate - -a) Development - - pip install -r reqs/dev.txt - -b) Production - - pip install -r reqs/prod.txt - -Once all dependencies have been installed, we need to edit our manage.py file -so that we are reading the proper settings.py file: - -a) Development - - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "karmaworld.settings.dev") - -b) Production - - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "karmaworld.settings.prod") - - -2. Set up database ------------------- - -In both development and production we do set up databases, but dev. uses -SQLite out of the box and requires minimal interaction. If in a production -environment, make sure to follow the instructions in section b first. - -a) Development - - ./manage.py syncdb - - ./manage.py migrate djcelery - -If notes / apps migrations exist, then: - - ./manage.py schemamigration notes --auto - - ./manage.py schemamigration courses --auto - -If they do not exist: - - ./manage.py schemamigration notes --init - - ./manage.py schemamigration courses --init - -b) Production - - sudo apt-get install postgresql-9.1 python-psycopg2 - - sudo passwd postgres - - sudo su postgres - - sudo -u postgres createuser -P djkarma - - psql template1 - - create database karmanotes owner djkarma encoding 'UTF8'; - - #### add this line to your postgres install's /etc/postgresql/9.1/main/pg_hba.conf #### - local karmanotes djkarma md5 - - sudo service postgresql restart - - ./manage.py syncdb - -Then create a file called karmaworld/secret/db_secret.py. Please see 'secrets.rst' in $SRC_ROOT/docs/source/secrets.rst. - -After we have configured postgresql and set our secret db_secret file, we then need to preform -the instructions in the beta section of this document. - -3. Import previous notes (needs more docs) ------------------------------------------- - -Materials from previous instances of karmaworld / djKarma can be imported into a new clean database via. json files. -Karmaworld has facilities built-in so that these json files can easily be imported. - -To get started, we need to get the .json files: - - git clone https://github.com/FinalsClub/notesjson.git - - mv notesjson/* . - -Then we run the imports (in our virtual environment): - - ./manage.py import_json all - - -4. Set up S3 bucket support (optional) --------------------------------------- - -S3 is a storage service that is provided by Amazon. Buckets -are storage lockers where files can be stored and served from. -The reason that we would want to serve files out of said buckets -is so that we can move some traffic from production and provide -a more reliable experience to the user. - - -See $SRC_ROOT/docs/source/secrets.rst - -5. Celeryd setup --------------------------------------- - -At the writing of this documentation, celeryd management -does NOT currently work with our fabric configuration. - -In order to deploy celery, we need to add an init script -to: - - /etc/init.d - -this script can be found in $SRC_ROOT/bin/celeryd . Just make sure -to modify: - - DJANGO_PROJECT_DIR - - VIRTUAL_ENV - -Also note that permissions for /var/run/*.pid and /var/log/*.log need -to be fixed for the user that is running celeryd. - -Run celeryd by: - - sudo service celeryd start - -Stop celeryd by: - - sudo service celeryd stop diff --git a/docs/source/virtualenv.rst b/docs/source/virtualenv.rst deleted file mode 100644 index 2655211..0000000 --- a/docs/source/virtualenv.rst +++ /dev/null @@ -1,20 +0,0 @@ -Virtualenv -========== - -`Virtualenv `_ is a tool for managing python requirements. -We use it to manage development and deployment requirements when we have multiple python projects on the same host. - -This isn't strictly required for the project, but we recommend it. -You can install virtualenv with - - $ pip install virtualenv - -Then create an environment to contain karmaworld packages: - - virtualenv venv - -And then activate the virtual enviroment with: - - source ./venv/bin/activate - -Run this activate script before trying to run ./manage.py commands or other server commands. -- 2.25.1