From bae0359554027c9ba65a3c57ca93a8d7c29d3fc7 Mon Sep 17 00:00:00 2001 From: Charles Connell Date: Fri, 25 Apr 2014 10:41:03 -0400 Subject: [PATCH] Timing instrumentation --- karmaworld/apps/courses/middleware.py | 113 ++++++++++++++++++++++++++ karmaworld/settings/dev.py | 22 +++++ karmaworld/settings/vmdev.py | 23 ++++++ reqs/dev.txt | 1 + reqs/vmdev.txt | 1 + 5 files changed, 160 insertions(+) create mode 100644 karmaworld/apps/courses/middleware.py diff --git a/karmaworld/apps/courses/middleware.py b/karmaworld/apps/courses/middleware.py new file mode 100644 index 0000000..9844066 --- /dev/null +++ b/karmaworld/apps/courses/middleware.py @@ -0,0 +1,113 @@ +# Orignal version taken from http://www.djangosnippets.org/snippets/186/ +# Original author: udfalkso +# Modified by: Shwagroo Team and Gun.io + +import sys +import os +import re +import hotshot, hotshot.stats +import tempfile +import StringIO + +from django.conf import settings + +words_re = re.compile( r'\s+' ) + +group_prefix_re = [ + re.compile( "^.*/django/[^/]+" ), + re.compile( "^(.*)/[^/]+$" ), # extract module path + re.compile( ".*" ), # catch strange entries +] + +class ProfileMiddleware(object): + """ + Displays hotshot profiling for any view. + http://yoursite.com/yourview/?prof + + Add the "prof" key to query string by appending ?prof (or &prof=) + and you'll see the profiling results in your browser. + It's set up to only be available in django's debug mode, is available for superuser otherwise, + but you really shouldn't add this middleware to any production configuration. + + WARNING: It uses hotshot profiler which is not thread safe. + """ + def process_request(self, request): + if (settings.DEBUG or request.user.is_superuser) and 'prof' in request.GET: + self.tmpfile = tempfile.mktemp() + self.prof = hotshot.Profile(self.tmpfile) + + def process_view(self, request, callback, callback_args, callback_kwargs): + if (settings.DEBUG or request.user.is_superuser) and 'prof' in request.GET: + return self.prof.runcall(callback, request, *callback_args, **callback_kwargs) + + def get_group(self, file): + for g in group_prefix_re: + name = g.findall( file ) + if name: + return name[0] + + def get_summary(self, results_dict, sum): + list = [ (item[1], item[0]) for item in results_dict.items() ] + list.sort( reverse = True ) + list = list[:40] + + res = " tottime\n" + for item in list: + res += "%4.1f%% %7.3f %s\n" % ( 100*item[0]/sum if sum else 0, item[0], item[1] ) + + return res + + def summary_for_files(self, stats_str): + stats_str = stats_str.split("\n")[5:] + + mystats = {} + mygroups = {} + + sum = 0 + + for s in stats_str: + fields = words_re.split(s); + if len(fields) == 7: + time = float(fields[2]) + sum += time + file = fields[6].split(":")[0] + + if not file in mystats: + mystats[file] = 0 + mystats[file] += time + + group = self.get_group(file) + if not group in mygroups: + mygroups[ group ] = 0 + mygroups[ group ] += time + + return "
" + \
+               " ---- By file ----\n\n" + self.get_summary(mystats,sum) + "\n" + \
+               " ---- By group ---\n\n" + self.get_summary(mygroups,sum) + \
+               "
" + + def process_response(self, request, response): + if (settings.DEBUG or request.user.is_superuser) and 'prof' in request.GET: + self.prof.close() + + out = StringIO.StringIO() + old_stdout = sys.stdout + sys.stdout = out + + stats = hotshot.stats.load(self.tmpfile) + stats.sort_stats('time', 'calls') + stats.print_stats() + + sys.stdout = old_stdout + stats_str = out.getvalue() + + if response and response.content and stats_str: + response.content = "
" + stats_str + "
" + + response.content = "\n".join(response.content.split("\n")[:40]) + + response.content += self.summary_for_files(stats_str) + + os.unlink(self.tmpfile) + + return response diff --git a/karmaworld/settings/dev.py b/karmaworld/settings/dev.py index 01a9e3f..3829d06 100644 --- a/karmaworld/settings/dev.py +++ b/karmaworld/settings/dev.py @@ -73,10 +73,26 @@ AWS_HEADERS = { # See: https://github.com/django-debug-toolbar/django-debug-toolbar#installation INSTALLED_APPS += ( 'debug_toolbar', + 'template_timings_panel', 'django_extensions', 'django_nose', ) +DEBUG_TOOLBAR_PANELS = ( + 'debug_toolbar.panels.versions.VersionsPanel', + 'debug_toolbar.panels.timer.TimerPanel', + 'debug_toolbar.panels.settings.SettingsPanel', + 'debug_toolbar.panels.headers.HeadersPanel', + 'debug_toolbar.panels.request.RequestPanel', + 'debug_toolbar.panels.sql.SQLPanel', + 'debug_toolbar.panels.staticfiles.StaticFilesPanel', + 'debug_toolbar.panels.templates.TemplatesPanel', + 'template_timings_panel.panels.TemplateTimings.TemplateTimings', + 'debug_toolbar.panels.cache.CachePanel', + 'debug_toolbar.panels.signals.SignalsPanel', + 'debug_toolbar.panels.logging.LoggingPanel', +) + # See: https://github.com/django-debug-toolbar/django-debug-toolbar#installation INTERNAL_IPS = ('127.0.0.1',) @@ -86,6 +102,12 @@ MIDDLEWARE_CLASSES += ( ) ########## END TOOLBAR CONFIGURATION +########## PROFILING CONFIGURATION +MIDDLEWARE_CLASSES += ( + 'karmaworld.apps.courses.middleware.ProfileMiddleware', +) +########## END PROFILING CONFIGURATION + ########## COMPRESSION CONFIGURATION COMPRESS_ENABLED = False ########## END COMPRESSION CONFIGURATION diff --git a/karmaworld/settings/vmdev.py b/karmaworld/settings/vmdev.py index 33b99bd..8524fa2 100644 --- a/karmaworld/settings/vmdev.py +++ b/karmaworld/settings/vmdev.py @@ -194,10 +194,26 @@ TEST_RUNNER = 'django_nose.NoseTestSuiteRunner' # See: https://github.com/django-debug-toolbar/django-debug-toolbar#installation INSTALLED_APPS += ( 'debug_toolbar', + 'template_timings_panel', 'django_extensions', 'django_nose', ) +DEBUG_TOOLBAR_PANELS = ( + 'debug_toolbar.panels.versions.VersionsPanel', + 'debug_toolbar.panels.timer.TimerPanel', + 'debug_toolbar.panels.settings.SettingsPanel', + 'debug_toolbar.panels.headers.HeadersPanel', + 'debug_toolbar.panels.request.RequestPanel', + 'debug_toolbar.panels.sql.SQLPanel', + 'debug_toolbar.panels.staticfiles.StaticFilesPanel', + 'debug_toolbar.panels.templates.TemplatesPanel', + 'template_timings_panel.panels.TemplateTimings.TemplateTimings', + 'debug_toolbar.panels.cache.CachePanel', + 'debug_toolbar.panels.signals.SignalsPanel', + 'debug_toolbar.panels.logging.LoggingPanel', +) + # See: https://github.com/django-debug-toolbar/django-debug-toolbar#installation INTERNAL_IPS = ('127.0.0.1',) @@ -206,3 +222,10 @@ MIDDLEWARE_CLASSES += ( 'debug_toolbar.middleware.DebugToolbarMiddleware', ) ########## END TOOLBAR CONFIGURATION + +########## PROFILING CONFIGURATION +MIDDLEWARE_CLASSES += ( + 'karmaworld.apps.courses.middleware.ProfileMiddleware', +) +########## END PROFILING CONFIGURATION + diff --git a/reqs/dev.txt b/reqs/dev.txt index 17e9725..c3409b3 100644 --- a/reqs/dev.txt +++ b/reqs/dev.txt @@ -1,6 +1,7 @@ -r common.txt django-kombu==0.9.4 django-debug-toolbar +django-debug-toolbar-template-timings ipython==0.13.1 ipdb==0.7 django-extensions==1.0.3 # some extra debugging tools for manage.py diff --git a/reqs/vmdev.txt b/reqs/vmdev.txt index 082bbd4..03bc7bd 100644 --- a/reqs/vmdev.txt +++ b/reqs/vmdev.txt @@ -1,5 +1,6 @@ -r prod.txt django-debug-toolbar +django-debug-toolbar-template-timings ipython==0.13.1 ipdb==0.7 django-extensions==1.0.3 # some extra debugging tools for manage.py -- 2.25.1