Changeset 1605:f2e583f7cab9

Timestamp:

11/04/08 16:04:49 (2 months ago)

Author:

Greg Von Kuster <greg@bx.psu.edu>

branch:

default

Message:

Add ability to track memory usage in code secments - requires config addition:

# Log memory usage
log_memory_usage = False

Added logging statements in tool execution methods and most methods in the user and root controllers.

Also fixes for checking config settings that should be boolean ( e.g., memdump, heartbeat ).

Files:

• lib/galaxy/app.py (modified) (2 diffs)
• lib/galaxy/config.py (modified) (1 diff)
• lib/galaxy/eggs/__init__.py (modified) (2 diffs)
• lib/galaxy/tools/actions/__init__.py (modified) (2 diffs)
• lib/galaxy/tools/actions/upload.py (modified) (2 diffs)
• lib/galaxy/util/memory_usage.py (added)
• lib/galaxy/web/controllers/root.py (modified) (23 diffs)
• lib/galaxy/web/controllers/user.py (modified) (5 diffs)
• universe_wsgi.ini.sample (modified) (1 diff)

lib/galaxy/app.py

@@ -39 +39 @@
-        self.heartbeat = None
-        self.memdump = None
+        self.memory_usage = None
-        # Start the heartbeat process if configured and available
-        if self.config.use_heartbeat:
@@ -50 +51 @@
-            if memdump.Memdump:
-                self.memdump = memdump.Memdump()
+        # Enable memory_usage logging if configured
+        if self.config.log_memory_usage:
+            from galaxy.util import memory_usage
+            self.memory_usage = memory_usage
-    def shutdown( self ):
-        self.job_manager.shutdown()

lib/galaxy/config.py

@@ -56 +56 @@
-        self.pbs_dataset_path = kwargs.get('pbs_dataset_path', "" )
-        self.pbs_stage_path = kwargs.get('pbs_stage_path', "" )
-
-
+
+
+
-        self.ucsc_display_sites = kwargs.get( 'ucsc_display_sites', "main,test,archaea" ).lower().split(",")
-        self.gbrowse_display_sites = kwargs.get( 'gbrowse_display_sites', "wormbase,flybase,elegans" ).lower().split(",")

lib/galaxy/eggs/__init__.py

@@ -485 +485 @@
-        elif egg_name == "threadframe":
-            try:
- == "True"
+
-                    return True
-                else:
@@ -493 +493 @@
-        elif egg_name == "guppy":
-            try:
- == "True"
+
-                    return True
-                else:

lib/galaxy/tools/actions/__init__.py

@@ -66 +66 @@
-    
-    def execute(self, tool, trans, incoming={}, set_output_hid=True ):
+        if trans.app.memory_usage:
+            # Keep track of memory usage
+            m0 = trans.app.memory_usage.memory()
-        out_data = {}
-        # Collect any input datasets from the incoming parameters
@@ -199 +202 @@
-            job.add_output_dataset( name, dataset )
-        trans.app.model.flush()
+        if trans.app.memory_usage:
+            m1 = trans.app.memory_usage.memory( m0, pretty=True )
+            log.info("End of tool %s execution for job id %d, memory used increased by %s"  % ( tool.id, job.id, m1 ) )
-        # Some tools are not really executable, but jobs are still created for them ( for record keeping ).
-        # Examples include tools that redirect to other applications ( epigraph ).  These special tools must

lib/galaxy/tools/actions/upload.py

@@ -16 +16 @@
-    
-    def execute( self, tool, trans, incoming={}, set_output_hid = True ):
+        if trans.app.memory_usage:
+            # Keep track of memory usage
+            m0 = trans.app.memory_usage.memory()
-        data_file = incoming['file_data']
-        file_type = incoming['file_type']
@@ -94 +97 @@
-        log.info( 'job id %d ended ok, file size: %s' % ( job.id, file_size_str ) )
-        trans.log_event( 'job id %d ended ok, file size: %s' % ( job.id, file_size_str ), tool_id=tool.id )
+        if trans.app.memory_usage:
+            m1 = trans.app.memory_usage.memory( m0, pretty=True )
+            log.info("End of tool %s execution for job id %d, memory used increased by %s"  % ( tool.id, job.id, m1 ) )
-        return dict( output=hda )
-

lib/galaxy/web/controllers/root.py

@@ -117 +117 @@
-        Sets the mime-type according to the extension
-        """
+        if trans.app.memory_usage:
+            # Keep track of memory usage
+            m0 = self.app.memory_usage.memory()
-        if hid is not None:
-            try:
@@ -147 +150 @@
-                trans.response.headers["Content-Disposition"] = "attachment; filename=GalaxyHistoryItem-%s-[%s]%s" % (data.hid, fname, toext)
-            trans.log_event( "Display dataset id: %s" % str(id) )
+            if self.app.memory_usage:
+                m1 = trans.app.memory_usage.memory( m0, pretty=True )
+                log.info( "End of root/display, memory used increased by %s"  % m1 )
-            try:
-                return open( data.file_name )
@@ -322 +328 @@
-    def history_delete( self, trans, id=None, **kwd):
-        """Deletes a list of histories, ensures that histories are owned by current user"""
+        if trans.app.memory_usage:
+            # Keep track of memory usage
+            m0 = self.app.memory_usage.memory()
-        history_names = []
-        if id:
@@ -349 +358 @@
-        else:
-            return trans.show_message( "You must select at least one history to delete." )
+        if self.app.memory_usage:
+            m1 = trans.app.memory_usage.memory( m0, pretty=True )
+            log.info( "End of root/history_delete, memory used increased by %s"  % m1 )
-        return trans.show_message( "History deleted: %s" % ",".join(history_names),
-                                           refresh_frames=['history'])
@@ -355 +367 @@
-    def history_undelete( self, trans, id=[], **kwd):
-        """Undeletes a list of histories, ensures that histories are owned by current user"""
+        if trans.app.memory_usage:
+            # Keep track of memory usage
+            m0 = self.app.memory_usage.memory()
-        history_names = []
-        errors = []
@@ -386 +401 @@
-        else:
-            errors.append( "You must select at least one history to undelete." )
+        if self.app.memory_usage:
+            m1 = trans.app.memory_usage.memory( m0, pretty=True )
+            log.info( "End of root/history_undelete, memory used increased by %s"  % m1 )
-        return self.history_available( trans, id=','.join( id ), show_deleted=True, ok_msg = ok_msg, error_msg = "  ".join( errors )  )
-    
@@ -391 +409 @@
-    def clear_history( self, trans ):
-        """Clears the history for a user"""
+        if trans.app.memory_usage:
+            # Keep track of memory usage
+            m0 = self.app.memory_usage.memory()
-        history = trans.get_history()
-        for dataset in history.datasets:
@@ -397 +418 @@
-        self.app.model.flush()
-        trans.log_event( "History id %s cleared" % (str(history.id)) )
+        if self.app.memory_usage:
+            m1 = trans.app.memory_usage.memory( m0, pretty=True )
+            log.info( "End of root/clear_history, memory used increased by %s"  % m1 )
-        trans.response.send_redirect( url_for("/index" ) )
-
@@ -402 +426 @@
-    @web.require_login( "share histories with other users" )
-    def history_share( self, trans, id=None, email="", **kwd ):
+        if trans.app.memory_usage:
+            # Keep track of memory usage
+            m0 = self.app.memory_usage.memory()
-        send_to_err = ""
-        if not id:
@@ -428 +455 @@
-            self.app.model.flush()
-            return trans.show_message( "History (%s) has been shared with: %s" % (",".join(history_names),email) )
+        if self.app.memory_usage:
+            m1 = trans.app.memory_usage.memory( m0, pretty=True )
+            log.info( "End of root/history_share, memory used increased by %s"  % m1 )
-        return trans.fill_template( "/history/share.mako", histories=histories, email=email, send_to_err=send_to_err)
-
@@ -436 +466 @@
-        List all available histories
-        """
+        if trans.app.memory_usage:
+            # Keep track of memory usage
+            m0 = self.app.memory_usage.memory()
-        if as_xml:
-            trans.response.set_content_type('text/xml')
@@ -442 +475 @@
-            id = id.split( "," )
-        trans.log_event( "History id %s available" % str( id ) )
-        
-        history_operations = dict( share=self.history_share, rename=self.history_rename, delete=self.history_delete, undelete=self.history_undelete )
-
+
+
+
-        if do_operation in history_operations:
-            return history_operations[do_operation]( trans, id=id, show_deleted=show_deleted, ok_msg=ok_msg, error_msg=error_msg, **kwd  )
-
-        return trans.fill_template( "/history/list.mako", ids=id,
-                                    user=trans.get_user(),
@@ -458 +491 @@
-    @web.expose
-    def history_import( self, trans, id=None, confirm=False, **kwd ):
+        if trans.app.memory_usage:
+            # Keep track of memory usage
+            m0 = self.app.memory_usage.memory()
-        msg = ""
-        user = trans.get_user()
@@ -498 +534 @@
-            trans.set_history( new_history )
-            trans.log_event( "History imported, id: %s, name: '%s': " % (str(new_history.id) , new_history.name ) )
+            if self.app.memory_usage:
+                m1 = trans.app.memory_usage.memory( m0, pretty=True )
+                log.info( "End of root/history_import, memory used increased by %s"  % m1 )
-            return trans.show_ok_message( """
-                History "%s" has been imported. Click <a href="%s">here</a>
@@ -512 +551 @@
-            return trans.response.send_redirect( web.url_for( action='history_available' ) )
-        else:
+            if trans.app.memory_usage:
+                # Keep track of memory usage
+                m0 = self.app.memory_usage.memory()
-            new_history = trans.app.model.History.get( id )
-            if new_history:
@@ -523 +565 @@
-                trans.set_history( new_history )
-                trans.log_event( "History switched to id: %s, name: '%s'" % (str(new_history.id), new_history.name ) )
+                if self.app.memory_usage:
+                    m1 = trans.app.memory_usage.memory( m0, pretty=True )
+                    log.info( "End of root/history_switch, memory used increased by %s"  % m1 )
-                return trans.show_message( "History switched to: %s" % new_history.name,
-                                           refresh_frames=['history'])
@@ -530 +575 @@
-    @web.expose
-    def history_new( self, trans ):
+        if trans.app.memory_usage:
+            # Keep track of memory usage
+            m0 = self.app.memory_usage.memory()
-        trans.new_history()
-        trans.log_event( "Created new History, id: %s." % str(trans.get_history().id) )
+        if self.app.memory_usage:
+            m1 = trans.app.memory_usage.memory( m0, pretty=True )
+            log.info( "End of root/history_new, memory used increased by %s"  % m1 )
-        return trans.show_message( "New history created", refresh_frames = ['history'] )
-
@@ -537 +588 @@
-    @web.require_login( "renames histories" )
-    def history_rename( self, trans, id=None, name=None, **kwd ):
+        if trans.app.memory_usage:
+            # Keep track of memory usage
+            m0 = self.app.memory_usage.memory()
-        user = trans.get_user()
-
@@ -573 +627 @@
-            else:
-                change_msg = change_msg + "<p>History: "+cur_names[i]+" does not appear to belong to you.</p>"
+        if self.app.memory_usage:
+            m1 = trans.app.memory_usage.memory( m0, pretty=True )
+            log.info( "End of root/history_rename, memory used increased by %s"  % m1 )
-        return trans.show_message( "<p>%s" % change_msg, refresh_frames=['history'] ) 
-
@@ -578 +635 @@
-    def history_add_to( self, trans, history_id=None, file_data=None, name="Data Added to History",info=None,ext="txt",dbkey="?",**kwd ):
-        """Adds a POSTed file to a History"""
+        if trans.app.memory_usage:
+            # Keep track of memory usage
+            m0 = self.app.memory_usage.memory()
-        try:
-            history = trans.app.model.History.get( history_id )
@@ -596 +656 @@
-            data.flush()
-            trans.log_event("Added dataset %d to history %d" %(data.id, trans.history.id))
+            if self.app.memory_usage:
+                m1 = trans.app.memory_usage.memory( m0, pretty=True )
+                log.info( "End of root/history_add_to, memory used increased by %s"  % m1 )
-            return trans.show_ok_message("Dataset "+str(data.hid)+" added to history "+str(history_id)+".")
-        except:
@@ -603 +666 @@
-    def dataset_make_primary( self, trans, id=None):
-        """Copies a dataset and makes primary"""
+        if trans.app.memory_usage:
+            # Keep track of memory usage
+            m0 = self.app.memory_usage.memory()
-        try:
-            old_data = self.app.model.HistoryDatasetAssociation.get( id )
@@ -611 +677 @@
-            history.add_dataset(new_data)
-            new_data.flush()
+            if self.app.memory_usage:
+                m1 = trans.app.memory_usage.memory( m0, pretty=True )
+                log.info( "End of root/dataset_make_primary, memory used increased by %s"  % m1 )
-            return trans.show_message( "<p>Secondary dataset has been made primary.</p>", refresh_frames=['history'] ) 
-        except:

lib/galaxy/web/controllers/user.py

@@ -71 +71 @@
-    @web.expose
-    def login( self, trans, email='', password='' ):
+        if trans.app.memory_usage:
+            # Keep track of memory usage
+            m0 = trans.app.memory_usage.memory()
-        email_error = password_error = None
-        # Attempt login
@@ -86 +89 @@
-                trans.log_event( "User logged in" )
-                return trans.show_ok_message( "Now logged in as " + user.email, refresh_frames=['masthead', 'history'] )
+        if trans.app.memory_usage:
+            m1 = trans.app.memory_usage.memory( m0, pretty=True )
+            log.info( "End of user/login, memory used increased by %s"  % m1 )
-        return trans.show_form( 
-            web.FormBuilder( web.url_for(), "Login", submit_text="Login" )
@@ -93 +99 @@
-    @web.expose
-    def logout( self, trans ):
+        if trans.app.memory_usage:
+            # Keep track of memory usage
+            m0 = trans.app.memory_usage.memory()
-        # Since logging an event requires a session, we'll log prior to ending the session
-        trans.log_event( "User logged out" )
-        trans.handle_user_logout()
+        if trans.app.memory_usage:
+            m1 = trans.app.memory_usage.memory( m0, pretty=True )
+            log.info( "End of user/logout, memory used increased by %s"  % m1 )
-        return trans.show_ok_message( "You are no longer logged in", refresh_frames=['masthead', 'history'] )
-            
-    @web.expose
-    def create( self, trans, email='', password='', confirm='',subscribe=False ):
+        if trans.app.memory_usage:
+            # Keep track of memory usage
+            m0 = trans.app.memory_usage.memory()
-        email_error = password_error = confirm_error = None
-        if email:
@@ -121 +136 @@
-                #subscribe user to email list
-                if subscribe:
-self
-self
+trans
+trans
-                    if mail.close():
-                        return trans.show_warn_message( "Now logged in as " + user.email+". However, subscribing to the mailing list has failed.", refresh_frames=['masthead', 'history'] )
+                if trans.app.memory_usage:
+                    m1 = trans.app.memory_usage.memory( m0, pretty=True )
+                    log.info( "End of user/create, memory used increased by %s"  % m1 )
-                return trans.show_ok_message( "Now logged in as " + user.email, refresh_frames=['masthead', 'history'] )
-        return trans.show_form( 
@@ -146 +164 @@
-                for i in range(15):
-                    new_pass = new_pass + choice(chars)
-self
+trans
-                mail.write("To: %s\nFrom: no-reply@%s\nSubject: Galaxy Password Reset\n\nYour password has been reset to \"%s\" (no quotes)." % (email, trans.request.remote_addr, new_pass) )
-                if mail.close():

universe_wsgi.ini.sample

@@ -16 +16 @@
-paste.app_factory = galaxy.web.buildapp:app_factory
-log_level = DEBUG
+
+# Log memory usage
+log_memory_usage = False
-
-# Should jobs be tracked through the database, rather than in memory