Changeset 511:635e3f5fc1c6

Timestamp:

08/03/07 12:37:44 (1 year ago)

Author:

Dan Blankenberg <dan@bx.psu.edu>

branch:

default

convert_revision:

svn:9bcadc22-80f8-0310-8a53-c8f022958886/galaxy/trunk@1832

Message:

Add the ability for secondary (child) and new primary datasets to be collected automatically. Datasets can now be set to not be visible, which will prevent their display in a user's history.

To have a child dataset harvested automatically, simply name the file:
child_PARENTID_DESIGNATION_VISIBILITY_EXT (To make a new primary dataset, simply use the form: primary_ASSOCIATEDWITHDATASETID_DESIGNATION_VISIBILITY_EXT)
and place this file in the directory specified by $new_file_path

For example:
You define the command in the tool XML as:
<command interpreter="python2.4">some_command.py $input1 $out_file1 $out_file1.id $new_file_path</command>

Suppose the input dataset is 1 and the output dataset is 2, the commandline becomes:
python2.4 some_command.py ./database/files/dataset_1.dat ./database/files/dataset_2.dat 2 ./database/tmp

In addition to the primary file (a HTML file), this program creates files in the ./database/tmp directory named:
child_2_SomeImage_invisible_jpg
child_2_SomeText_visible_text

These files are discovered and added as children to the appropriate dataset. The text file will appear in the user's history, but the jpg will not.

All files can be viewed however, by making links in the primary (HTML) history item like:
<img src="display_child?parent_id=2&designation=SomeImage?" alt="Some Image"/>
<a href="display_child?parent_id=2&designation=SomeText?">Some Text</a>

Designations need to be unique for each originally declared output file, simply using a counter can work well.

These new files can be accessed in the exec_after_process hook under param_dict__collected_datasets__?.

Note the update to the universe_wsgi.ini.sample file.

Some database changes are required:
ALTER TABLE dataset ADD visible boolean;
UPDATE dataset SET visible = true WHERE visible is NULL;

Files:

• lib/galaxy/config.py (modified) (1 diff)
• lib/galaxy/jobs/__init__.py (modified) (1 diff)
• lib/galaxy/model/__init__.py (modified) (3 diffs)
• lib/galaxy/model/mapping.py (modified) (1 diff)
• lib/galaxy/tools/__init__.py (modified) (4 diffs)
• lib/galaxy/web/controllers/root.py (modified) (1 diff)
• templates/history.tmpl (modified) (3 diffs)
• universe_wsgi.ini.sample (modified) (1 diff)

lib/galaxy/config.py

@@ -26 +26 @@
-        self.database_connection =  kwargs.get( "database_connection", False )
-        self.file_path = resolve_path( kwargs.get( "file_path", "database/files" ), self.root )
+        self.new_file_path = resolve_path( kwargs.get( "new_file_path", "database/tmp" ), self.root )
-        self.tool_path = resolve_path( kwargs.get( "tool_path", "tools" ), self.root )
-        self.test_conf = resolve_path( kwargs.get( "test_conf", "" ), self.root )

lib/galaxy/jobs/__init__.py

@@ -287 +287 @@
-        param_dict = dict( [ ( p.name, p.value ) for p in job.parameters ] )
-        param_dict = self.tool.params_from_strings( param_dict, self.app )
+        # Create generated output children and primary datasets and add to param_dict
+        collected_datasets = {'children':self.tool.collect_child_datasets(out_data),'primary':self.tool.collect_primary_datasets(out_data)}
+        param_dict.update({'__collected_datasets__':collected_datasets})
+        # Call 'exec_after_process' hook
-        self.tool.call_hook( 'exec_after_process', self.queue.app, inp_data=inp_data, 
-                             out_data=out_data, param_dict=param_dict, 

lib/galaxy/model/__init__.py

@@ -144 +144 @@
-    def __init__( self, id=None, hid=None, name=None, info=None, blurb=None, peek=None, extension=None, 
-                  dbkey=None, state=None, metadata=None, history=None, parent_id=None, designation=None,
-
+, visible=True
-        self.name = name or "Unnamed dataset"
-        self.id = id
@@ -159 +159 @@
-        self.deleted = False
-        self.purged = False
+        self.visible = visible
-        # Relationships
-        self.history = history
@@ -268 +269 @@
-        #             if designation == data.designation:
-        #                 return data
-
+i
-            if child_association.designation == designation:
-
+_association.child
-        return None
-    def purge( self ):

lib/galaxy/model/mapping.py

@@ -74 +74 @@
-    Column( "deleted", Boolean ),
-    Column( "purged", Boolean ),
+    Column( "visible", Boolean ),
-    ForeignKeyConstraint(['parent_id'],['dataset.id'], ondelete="CASCADE") )
-

lib/galaxy/tools/__init__.py

@@ -7 +7 @@
-pkg_resources.require( "simplejson" )
-
-
+, glob, shutil
-import simplejson
-import sha, hmac, binascii
@@ -783 +783 @@
-                key = "_CHILD___%s___%s" % ( name, child.designation ) 
-                param_dict[ key ] = DatasetFilenameWrapper( child )
-
+
+
+
+
-        return param_dict
-    
@@ -860 +863 @@
-            raise 
-        
+    def collect_child_datasets( self, output):
+        children = {}
+        #Loop through output file names, looking for generated children in form of 'child_parentId_designation_visibility_extension'
+        for name, outdata in output.items():
+            for filename in glob.glob(os.path.join(self.app.config.new_file_path,"child_%i_*" % outdata.id) ):
+                if not name in children:
+                    children[name] = {}
+                fields = os.path.basename(filename).split("_")
+                fields.pop(0)
+                parent_id = int(fields.pop(0))
+                designation = fields.pop(0)
+                visible = fields.pop(0).lower()
+                if visible == "visible": visible = True
+                else: visible = False
+                ext = fields.pop(0).lower()
+                # Create new child dataset
+                child_data = self.app.model.Dataset(extension=ext, parent_id=parent_id, designation=designation, visible=visible, dbkey=outdata.dbkey)
+                child_data.flush()
+                # Move data from temp location to dataset location
+                shutil.move(filename, child_data.file_name)
+                child_data.name = "Secondary Dataset (%s)" % (designation)
+                child_data.state = child_data.states.OK
+                child_data.init_meta()
+                child_data.set_peek()
+                child_data.flush()
+                # Add to child accociation table
+                assoc = self.app.model.DatasetChildAssociation()
+                assoc.child = child_data
+                assoc.designation = child_data.designation
+                outdata.children.append( assoc )
+                # Add child to return dict 
+                children[name][designation] = child_data
+        return children
+        
+    def collect_primary_datasets( self, output):
+        primary_datasets = {}
+        #Loop through output file names, looking for generated primary datasets in form of 'primary_associatedWithDatasetID_designation_visibility_extension'
+        for name, outdata in output.items():
+            for filename in glob.glob(os.path.join(self.app.config.new_file_path,"primary_%i_*" % outdata.id) ):
+                if not name in primary_datasets:
+                    primary_datasets[name] = {}
+                fields = os.path.basename(filename).split("_")
+                fields.pop(0)
+                parent_id = int(fields.pop(0))
+                designation = fields.pop(0)
+                visible = fields.pop(0).lower()
+                if visible == "visible": visible = True
+                else: visible = False
+                ext = fields.pop(0).lower()
+                # Create new primary dataset
+                primary_data = self.app.model.Dataset(extension=ext, designation=designation, visible=visible, dbkey=outdata.dbkey)
+                primary_data.flush()
+                self.app.model.History.get(outdata.history_id).add_dataset(primary_data)
+                # Move data from temp location to dataset location
+                shutil.move(filename, primary_data.file_name)
+                primary_data.name = outdata.name
+                primary_data.info = outdata.info
+                primary_data.state = primary_data.states.OK
+                primary_data.init_meta(copy_from=outdata)
+                primary_data.set_peek()
+                primary_data.flush()
+                # Add dataset to return dict 
+                primary_datasets[name][designation] = primary_data
+        return primary_datasets
+
+        
-# ---- Utility classes to be factored out -----------------------------------
-        
@@ -866 +935 @@
-        self.value = value
-        
+
+class RawObjectWrapper( object ):
+    """
+    Wraps an object so that __str__ returns module_name:class_name.
+    """
+    def __init__( self, obj ):
+        self.obj = obj
+    def __str__( self ):
+        return "%s:%s" % (self.obj.__module__, self.obj.__class__.__name__)
+    def __getattr__( self, key ):
+        return getattr( self.obj, key )
+
-class InputValueWrapper( object ):
-    """

lib/galaxy/web/controllers/root.py

@@ -122 +122 @@
-        else:
-            return "No data with id=%d" % id
+
+    @web.expose
+    def display_child(self, trans, parent_id=None, designation=None, tofile=None, toext=".txt"):
+        """
+        Returns child data directly into the browser, based upon parent_id and designation.
+        """
+        try:
+            data = self.app.model.Dataset.get( parent_id )
+            if data:
+                child = data.get_child_by_designation(designation)
+                if child:
+                    return self.display(trans, id=child.id, tofile=tofile, toext=toext)
+        except Exception:
+            pass
+        return "A child named %s could not be found for data %s" % ( designation, parent_id )
-
-    @web.expose

templates/history.tmpl

@@ -258 +258 @@
-                    #if $data.has_data:
-                        <a href="display?id=$data.id&tofile=yes&toext=$data.ext" target="_blank">save</a>
-   
+    
-                            #set $display_links = $data.datatype.get_display_links($data, $display_app, $app, $request.base)
-                            #if $len($display_links) > 0:
@@ -277 +277 @@
-            ## Child datasets
-            ## 
-                  
+
-            #if $len( $data.children ) > 0:
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-    </div>
-
@@ -300 +307 @@
-    ## Render all active (not deleted) datasets, ordered from newest to oldest
-    #for $data in reversed( $history.active_datasets )
-
+
+
+
-    #end for
-#end if

universe_wsgi.ini.sample

@@ -30 +30 @@
-# Where dataset files are saved
-file_path = database/files
+# Temporary storage for additional datasets, this should be shared through the cluster
+new_file_path = database/tmp
-
-# Tools