Changeset 1172:c591911a6497

Show
Ignore:
Timestamp:
04/11/08 16:11:21 (9 months ago)
Author:
Nate Coraor <nate@bx.psu.edu>
branch:
default
convert_revision:
svn:9bcadc22-80f8-0310-8a53-c8f022958886/galaxy/trunk@2533
Message:

Separate framework and tool environments.

Whether running locally or via PBS, $PATH should now be handled in your
user's environment.

This also fixes problems like PYTHONPATH on the new blast nodes being
wrong (because they are x86_64 and the frontends are i686).

Note that you can actually start Galaxy without having an eggs dir now.

Files:

Legend:

Unmodified
Added
Removed
Modified
Copied
Moved

  • eggs.ini

    r1143 r1172  
    11; 
    2 ; Galaxy egg version, download and build configuration. 
     2; Galaxy egg version configuration 
     3; 
     4; This file is version controlled and should not be edited by hand! 
     5; For more information, see: 
     6;   http://g2.trac.bx.psu.edu/wiki/GalaxyEggs 
    37; 
    48 
    …… 
    610repository = http://eggs.g2.bx.psu.edu 
    711 
    8 ; <python_module> = <version> <download_url> 
    9 [eggs] 
    10 ; platform specific 
    11 bx_python = 0.5.0 http://dist.g2.bx.psu.edu/bx-python_dist-r414.tar.bz2 
    12 Cheetah = 1.0 http://umn.dl.sourceforge.net/sourceforge/cheetahtemplate/Cheetah-1.0.tar.gz 
    13 MySQL_python = 1.2.2 http://superb-west.dl.sourceforge.net/sourceforge/mysql-python/MySQL-python-1.2.2.tar.gz 
    14 pbs_python = 2.9.4 http://ftp.sara.nl/pub/outgoing/pbs_python-2.9.4.tar.gz 
    15 psycopg2 = 2.0.6 http://initd.org/pub/software/psycopg/psycopg2-2.0.6.tar.gz 
    16 pycrypto = 2.0.1 http://www.amk.ca/files/python/crypto/pycrypto-2.0.1.tar.gz 
    17 pysqlite = 2.3.5 http://initd.org/pub/software/pysqlite/releases/2.3/2.3.5/pysqlite-2.3.5.tar.gz 
    18 python_lzo = 1.08 http://www.oberhumer.com/opensource/lzo/download/LZO-v1/python-lzo-1.08.tar.gz 
    19 threadframe = 0.2 http://www.majid.info/python/threadframe/threadframe-0.2.tar.gz 
    20 ; platform inspecific 
    21 Beaker = 0.5 http://cheeseshop.python.org/packages/source/B/Beaker/Beaker-0.5.tar.gz 
    22 docutils = 0.4 http://downloads.sourceforge.net/docutils/docutils-0.4.tar.gz 
    23 elementtree = 1.2.6_20050316 http://effbot.org/downloads/elementtree-1.2.6-20050316.tar.gz 
    24 flup = 0.5 http://www.saddi.com/software/flup/dist/flup-r2311.tar.gz 
    25 lrucache = 0.2 http://evan.prodromou.name/lrucache/lrucache-0.2.tar.gz 
     12[eggs:platform] 
     13bx_python = 0.5.0 
     14Cheetah = 1.0 
     15MySQL_python = 1.2.2 
     16pbs_python = 2.9.4 
     17psycopg2 = 2.0.6 
     18pycrypto = 2.0.1 
     19pysqlite = 2.3.5 
     20python_lzo = 1.08 
     21threadframe = 0.2 
     22 
     23[eggs:noplatform] 
     24Beaker = 0.5 
     25docutils = 0.4 
     26elementtree = 1.2.6_20050316 
     27flup = 0.5 
     28lrucache = 0.2 
    2629;lsprof - james 
    27 Mako = 0.1.10 http://www.makotemplates.org/downloads/Mako-0.1.10.tar.gz 
    28 MyghtyUtils = 0.52 http://cheeseshop.python.org/packages/source/M/MyghtyUtils/MyghtyUtils-0.52.zip 
    29 nose = 0.9.1 http://www.somethingaboutorange.com/mrl/projects/nose/nose-0.9.1.tar.gz 
    30 NoseHTML = 0.2 http://dist.g2.bx.psu.edu/nosehtml_dist-0.2.tar.bz2 
    31 Paste = 1.5.1 http://cheeseshop.python.org/packages/source/P/Paste/Paste-1.5.1.tar.gz 
    32 PasteDeploy = 1.3.1 http://cheeseshop.python.org/packages/source/P/PasteDeploy/PasteDeploy-1.3.1.tar.gz 
    33 PasteScript = 1.3.6 http://cheeseshop.python.org/packages/source/P/PasteScript/PasteScript-1.3.6.tar.gz 
    34 Routes = 1.6.3 http://pypi.python.org/packages/source/R/Routes/Routes-1.6.3.tar.gz 
    35 simplejson = 1.5 http://cheeseshop.python.org/packages/source/s/simplejson/simplejson-1.5.tar.gz 
    36 SQLAlchemy = 0.3.11 http://pypi.python.org/packages/source/S/SQLAlchemy/SQLAlchemy-0.3.11.tar.gz 
    37 Tempita = 0.1 http://pypi.python.org/packages/source/T/Tempita/Tempita-0.1.tar.gz 
    38 twill = 0.8.3 http://darcs.idyll.org/~t/projects/twill-0.8.3.tar.gz 
    39 ;twill = 0.9 http://darcs.idyll.org/~t/projects/twill-0.9.tar.gz 
    40 WebError = 0.8a http://pypi.python.org/packages/source/W/WebError/WebError-0.8a.tar.gz 
    41 WebHelpers = 0.2 http://pypi.python.org/packages/source/W/WebHelpers/WebHelpers-0.2.tar.gz 
    42 WebOb = 0.8.5 http://pypi.python.org/packages/source/W/WebOb/WebOb-0.8.5.tar.gz 
    43 wsgiref = 0.1.2 http://pypi.python.org/packages/source/w/wsgiref/wsgiref-0.1.2.zip 
     30Mako = 0.1.10 
     31MyghtyUtils = 0.52 
     32nose = 0.9.1 
     33NoseHTML = 0.2 
     34Paste = 1.5.1 
     35PasteDeploy = 1.3.1 
     36PasteScript = 1.3.6 
     37Routes = 1.6.3 
     38simplejson = 1.5 
     39SQLAlchemy = 0.3.11 
     40Tempita = 0.1 
     41twill = 0.8.3 
     42WebError = 0.8a 
     43WebHelpers = 0.2 
     44WebOb = 0.8.5 
     45wsgiref = 0.1.2 
    4446 
    4547; extra version information 
    …… 
    5456nose = .dev_r101 
    5557 
    56 ; When using scramble.py, you may force scramble to build eggs for one of the 
    57 ; following platforms by providing the platform as the second command line 
    58 ; argument.  Scramble will then ssh to the host defined and use the python 
    59 ; interpreter defined to build the egg.  This is mainly intended for Galaxy 
    60 ; developers at Penn State who maintain the pre-built egg repository.  To build 
    61 ; eggs for your own platform, just leave off the platform argument to 
    62 ; scramble.py. 
    63 [hosts] 
    64 py2.4-linux-i686-ucs2 = scofield.bx.psu.edu /depot/projects/pythons/linux-i686-ucs2/bin/python2.4 
    65 py2.4-linux-i686-ucs4 = scofield.bx.psu.edu /depot/projects/pythons/linux-i686-ucs4/bin/python2.4 
    66 py2.5-linux-i686-ucs2 = scofield.bx.psu.edu /depot/projects/pythons/linux-i686-ucs2/bin/python2.5 
    67 py2.5-linux-i686-ucs4 = scofield.bx.psu.edu /depot/projects/pythons/linux-i686-ucs4/bin/python2.5 
    68 py2.4-linux-x86_64-ucs2 = herbie.bx.psu.edu /depot/projects/pythons/linux-x86_64-ucs2/bin/python2.4 
    69 py2.4-linux-x86_64-ucs4 = herbie.bx.psu.edu /depot/projects/pythons/linux-x86_64-ucs4/bin/python2.4 
    70 py2.5-linux-x86_64-ucs2 = herbie.bx.psu.edu /depot/projects/pythons/linux-x86_64-ucs2/bin/python2.5 
    71 py2.5-linux-x86_64-ucs4 = herbie.bx.psu.edu /depot/projects/pythons/linux-x86_64-ucs4/bin/python2.5 
    72 py2.4-macosx-10.3-fat-ucs2 = medeski.bx.psu.edu /usr/local/bin/python2.4 
    73 py2.5-macosx-10.3-fat-ucs2 = medeski.bx.psu.edu /usr/local/bin/python2.5 
    74 py2.4-solaris-2.11-i86pc-ucs2 = victory.bx.psu.edu /depot/projects/pythons/solaris-2.11-i86pc-ucs2/bin/python2.4 
    75 py2.5-solaris-2.11-i86pc-ucs2 = victory.bx.psu.edu /depot/projects/pythons/solaris-2.11-i86pc-ucs2/bin/python2.5 
    76 py2.4-solaris-2.10-sun4u-ucs2 = v880.bx.psu.edu /depot/projects/pythons/solaris-2.10-sun4u-ucs2/bin/python2.4 
    77 py2.5-solaris-2.10-sun4u-ucs2 = v880.bx.psu.edu /depot/projects/pythons/solaris-2.10-sun4u-ucs2/bin/python2.5 
    78  
    79 ; Instead of providing a platform name as the third argument, you may also 
    80 ; provide one of these platform group names.  Platforms on the right side of the 
    81 ; assignment in [groups] must exist on the left side of an assignment in the 
    82 ; [hosts] section.  Penn State Galaxy developers building eggs for distribution 
    83 ; should use 'all' for platform-specific eggs and 'all-noplatform' for 
    84 ; platform-inspecific eggs. 
    85 [groups] 
    86 py2.4-linux = py2.4-linux-i686-ucs2 py2.4-linux-i686-ucs4 py2.4-linux-x86_64-ucs2 py2.4-linux-x86_64-ucs4 
    87 py2.5-linux = py2.5-linux-i686-ucs2 py2.5-linux-i686-ucs4 py2.5-linux-x86_64-ucs2 py2.5-linux-x86_64-ucs4 
    88 linux = py2.4-linux-i686-ucs2 py2.4-linux-i686-ucs4 py2.4-linux-x86_64-ucs2 py2.4-linux-x86_64-ucs4 py2.5-linux-i686-ucs2 py2.5-linux-i686-ucs4 py2.5-linux-x86_64-ucs2 py2.5-linux-x86_64-ucs4 
    89 py2.4-macosx = py2.4-macosx-10.3-fat-ucs2 
    90 py2.5-macosx = py2.5-macosx-10.3-fat-ucs2 
    91 macosx = py2.4-macosx-10.3-fat-ucs2 py2.5-macosx-10.3-fat-ucs2 
    92 py2.4-solaris = py2.4-solaris-2.11-i86pc-ucs2 py2.4-solaris-2.10-sun4u-ucs2 
    93 py2.5-solaris = py2.5-solaris-2.11-i86pc-ucs2 py2.5-solaris-2.10-sun4u-ucs2 
    94 solaris = py2.4-solaris-2.11-i86pc-ucs2 py2.4-solaris-2.10-sun4u-ucs2 py2.5-solaris-2.11-i86pc-ucs2 py2.5-solaris-2.10-sun4u-ucs2 
    95 py2.4-all = py2.4-linux-i686-ucs2 py2.4-linux-i686-ucs4 py2.4-linux-x86_64-ucs2 py2.4-linux-x86_64-ucs4 py2.4-macosx-10.3-fat-ucs2 py2.4-solaris-2.11-i86pc-ucs2 py2.4-solaris-2.10-sun4u-ucs2 
    96 py2.5-all = py2.5-linux-i686-ucs2 py2.5-linux-i686-ucs4 py2.5-linux-x86_64-ucs2 py2.5-linux-x86_64-ucs4 py2.5-macosx-10.3-fat-ucs2 py2.5-solaris-2.11-i86pc-ucs2 py2.5-solaris-2.10-sun4u-ucs2 
    97 all = py2.4-linux-i686-ucs2 py2.4-linux-i686-ucs4 py2.4-linux-x86_64-ucs2 py2.4-linux-x86_64-ucs4 py2.5-linux-i686-ucs2 py2.5-linux-i686-ucs4 py2.5-linux-x86_64-ucs2 py2.5-linux-x86_64-ucs4 py2.4-macosx-10.3-fat-ucs2 py2.5-macosx-10.3-fat-ucs2 py2.4-solaris-2.11-i86pc-ucs2 py2.4-solaris-2.10-sun4u-ucs2 py2.5-solaris-2.11-i86pc-ucs2 py2.5-solaris-2.10-sun4u-ucs2 
    98 all-noplatform = py2.4-solaris-2.11-i86pc-ucs2 py2.5-solaris-2.11-i86pc-ucs2 
     58; source location, necessary for scrambling 
     59[source] 
     60bx_python = http://dist.g2.bx.psu.edu/bx-python_dist-r414.tar.bz2 
     61Cheetah = http://umn.dl.sourceforge.net/sourceforge/cheetahtemplate/Cheetah-1.0.tar.gz 
     62MySQL_python = http://superb-west.dl.sourceforge.net/sourceforge/mysql-python/MySQL-python-1.2.2.tar.gz http://mysql.mirrors.pair.com/Downloads/MySQL-5.0/mysql-5.0.51a.tar.gz 
     63pbs_python = http://ftp.sara.nl/pub/outgoing/pbs_python-2.9.4.tar.gz http://www.clusterresources.com/downloads/torque/torque-%s.tar.gz 
     64psycopg2 = http://initd.org/pub/software/psycopg/psycopg2-2.0.6.tar.gz http://ftp8.us.postgresql.org/postgresql/source/v8.2.6/postgresql-8.2.6.tar.bz2 
     65pycrypto = http://www.amk.ca/files/python/crypto/pycrypto-2.0.1.tar.gz 
     66pysqlite = http://initd.org/pub/software/pysqlite/releases/2.3/2.3.5/pysqlite-2.3.5.tar.gz http://www.sqlite.org/sqlite-source-3_5_4.zip 
     67python_lzo = http://www.oberhumer.com/opensource/lzo/download/LZO-v1/python-lzo-1.08.tar.gz http://www.oberhumer.com/opensource/lzo/download/LZO-v1/lzo-1.08.tar.gz 
     68threadframe = http://www.majid.info/python/threadframe/threadframe-0.2.tar.gz 
     69Beaker = http://cheeseshop.python.org/packages/source/B/Beaker/Beaker-0.5.tar.gz 
     70docutils = http://downloads.sourceforge.net/docutils/docutils-0.4.tar.gz 
     71elementtree = http://effbot.org/downloads/elementtree-1.2.6-20050316.tar.gz 
     72flup = http://www.saddi.com/software/flup/dist/flup-r2311.tar.gz 
     73lrucache = http://evan.prodromou.name/lrucache/lrucache-0.2.tar.gz 
     74Mako = http://www.makotemplates.org/downloads/Mako-0.1.10.tar.gz 
     75MyghtyUtils = http://cheeseshop.python.org/packages/source/M/MyghtyUtils/MyghtyUtils-0.52.zip 
     76nose = http://www.somethingaboutorange.com/mrl/projects/nose/nose-0.9.1.tar.gz 
     77NoseHTML = http://dist.g2.bx.psu.edu/nosehtml-0.2.tar.bz2 
     78Paste = http://cheeseshop.python.org/packages/source/P/Paste/Paste-1.5.1.tar.gz 
     79PasteDeploy = http://cheeseshop.python.org/packages/source/P/PasteDeploy/PasteDeploy-1.3.1.tar.gz 
     80PasteScript = http://cheeseshop.python.org/packages/source/P/PasteScript/PasteScript-1.3.6.tar.gz 
     81Routes = http://pypi.python.org/packages/source/R/Routes/Routes-1.6.3.tar.gz 
     82simplejson = http://cheeseshop.python.org/packages/source/s/simplejson/simplejson-1.5.tar.gz 
     83SQLAlchemy = http://pypi.python.org/packages/source/S/SQLAlchemy/SQLAlchemy-0.3.11.tar.gz 
     84Tempita = http://pypi.python.org/packages/source/T/Tempita/Tempita-0.1.tar.gz 
     85twill = http://darcs.idyll.org/~t/projects/twill-0.8.3.tar.gz 
     86WebError = http://pypi.python.org/packages/source/W/WebError/WebError-0.8a.tar.gz 
     87WebHelpers = http://pypi.python.org/packages/source/W/WebHelpers/WebHelpers-0.2.tar.gz 
     88WebOb = http://pypi.python.org/packages/source/W/WebOb/WebOb-0.8.5.tar.gz 
     89wsgiref = http://pypi.python.org/packages/source/w/wsgiref/wsgiref-0.1.2.zip 

  • lib/galaxy/eggs/__init__.py

    r1171 r1172  
    44 
    55import os, sys, shutil, tarfile, zipfile, subprocess, ConfigParser, glob, urllib2 
     6from types import ModuleType 
    67 
    78import logging 
    …… 
    1314# within tools.  i don't know of any way around this. -ndc 
    1415galaxy_dir = os.path.abspath( os.path.join( os.path.dirname( __file__ ), "..", "..", ".." ) ) 
     16 
     17class NewEgg( Exception ): 
     18    pass 
    1519 
    1620class EggNotFetchable( Exception ): 
    …… 
    407411            pkg_resources.working_set.require( "%s==%s" % ( name, egg.get_vertag() ) ) 
    408412        return 
     413    except UserWarning, e: 
     414        sys.exit( 1 ) 
    409415    except pkg_resources.VersionConflict, e: 
    410416        # there's a conflicting egg on the pythonpath, remove it 
    …… 
    420426            if not egg.fetch(): 
    421427                raise EggNotFetchable( egg.name ) 
     428            pkg_resources.working_set.require( "%s==%s" % ( name, egg.get_vertag() ) ) 
     429            if dist.project_name in sys.modules: 
     430                try: 
     431                    mod = sys.modules[dist.project_name] 
     432                    reload( mod ) 
     433                except: 
     434                    raise NewEgg( "Galaxy downloaded a new egg (%s) but was unable to reload the module it contained.  Please try starting Galaxy again." % egg.name ) 
    422435    except pkg_resources.DistributionNotFound, e: 
    423436        # the initial require itself is the first dep, but it can have 
    424437        # multiple deps, which will be fetched by the require below. 
     438        print >>sys.stdout, "foo" 
    425439        dep = pkg_resources.Requirement.parse( str( e ) ).project_name 
    426440        egg = c.get_for_require( dep ) 
    …… 
    432446            if not egg.fetch(): 
    433447                raise EggNotFetchable( egg.name ) 
    434     require( pkg ) 
     448            pkg_resources.working_set.require( "%s==%s" % ( name, egg.get_vertag() ) ) 
     449    #require( pkg ) 
    435450 
    436451# convenience stuff 

  • lib/galaxy/jobs/__init__.py

    r1161 r1172  
    265265        self.working_directory = None 
    266266        self.command_line = None 
     267        self.galaxy_lib_dir = None 
    267268         
    268269    def get_param_dict( self ): 
    …… 
    319320        # Build the job's command line 
    320321        self.command_line = self.tool.build_command_line( param_dict ) 
     322        # FIXME: for now, tools get Galaxy's lib dir in their path 
     323        if self.command_line.startswith( 'python' ): 
     324            self.galaxy_lib_dir = os.path.abspath( "lib" ) # cwd = galaxy root 
    321325        # command_line won't actually be set in the db until finish unless you do it here 
    322326        # We need it in the db to be able to restart jobs -ndc 

  • lib/galaxy/jobs/runners/local.py

    r1160 r1172  
    5454        # If we were able to get a command line, run the job 
    5555        if command_line: 
     56            if job_wrapper.galaxy_lib_dir is not None: 
     57                if 'PYTHONPATH' in os.environ: 
     58                    env = { 'PYTHONPATH' : "%s:%s" % ( os.environ['PYTHONPATH'], job_wrapper.galaxy_lib_dir ) }  
     59                else: 
     60                    env = { 'PYTHONPATH' : job_wrapper.galaxy_lib_dir } 
     61            else: 
     62                env = {} 
    5663            try: 
    5764                log.debug( 'executing: %s' % command_line ) 
    …… 
    6168                                         stdout = subprocess.PIPE,  
    6269                                         stderr = subprocess.PIPE, 
     70                                         env = env, 
    6371                                         preexec_fn = os.setpgrp ) 
    6472                job_wrapper.set_runner( 'local:///', proc.pid ) 

  • lib/galaxy/jobs/runners/pbs.py

    r1160 r1172  
    77 
    88from galaxy import model 
     9from paste.deploy.converters import asbool 
    910 
    1011import pkg_resources 
    …… 
    1920 
    2021pbs_template = """#!/bin/sh 
    21 export LC_ALL='%s' 
    22 export PATH='%s' 
    23 export PYTHONPATH='%s' 
     22GALAXY_LIB="%s" 
     23if [ "$GALAXY_LIB" != "None" ]; then 
     24    if [ -n "$PYTHONPATH" ]; then 
     25        export PYTHONPATH="$GALAXY_LIB:$PYTHONPATH" 
     26    else 
     27        export PYTHONPATH="$GALAXY_LIB" 
     28    fi 
     29fi 
    2430cd %s 
    2531%s 
    …… 
    2733 
    2834pbs_symlink_template = """#!/bin/sh 
    29 export LC_ALL='%s' 
    30 export PATH='%s' 
    31 export PYTHONPATH='%s' 
     35GALAXY_LIB="%s" 
     36if [ "$GALAXY_LIB" != "None" ]; then 
     37    if [ -n "$PYTHONPATH" ]; then 
     38        export PYTHONPATH="$GALAXY_LIB:$PYTHONPATH" 
     39    else 
     40        export PYTHONPATH="$GALAXY_LIB" 
     41    fi 
     42fi 
    3243for dataset in %s; do 
    3344    dir=`dirname $dataset` 
    …… 
    169180        # write the job script 
    170181        if self.app.config.pbs_stage_path != '': 
    171             script = pbs_symlink_template % (os.environ['LC_ALL'], os.environ['NODEPATH'], os.environ['PYTHONPATH'], " ".join(job_wrapper.get_input_fnames() + job_wrapper.get_output_fnames()), self.app.config.pbs_stage_path, exec_dir, command_line) 
    172         else: 
    173             script = pbs_template % (os.environ['LC_ALL'], os.environ['NODEPATH'], os.environ['PYTHONPATH'], exec_dir, command_line) 
     182            script = pbs_symlink_template % (job_wrapper.galaxy_lib_dir, " ".join(job_wrapper.get_input_fnames() + job_wrapper.get_output_fnames()), self.app.config.pbs_stage_path, exec_dir, command_line) 
     183        else: 
     184            script = pbs_template % (job_wrapper.galaxy_lib_dir, exec_dir, command_line) 
    174185        job_file = "%s/database/pbs/%s.sh" % (os.getcwd(), job_wrapper.job_id) 
    175186        fh = file(job_file, "w") 
    …… 
    348359 
    349360        # clean up the job_file, ofile, efile 
    350         if os.access(ofile, os.R_OK): 
    351             os.unlink(ofile) 
    352         if os.access(efile, os.R_OK): 
    353             os.unlink(efile) 
    354         if os.access(job_file, os.R_OK): 
    355             os.unlink(job_file) 
     361        if not asbool( self.app.config.get( 'debug', False ) ): 
     362            if os.access(ofile, os.R_OK): 
     363                os.unlink(ofile) 
     364            if os.access(efile, os.R_OK): 
     365                os.unlink(efile) 
     366            if os.access(job_file, os.R_OK): 
     367                os.unlink(job_file) 
    356368             
    357369    def put( self, job_wrapper ): 

  • run.sh

    r1125 r1172  
    11#!/bin/sh 
    22 
    3 . ./scripts/get_python.sh 
    4 . ./setup_paths.sh 
    5  
    6 # Create directories 
    7 CREATE_DIRS=" 
    8 $UNIVERSE_HOME/database/$CREATE_DIR/files 
    9 $UNIVERSE_HOME/database/$CREATE_DIR/tmp 
    10 " 
    11  
    12 for CREATE_DIR in $CREATE_DIRS; do 
    13     if [ ! -d $CREATE_DIR -o ! -h $CREATE_DIR ]; then 
    14         mkdir -p $CREATE_DIR 
    15     fi 
    16 done 
    17  
    18 $GALAXY_PYTHON ./scripts/paster.py serve universe_wsgi.ini $@ 
     3cd `dirname $0` 
     4python -ES ./scripts/paster.py serve universe_wsgi.ini $@ 

  • run_functional_tests.sh

    r717 r1172  
    66 
    77if [ ! $1 ]; then 
    8         ./nosetests.sh -v -w test --with-nosehtml --html-report-file run_functional_tests.html --exclude="^get" functional 
     8        python -ES ./scripts/nosetests.py -v -w test --with-nosehtml --html-report-file run_functional_tests.html --exclude="^get" functional 
    99elif [ $1 = 'help' ]; then 
    1010        echo "'run_functional_tests.sh'                       for testing all the tools in functional directory" 
    …… 
    1414        echo "'run_functional_tests.sh -list'                 for listing all the tool ids" 
    1515elif [ $1 = '-id' ]; then 
    16         ./nosetests.sh -v -w test functional.test_toolbox:TestForTool_$2 --with-nosehtml --html-report-file run_functional_tests.html 
     16        python -ES ./scripts/nosetests.py -v -w test functional.test_toolbox:TestForTool_$2 --with-nosehtml --html-report-file run_functional_tests.html 
    1717elif [ $1 = '-sid' ]; then 
    18         ./nosetests.sh --with-nosehtml --html-report-file run_functional_tests.html -v -w test `python tool_list.py $2` 
     18        python -ES ./scripts/nosetests.py --with-nosehtml --html-report-file run_functional_tests.html -v -w test `python tool_list.py $2` 
    1919elif [ $1 = '-list' ]; then 
    2020        python tool_list.py 
    …… 
    2323        echo "'run_functional_tests.sh -sid ccc'              for testing one section with sid 'ccc' ('ccc' is the string after 'section::')" 
    2424else 
    25         ./nosetests.sh -v -w test --with-nosehtml --html-report-file run_functional_tests.html $1 
     25        python -ES ./scripts/nosetests.py -v -w test --with-nosehtml --html-report-file run_functional_tests.html $1 
    2626fi 
    2727 

  • scripts/check_eggs.py

    r1125 r1172  
    1 import os, sys, pickle, subprocess 
    2 from eggs import * 
     1""" 
     2usage: check_eggs.py 
     3""" 
     4import os, sys, logging 
    35 
    4 config_eggs = ConfigEggs() 
    5 galaxy_eggs = GalaxyEggs() 
     6root = logging.getLogger() 
     7root.setLevel( 10 ) 
     8root.addHandler( logging.StreamHandler( sys.stdout ) ) 
     9 
     10lib = os.path.abspath( os.path.join( os.path.dirname( __file__ ), "..", "lib" ) ) 
     11sys.path.append( lib ) 
     12 
     13from galaxy.eggs import * 
     14 
     15c = Crate() 
     16c.parse() 
    617galaxy_config = GalaxyConfig() 
    7 pickle_name = ".eggs.pickle" 
    8  
    9 if os.environ.has_key( "GALAXY_IGNORE_EGGS" ): 
    10     ignore = int( os.environ[ "GALAXY_IGNORE_EGGS" ] ) 
    11 else: 
    12     ignore = False 
    13  
    14 print "Checking Galaxy eggs" 
    15 missing_eggs = get_missing_eggs( config_eggs, galaxy_eggs, galaxy_config ) 
    16 if len( missing_eggs ) != 0: 
    17     if ignore: 
    18         # create the pickle if necessary 
    19         if not os.access( pickle_name, os.F_OK ): 
    20             dict = {} 
    21             f = open( pickle_name, "w" ) 
    22             pickle.dump( dict, f ) 
    23             f.close() 
    24         # check the last run rev for this arch 
    25         f = open( pickle_name, "r" ) 
    26         last_revs = pickle.load( f ) 
    27         f.close() 
    28         entries = os.path.abspath( os.path.join( os.path.dirname( sys.argv[0] ), "..", ".svn", "entries" ) ) 
    29         f = open( entries, "r" ) 
    30         for i in range(0, 4): 
    31             this_rev = f.readline().strip() 
    32         f.close() 
    33         plat = get_full_platform() 
    34         if last_revs.has_key( plat ): 
    35             last_rev = last_revs[plat] 
    36         else: 
    37             last_rev = None 
    38         if this_rev == last_rev: 
    39             print "" 
    40             print "WARNING: Some eggs are out of date or missing, but" 
    41             print "GALAXY_IGNORE_EGGS is set, so Galaxy will start anyway." 
    42             print "" 
    43             sys.exit( 0 ) 
    44         else: 
    45             print "" 
    46             print "ERROR:" 
    47             print "" 
    48             print "The environment variable GALAXY_IGNORE_EGGS has been set to" 
    49             print "force Galaxy to start, regardless of the fact that some required" 
    50             print "eggs are missing or outdated.  However, this revision of Galaxy has" 
    51             print "changed since the last time you ran it on this platform (probably" 
    52             print "due to an 'svn update').  Please be aware that running a new" 
    53             print "revision on an outdated module can cause unknown problems (if it" 
    54             print "works at all)." 
    55             print "" 
    56             print "Galaxy startup will terminate now, but if you execute run.sh" 
    57             print "again, startup will proceed normally." 
    58             print "" 
    59             last_revs[plat] = this_rev 
    60             f = open( pickle_name, "w" ) 
    61             pickle.dump( last_revs, f ) 
    62             f.close() 
    63             sys.exit( 1 ) 
    64     else: 
    65         print "" 
    66         print "ERROR:" 
    67         print "" 
    68         print "Your Galaxy eggs are missing or out of date.  Please run:" 
    69         print "" 
    70         print " ", sys.executable, os.path.join( os.path.dirname( sys.argv[0] ), "fetch_eggs.py" ) 
    71         print "" 
    72         print "to download the correct eggs for this version of Galaxy." 
    73         print "" 
    74         sys.exit( 1 ) 
    75  
    76 # if we made it this far, we're good. 
    77 print "All eggs are up to date for this revision of Galaxy" 
    78 sys.exit( 0 ) 
     18ignore = [] 
     19for name in c.get_names(): 
     20    if not galaxy_config.check_conditional( name ): 
     21        ignore.append( name ) 
     22if not c.find( ignore=ignore ): 
     23    print "Some of your Galaxy eggs are out of date.  Please update them" 
     24    print "by running:" 
     25    print "  python scripts/fetch_eggs.py" 
     26    sys.exit( 1 ) 

  • scripts/fetch_eggs.py

    r1125 r1172  
    1 import sys 
    2 from eggs import * 
     1""" 
     2usage: fetch_eggs.py [egg_name] 
     3    With no arguments, fetches all eggs necessary according to the 
     4    settings in universe_wsgi.ini. 
     5  egg_name - Fetch only this egg (as defined in eggs.ini) or 'all' for 
     6    all eggs (even those not required by your settings). 
     7""" 
     8import os, sys, logging 
    39 
    4 config_eggs = ConfigEggs() 
    5 galaxy_eggs = GalaxyEggs() 
     10root = logging.getLogger() 
     11root.setLevel( 10 ) 
     12root.addHandler( logging.StreamHandler( sys.stdout ) ) 
    613 
    7 # "all" argument used to fetch all eggs, for buildbot 
    8 if len( sys.argv ) == 2: 
    9     if sys.argv[1] == "all": 
    10         galaxy_config = "all" 
     14lib = os.path.abspath( os.path.join( os.path.dirname( __file__ ), "..", "lib" ) ) 
     15sys.path.append( lib ) 
     16 
     17from galaxy.eggs import * 
     18 
     19c = Crate() 
     20c.parse() 
     21try: 
     22    if len( sys.argv ) == 1: 
     23        galaxy_config = GalaxyConfig() 
     24        ignore = [] 
     25        for name in c.get_names(): 
     26            if not galaxy_config.check_conditional( name ): 
     27                ignore.append( name ) 
     28        c.fetch( ignore=ignore ) 
    1129    else: 
    12         print "Unknown argument:", sys.argv[1] 
    13         sys.exit( 1 ) 
    14 else: 
    15     galaxy_config = GalaxyConfig() 
    16  
    17 missing_eggs = get_missing_eggs( config_eggs, galaxy_eggs, galaxy_config ) 
    18 if len( missing_eggs ) != 0: 
    19     ret = fetch_eggs( config_eggs.repo, missing_eggs ) 
    20     if ret == []: 
    21         print "Eggs fetched successfully" 
    22         sys.exit( 0 ) 
    23     else: 
    24         print "" 
    25         print 'fetch_eggs.py was unable to download some eggs.  You may have success' 
    26         print '"scrambling" them yourself:' 
    27         print "" 
    28         for name in ret: 
    29             print " ", sys.executable, os.path.join( os.path.dirname( sys.argv[0] ), "scramble.py" ), name 
    30         print "" 
    31         sys.exit( 1 ) 
    32 else: 
    33     print "All eggs are up to date for this revision of Galaxy" 
    34 sys.exit( 0 ) 
     30        if sys.argv[1] == 'all': 
     31            c.fetch() 
     32        else: 
     33            egg = c.get( sys.argv[1] ) 
     34            if egg is None: 
     35                print "error: %s not in eggs.ini" % sys.argv[1] 
     36                sys.exit( 1 ) 
     37            egg.fetch() 
     38except EggNotFetchable, e: 
     39    print "One of the python eggs necessary to run Galaxy couldn't be downloaded" 
     40    print "automatically.  You may want to try building it by hand with:" 
     41    print "  python scripts/scramble.py %s" % e 
     42    sys.exit( 1 ) 
     43except PlatformNotSupported, e: 
     44    print "Your platform (%s) is not supported." % e 
     45    print "Pre-built galaxy eggs are not available from the Galaxy developers for" 
     46    print "your platform.  You may be able to build them by hand with:" 
     47    print "  python scripts/scramble.py" 
     48    sys.exit( 1 ) 

  • scripts/nosetests.py

    r1129 r1172  
    11#!/usr/bin/env python 
    22# EASY-INSTALL-ENTRY-SCRIPT: 'nose','console_scripts','nosetests' 
    3 __requires__ = 'nose' 
    4 import sys 
     3#__requires__ = 'nose' 
     4import os, sys 
     5 
     6new_path = [ os.path.join( os.getcwd(), "lib" ) ] 
     7new_path.extend( sys.path[1:] ) 
     8sys.path = new_path 
     9 
     10from galaxy import eggs 
     11eggs.require( 'nose' ) 
     12eggs.require( 'NoseHTML' ) 
     13 
    514from pkg_resources import load_entry_point 
    615 

  • scripts/paster.py

    r1129 r1172  
    1 #!/usr/bin/env python 
     1""" 
     2Bootstrap the Galaxy framework. 
    23 
    3 import sys 
     4This should not be called directly!  Use the run.sh script in Galaxy's 
     5top level directly. 
     6""" 
     7 
     8import os, sys 
    49 
    510assert sys.version_info[:2] >= ( 2, 4 ) 
    611 
    7 import pkg_resources;  
     12new_path = [ os.path.join( os.getcwd(), "lib" ) ] 
     13new_path.extend( sys.path[1:] ) # remove scripts/ from the path 
     14sys.path = new_path 
     15 
     16from galaxy import eggs 
     17import pkg_resources 
     18 
    819pkg_resources.require( "PasteScript" ) 
    920 

  • scripts/scramble.py

    r1125 r1172  
    11""" 
    2 usage: scramble.py <egg_name> [platform] 
    3     egg_name - The name of an egg to build, as defined in eggs.ini 
    4     platform - An optional platform to build the egg for. 
     2usage: scramble.py [egg_name] 
     3    With no arguments, scrambles all eggs necessary according to the 
     4    settings in universe_wsgi.ini. 
     5  egg_name - Scramble only this egg (as defined in eggs.ini) or 'all' 
     6    for all eggs (even those not required by your settings). 
     7""" 
     8import os, sys, logging 
    59 
    6 The platform option is mostly intended for the Galaxy developers who distribute 
    7 eggs via the Galaxy Eggs site.  Please see the comments in eggs.ini for more 
    8 information about using the platform option. 
    9 """ 
    10 import os, sys 
    11 from eggs import scramble 
     10root = logging.getLogger() 
     11root.setLevel( 10 ) 
     12root.addHandler( logging.StreamHandler( sys.stdout ) ) 
    1213 
    13 if len( sys.argv ) < 2: 
    14     print __doc__ 
    15     sys.exit( 1 ) 
     14lib = os.path.abspath( os.path.join( os.path.dirname( __file__ ), "..", "lib" ) ) 
     15sys.path.append( lib ) 
    1616 
    17 #scramble_lib = os.path.join( os.path.dirname( sys.argv[0] ), "scramble", "lib" ) 
    18 #sys.path.append( scramble_lib ) 
     17from galaxy.eggs import Crate, GalaxyConfig 
    1918 
    20 #try: 
    21 #    from setuptools import * 
    22 #except: 
    23 #    from ez_setup import use_setuptools 
    24 #    use_setuptools( download_delay=8, to_dir=scramble_lib ) 
    25 #    from setuptools import * 
    26  
    27 if len( sys.argv ) == 3: 
    28     plat = sys.argv[2] 
     19c = Crate() 
     20c.parse() 
     21if len( sys.argv ) == 1: 
     22    galaxy_config = GalaxyConfig() 
     23    ignore = [] 
     24    for name in c.get_names(): 
     25        if not galaxy_config.check_conditional( name ): 
     26            ignore.append( name ) 
     27    c.scramble( ignore=ignore ) 
    2928else: 
    30     plat = "default" 
    31  
    32 scramble( sys.argv[1], plat ) 
     29    if sys.argv[1] == 'all': 
     30        c.scramble() 
     31    else: 
     32        egg = c.get( sys.argv[1] ) 
     33        if egg is None: 
     34            print "error: %s not in eggs.ini" % sys.argv[1] 
     35            sys.exit( 1 ) 
     36        egg.scramble() 

  • scripts/scramble/scripts/MySQL_python.py

    r1125 r1172  
    1515 
    1616def build_mysql(): 
    17     # download 
    18     if not os.access( MYSQL_ARCHIVE, os.F_OK ): 
    19         pkg_resources.require( "twill" ) 
    20         import twill.commands as tc 
    21         import twill.errors as te 
    22         try: 
    23             print "build_mysql(): Downloading mysql source archive from:" 
    24             print " ", MYSQL_URL 
    25             tc.go( MYSQL_URL ) 
    26             tc.code( 200 ) 
    27             tc.save_html( MYSQL_ARCHIVE ) 
    28         except te.TwillAssertionError, e: 
    29             print "build_mysql(): Unable to fetch mysql source archive from:" 
    30             print " ", MYSQL_URL 
    31             sys.exit( 1 ) 
    3217    # untar 
    3318    print "build_mysql(): Unpacking mysql source archive from:" 
    …… 
    9378MYSQL_ARCHIVE = os.path.abspath( os.path.join( "..", "..", "..", "archives", "mysql-%s.tar.gz" %MYSQL_VERSION ) ) 
    9479MYSQL_BINARY_ARCHIVE = os.path.abspath( os.path.join( "..", "..", "..", "archives", "mysql-%s-%s.tar.bz2" %( MYSQL_VERSION, pkg_resources.get_platform() ) ) ) 
    95 MYSQL_URL = "http://mysql.mirrors.pair.com/Downloads/MySQL-5.0/mysql-%s.tar.gz" %MYSQL_VERSION 
    9680# there's no need to have a completely separate build script for this 
    9781if pkg_resources.get_platform() == "macosx-10.3-fat": 

  • scripts/scramble/scripts/generic.py

    r1125 r1172  
    2222for dir in [ "build", "dist" ]: 
    2323    if os.access( dir, os.F_OK ): 
    24         print "scramble_it.py: removing dir:", dir 
     24        print "scramble.py: removing dir:", dir 
    2525        shutil.rmtree( dir ) 
    2626 

  • scripts/scramble/scripts/pbs_python-macosx.py

    r1132 r1172  
    1515 
    1616def build_torque(): 
    17     # download 
    18     if not os.access( TORQUE_ARCHIVE, os.F_OK ): 
    19         pkg_resources.require( "twill" ) 
    20         import twill.commands as tc 
    21         import twill.errors as te 
    22         try: 
    23             print "build_torque(): Downloading Torque source archive from:" 
    24             print " ", TORQUE_URL 
    25             tc.go( TORQUE_URL ) 
    26             tc.code( 200 ) 
    27             tc.save_html( TORQUE_ARCHIVE ) 
    28         except te.TwillAssertionError, e: 
    29             print "build_torque(): Unable to fetch Torque source archive from:" 
    30             print " ", TORQUE_URL 
    31             sys.exit( 1 ) 
    3217    # untar 
    3318    print "build_torque(): Unpacking Torque source archive from:" 
    …… 
    10893TORQUE_ARCHIVE = os.path.abspath( os.path.join( "..", "..", "..", "archives", "torque-%s.tar.gz" %TORQUE_VERSION ) ) 
    10994TORQUE_BINARY_ARCHIVE = os.path.abspath( os.path.join( "..", "..", "..", "archives", "torque-%s-%s.tar.bz2" %( TORQUE_VERSION, pkg_resources.get_platform() ) ) ) 
    110 TORQUE_URL = "http://www.clusterresources.com/downloads/torque/torque-%s.tar.gz" %TORQUE_VERSION 
    11195CONFIGURE  = "CFLAGS='-O -g -isysroot /Developer/SDKs/MacOSX10.4u.sdk -arch i386 -arch ppc' " 
    11296CONFIGURE += "LDFLAGS='-Wl,-syslibroot,/Developer/SDKs/MacOSX10.4u.sdk -arch i386 -arch ppc' " 

  • scripts/scramble/scripts/pbs_python.py

    r1132 r1172  
    1515 
    1616def build_torque(): 
    17     # download 
    18     if not os.access( TORQUE_ARCHIVE, os.F_OK ): 
    19         pkg_resources.require( "twill" ) 
    20         import twill.commands as tc 
    21         import twill.errors as te 
    22         try: 
    23             print "build_torque(): Downloading Torque source archive from:" 
    24             print " ", TORQUE_URL 
    25             tc.go( TORQUE_URL ) 
    26             tc.code( 200 ) 
    27             tc.save_html( TORQUE_ARCHIVE ) 
    28         except te.TwillAssertionError, e: 
    29             print "build_torque(): Unable to fetch Torque source archive from:" 
    30             print " ", TORQUE_URL 
    31             sys.exit( 1 ) 
    3217    # untar 
    3318    print "build_torque(): Unpacking Torque source archive from:" 
    …… 
    10388TORQUE_ARCHIVE = os.path.abspath( os.path.join( "..", "..", "..", "archives", "torque-%s.tar.gz" %TORQUE_VERSION ) ) 
    10489TORQUE_BINARY_ARCHIVE = os.path.abspath( os.path.join( "..", "..", "..", "archives", "torque-%s-%s.tar.bz2" %( TORQUE_VERSION, pkg_resources.get_platform() ) ) ) 
    105 TORQUE_URL = "http://www.clusterresources.com/downloads/torque/torque-%s.tar.gz" %TORQUE_VERSION 
    10690CONFIGURE = "CFLAGS='-fPIC' ./configure --prefix=/usr/local --without-tcl --without-tk" 
    10791 

  • scripts/scramble/scripts/psycopg2.py

    r1125 r1172  
    1515 
    1616def build_postgres(): 
    17     # download 
    18     if not os.access( POSTGRES_ARCHIVE, os.F_OK ): 
    19         pkg_resources.require( "twill" ) 
    20         import twill.commands as tc 
    21         import twill.errors as te 
    22         try: 
    23             print "build_postgres(): Downloading postgres source archive from:" 
    24             print " ", POSTGRES_URL 
    25             tc.go( POSTGRES_URL ) 
    26             tc.code( 200 ) 
    27             tc.save_html( POSTGRES_ARCHIVE ) 
    28         except te.TwillAssertionError, e: 
    29             print "build_postgres(): Unable to fetch postgres source archive from:" 
    30             print " ", POSTGRES_URL 
    31             sys.exit( 1 ) 
    3217    # untar 
    3318    print "build_postgres(): Unpacking postgres source archive from:" 
    …… 
    9479POSTGRES_ARCHIVE = os.path.abspath( os.path.join( "..", "..", "..", "archives", "postgresql-%s.tar.bz2" %POSTGRES_VERSION ) ) 
    9580POSTGRES_BINARY_ARCHIVE = os.path.abspath( os.path.join( "..", "..", "..", "archives", "postgresql-%s-%s.tar.bz2" %( POSTGRES_VERSION, pkg_resources.get_platform() ) ) ) 
    96 POSTGRES_URL = "http://ftp8.us.postgresql.org/postgresql/source/v%s/postgresql-%s.tar.bz2" %( POSTGRES_VERSION, POSTGRES_VERSION ) 
    9781# there's no need to have a completely separate build script for this 
    9882if pkg_resources.get_platform() == "macosx-10.3-fat": 

  • scripts/scramble/scripts/pysqlite.py

    r1125