[galaxy-commits] commit/galaxy-central: greg: Add the ability for a tool shed admin to browse repository metadata, and implement fixes for setting repository metadata for repositories with multiple tools.

Bitbucket commits-noreply at bitbucket.org
Mon Aug 15 16:12:19 EDT 2011


1 new changeset in galaxy-central:

http://bitbucket.org/galaxy/galaxy-central/changeset/824c59db3c3a/
changeset:   824c59db3c3a
user:        greg
date:        2011-08-15 22:12:12
summary:     Add the ability for a tool shed admin to browse repository metadata, and implement fixes for setting repository metadata for repositories with multiple tools.
affected #:  4 files (5.8 KB)

--- a/lib/galaxy/webapps/community/controllers/admin.py	Mon Aug 15 13:28:40 2011 -0400
+++ b/lib/galaxy/webapps/community/controllers/admin.py	Mon Aug 15 16:12:12 2011 -0400
@@ -3,7 +3,7 @@
 from galaxy.model.orm import *
 from galaxy.web.framework.helpers import time_ago, iff, grids
 from galaxy.util import inflector
-from common import get_category, get_repository
+from common import get_category, get_repository, get_repository_metadata_by_id
 from repository import RepositoryListGrid, CategoryListGrid
 import logging
 log = logging.getLogger( __name__ )
@@ -308,6 +308,65 @@
                                             async_compatible=False ) )
     standard_filters = []
 
+class RepositoryMetadataListGrid( grids.Grid ):
+    class IdColumn( grids.IntegerColumn ):
+        def get_value( self, trans, grid, repository_metadata ):
+            return repository_metadata.id
+    class RepositoryIdColumn( grids.IntegerColumn ):
+        def get_value( self, trans, grid, repository_metadata ):
+            return repository_metadata.repository_id
+    class ChangesetRevisionColumn( grids.TextColumn ):
+        def get_value( self, trans, grid, repository_metadata ):
+            return repository_metadata.changeset_revision
+    class MetadataColumn( grids.TextColumn ):
+        def get_value( self, trans, grid, repository_metadata ):
+            metadata_str = ''
+            if repository_metadata:
+                metadata = repository_metadata.metadata
+                if metadata:
+                    if 'tools' in metadata:
+                        metadata_str += '<b>Tools:</b><br/>'
+                        for tool_metadata_dict in metadata[ 'tools' ]:
+                            metadata_str += '%s <b>%s</b><br/>' % \
+                                ( tool_metadata_dict[ 'id' ], tool_metadata_dict[ 'version' ] )
+                    if 'workflows' in metadata:
+                        metadata_str += '<b>Workflows:</b><br/>'
+                        for workflow_metadata_dict in metadata[ 'workflows' ]:
+                            metadata_str += '%s <b>%s</b><br/>' % \
+                                ( workflow_metadata_dict[ 'name' ], workflow_metadata_dict[ 'format-version' ] )
+            return metadata_str
+    class MaliciousColumn( grids.BooleanColumn ):
+        def get_value( self, trans, grid, repository_metadata ):
+            return repository_metadata.malicious
+    # Grid definition
+    title = "Repository Metadata"
+    model_class = model.RepositoryMetadata
+    template='/webapps/community/repository/grid.mako'
+    default_sort_key = "repository_id"
+    columns = [
+        IdColumn( "Id",
+                  visible=False,
+                  attach_popup=False ),
+        RepositoryIdColumn( "Repository Id",
+                            key="repository_id",
+                            attach_popup=False ),
+        ChangesetRevisionColumn( "Revision",
+                                 attach_popup=False ),
+        MetadataColumn( "Metadata",
+                        attach_popup=False ),
+        MaliciousColumn( "Malicious",
+                         attach_popup=False )
+    ]
+    operations = [ grids.GridOperation( "Delete",
+                                        allow_multiple=True ) ]
+    standard_filters = []
+    default_filter = {}
+    num_rows_per_page = 50
+    preserve_state = False
+    use_paging = True
+    def build_initial_query( self, trans, **kwd ):
+        return trans.sa_session.query( self.model_class )
+
 class AdminController( BaseController, Admin ):
     
     user_list_grid = UserListGrid()
@@ -315,9 +374,44 @@
     group_list_grid = GroupListGrid()
     manage_category_list_grid = ManageCategoryListGrid()
     repository_list_grid = AdminRepositoryListGrid()
+    repository_metadata_list_grid = RepositoryMetadataListGrid()
 
     @web.expose
     @web.require_admin
+    def browse_repository_metadata( self, trans, **kwd ):
+        if 'operation' in kwd:
+            operation = kwd[ 'operation' ].lower()
+            if operation == "delete":
+                return self.delete_repository_metadata( trans, **kwd )
+        # Render the list view
+        return self.repository_metadata_list_grid( trans, **kwd )
+    @web.expose
+    @web.require_admin
+    def delete_repository_metadata( self, trans, **kwd ):
+        # TODO: Add a javascript confirm before this method executes....
+        params = util.Params( kwd )
+        message = util.restore_text( params.get( 'message', ''  ) )
+        status = params.get( 'status', 'done' )
+        id = kwd.get( 'id', None )
+        if id:
+            ids = util.listify( id )
+            count = 0
+            for repository_metadata_id in ids:
+                repository_metadata = get_repository_metadata_by_id( trans, repository_metadata_id )
+                trans.sa_session.delete( repository_metadata )
+                trans.sa_session.flush()
+                count += 1
+            if count:
+                message = "Deleted %d repository metadata %s" % ( count, inflector.cond_plural( len( ids ), "record" ) )
+        else:
+            message = "No repository metadata ids received for deleting."
+            status = 'error'
+        trans.response.send_redirect( web.url_for( controller='admin',
+                                                   action='browse_repository_metadata',
+                                                   message=util.sanitize_text( message ),
+                                                   status=status ) )
+    @web.expose
+    @web.require_admin
     def browse_repositories( self, trans, **kwd ):
         # We add params to the keyword dict in this method in order to rename the param
         # with an "f-" prefix, simulating filtering by clicking a search link.  We have


--- a/lib/galaxy/webapps/community/controllers/common.py	Mon Aug 15 13:28:40 2011 -0400
+++ b/lib/galaxy/webapps/community/controllers/common.py	Mon Aug 15 16:12:12 2011 -0400
@@ -78,52 +78,54 @@
                            .filter( and_( trans.model.RepositoryMetadata.table.c.repository_id == trans.security.decode_id( id ),
                                           trans.model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) ) \
                            .first()
+def get_repository_metadata_by_id( trans, id ):
+    """Get repository metadata from the database"""
+    return trans.sa_session.query( trans.model.RepositoryMetadata ).get( trans.security.decode_id( id ) )
 def get_latest_repository_metadata( trans, id ):
     """Get last metadata defined for a specified repository from the database"""
     return trans.sa_session.query( trans.model.RepositoryMetadata ) \
                            .filter( trans.model.RepositoryMetadata.table.c.repository_id == trans.security.decode_id( id ) ) \
                            .order_by( trans.model.RepositoryMetadata.table.c.update_time.desc() ) \
                            .first()
-def set_workflowl_metadata( trans, id, change_set_revision, exported_workflow_dict, metadata_dict ):
-    # We'll store everything except the workflow steps in the database.
-    flush_needed = False
+def generate_workflow_metadata( trans, id, change_set_revision, exported_workflow_dict, metadata_dict ):
+    """
+    Update the received metadata_dict with changes that have been applied
+    to the received exported_workflow_dict.  Store everything except the
+    workflow steps in the database.
+    """
     workflow_dict = { 'a_galaxy_workflow' : exported_workflow_dict[ 'a_galaxy_workflow' ],
                       'name' :exported_workflow_dict[ 'name' ],
                       'annotation' : exported_workflow_dict[ 'annotation' ],
                       'format-version' : exported_workflow_dict[ 'format-version' ] }
-    repository_metadata = get_latest_repository_metadata( trans, id )
-    if repository_metadata:
-        metadata = repository_metadata.metadata
-        if metadata and 'workflows' in metadata:
-            metadata_workflows = metadata[ 'workflows' ]
-            found = False
-            for workflow_metadata_dict in metadata_workflows:
-                if 'a_galaxy_workflow' in workflow_metadata_dict and util.string_as_bool( workflow_metadata_dict[ 'a_galaxy_workflow' ] ) and \
-                    'name' in workflow_metadata_dict and workflow_metadata_dict[ 'name' ] == exported_workflow_dict[ 'name' ] and \
-                    'annotation' in workflow_metadata_dict and workflow_metadata_dict[ 'annotation' ] == exported_workflow_dict[ 'annotation' ] and \
-                    'format-version' in workflow_metadata_dict and workflow_metadata_dict[ 'format-version' ] == exported_workflow_dict[ 'format-version' ]:
-                    found = True
-                    break
-            if not found:
-                metadata_workflows.append( workflow_dict )
+    if 'workflows' in metadata_dict:
+        metadata_dict[ 'workflows' ].append( workflow_dict )
+    else:
+        metadata_dict[ 'workflows' ] = [ workflow_dict ]
+    return metadata_dict
+def new_workflow_metadata_required( trans, id, metadata_dict ):
+    """
+    TODO: Currently everything about an exported workflow except the name is hard-coded, so
+    there's no real way to differentiate versions of exported workflows.  If this changes at
+    some future time, this method should be enhanced accordingly...
+    """
+    if 'workflows' in metadata_dict:
+        repository_metadata = get_latest_repository_metadata( trans, id )
+        if repository_metadata:
+            if repository_metadata.metadata:
+                # The repository has metadata, so update the workflows value - no new record is needed.
+                return False
         else:
-            if metadata is None:
-                repository_metadata.metadata = {}
-            repository_metadata.metadata[ 'workflows' ] = workflow_dict
-            repository_metadata.changeset_revision = change_set_revision
-            trans.sa_session.add( repository_metadata )
-            if not flush_needed:
-                flush_needed = True
-    else:
-        if 'workflows' in metadata_dict:
-            metadata_dict[ 'workflows' ].append( workflow_dict )
-        else:
-            metadata_dict[ 'workflows' ] = [ workflow_dict ]
-    return flush_needed, metadata_dict
-
-def set_tool_metadata( trans, id, change_set_revision, root, name, tool, metadata_dict ):
+            # There is no saved repository metadata, so we need to create a new repository_metadata table record.
+            return True
+    # The received metadata_dict includes no metadata for workflows, so a new repository_metadata table
+    # record is not needed.
+    return False
+def generate_tool_metadata( trans, id, change_set_revision, root, name, tool, metadata_dict ):
+    """
+    Update the received metadata_dict with changes that have been
+    applied to the received tool.
+    """
     tool_requirements = []
-    flush_needed = False
     for tr in tool.requirements:
         requirement_dict = dict( name=tr.name,
                                  type=tr.type,
@@ -145,54 +147,42 @@
                       tool_config=os.path.join( root, name ),
                       requirements=tool_requirements,
                       tests=tool_tests )
-    repository_metadata = get_latest_repository_metadata( trans, id )
-    if repository_metadata:
-        # We have the last RepositoryMetadata record generated for this repository.
-        metadata = repository_metadata.metadata
-        if metadata and 'tools' in metadata:
-            # The metadata for 1 or more tools was successfully generated in the past for this repository..
-            metadata_tools = metadata[ 'tools' ]
-            found = False
-            for tool_metadata_dict in metadata_tools:
-                if 'id' in tool_metadata_dict and tool_metadata_dict[ 'id' ] == tool.id and \
-                    'version' in tool_metadata_dict and tool_metadata_dict[ 'version' ] == tool.version:
-                    # We've found a record for the current tool id and version, so we'll 
-                    # update the metadata associated with the record.
-                    found = True
-                    tool_metadata_dict[ 'name' ] = tool.name
-                    tool_metadata_dict[ 'description' ] = tool.description
-                    tool_metadata_dict[ 'version_string_cmd' ] = tool.version_string_cmd
-                    tool_metadata_dict[ 'tool_config' ] = os.path.join( root, name )
-                    tool_metadata_dict[ 'requirements' ] = tool_requirements
-                    tool_metadata_dict[ 'tests' ] = tool_tests
-                    if not flush_needed:
-                        flush_needed = True
-            if found:
-                repository_metadata.changeset_revision = change_set_revision
+    if 'tools' in metadata_dict:
+        metadata_dict[ 'tools' ].append( tool_dict )
+    else:
+        metadata_dict[ 'tools' ] = [ tool_dict ]
+    return metadata_dict
+def new_tool_metadata_required( trans, id, metadata_dict ):
+    """
+    Compare the last saved metadata for each tool in the repository with the new metadata
+    in metadata_dict to determine if a new repository_metadata table record is required, or
+    if the last saved metadata record can updated instead.
+    """
+    if 'tools' in metadata_dict:
+        repository_metadata = get_latest_repository_metadata( trans, id )
+        if repository_metadata:
+            metadata = repository_metadata.metadata
+            if metadata and 'tools' in metadata:
+                # The metadata for one or more tools was successfully generated in the past
+                # for this repository, so we compare the version string for each tool id
+                # in metadata_dict with what was previously saved to see if we need to create
+                # a new table record or if we can simply update the existing record.
+                for new_tool_metadata_dict in metadata_dict[ 'tools' ]:
+                    for saved_tool_metadata_dict in metadata[ 'tools' ]:
+                        if new_tool_metadata_dict[ 'id' ] == saved_tool_metadata_dict[ 'id' ]:
+                            if new_tool_metadata_dict[ 'version' ] != saved_tool_metadata_dict[ 'version' ]:
+                                return True
             else:
-                if 'tools' in metadata_dict:
-                    metadata_dict[ 'tools' ].append( tool_dict )
-                else:
-                    metadata_dict[ 'tools' ] = [ tool_dict ]
-                if not flush_needed:
-                    flush_needed = True
-            if not flush_needed:
-                flush_needed = True
+                # We have repository metadata that does not include metadata for any tools in the
+                # repository, so we can update the existing repository metadata.
+                return False
         else:
-            # Either the metadata is Null, or it does not include the key 'tools'.
-            if metadata is None:
-                repository_metadata.metadata = {}
-            repository_metadata.metadata[ 'tools' ] = [ tool_dict ]
-            repository_metadata.changeset_revision = change_set_revision
-            trans.sa_session.add( repository_metadata )
-            if not flush_needed:
-                flush_needed = True
-    else:
-        if 'tools' in metadata_dict:
-            metadata_dict[ 'tools' ].append( tool_dict )
-        else:
-            metadata_dict[ 'tools' ] = [ tool_dict ]
-    return flush_needed, metadata_dict
+            # There is no saved repository metadata, so we need to create a new repository_metadata
+            # table record.
+            return True
+    # The received metadata_dict includes no metadata for tools, so a new repository_metadata table
+    # record is not needed.
+    return False
 def set_repository_metadata( trans, id, change_set_revision, **kwd ):
     """Set repository metadata"""
     message = ''
@@ -202,7 +192,6 @@
     repo = hg.repository( get_configured_ui(), repo_dir )
     change_set = get_change_set( trans, repo, change_set_revision )
     invalid_files = []
-    flush_needed = False
     if change_set is not None:
         metadata_dict = {}
         for root, dirs, files in os.walk( repo_dir ):
@@ -221,7 +210,8 @@
                             full_path = os.path.abspath( os.path.join( root, name ) )
                             tool = load_tool( trans, full_path )
                             if tool is not None:
-                                flush_needed, metadata_dict = set_tool_metadata( trans, id, change_set_revision, root, name, tool, metadata_dict  )
+                                # Update the list metadata dictionaries for tools in metadata_dict.
+                                metadata_dict = generate_tool_metadata( trans, id, change_set_revision, root, name, tool, metadata_dict )
                         except Exception, e:
                             invalid_files.append( ( name, str( e ) ) )
                     # Find all exported workflows
@@ -233,16 +223,22 @@
                             workflow_text = fp.read()
                             fp.close()
                             exported_workflow_dict = from_json_string( workflow_text )
-                            flush_needed, metadata_dict = set_workflowl_metadata( trans, id, change_set_revision, exported_workflow_dict, metadata_dict )
+                            # Update the list of metadata dictionaries for workflows in metadata_dict.
+                            metadata_dict = generate_workflow_metadata( trans, id, change_set_revision, exported_workflow_dict, metadata_dict )
                         except Exception, e:
                             invalid_files.append( ( name, str( e ) ) )
         if metadata_dict:
-            # The metadata_dict dictionary will contain items only
-            # if the repository did not already have metadata set.  
-            repository_metadata = trans.model.RepositoryMetadata( repository.id, change_set_revision, metadata_dict )
-            trans.sa_session.add( repository_metadata )
-            if not flush_needed:
-                flush_needed = True
+            if new_tool_metadata_required( trans, id, metadata_dict ) or new_workflow_metadata_required( trans, id, metadata_dict ):
+                # Create a new repository_metadata table row.
+                repository_metadata = trans.model.RepositoryMetadata( repository.id, change_set_revision, metadata_dict )
+                trans.sa_session.add( repository_metadata )
+                trans.sa_session.flush()
+            else:
+               # Update the last saved repository_metadata table row.
+               repository_metadata = get_latest_repository_metadata( trans, id )
+               repository_metadata.metadata = metadata_dict
+               trans.sa_session.add( repository_metadata )
+               trans.sa_session.flush()
     else:
         # change_set is None
         message = "Repository does not include changeset revision '%s'." % str( change_set_revision )
@@ -284,10 +280,6 @@
                correction_msg = exception_msg
             message += "<b>%s</b> - %s<br/>" % ( tool_file, correction_msg )
         status = 'error'
-    elif flush_needed:
-        # We only flush if there are no tool config errors, so change sets will only have metadata
-        # if everything in them is valid.
-        trans.sa_session.flush()
     return message, status
 def get_repository_by_name( trans, name ):
     """Get a repository from the database via name"""


--- a/templates/webapps/community/admin/index.mako	Mon Aug 15 13:28:40 2011 -0400
+++ b/templates/webapps/community/admin/index.mako	Mon Aug 15 16:12:12 2011 -0400
@@ -43,14 +43,30 @@
     <div class="page-container" style="padding: 10px;"><div class="toolMenu"><div class="toolSectionList">
-                <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_categories', webapp='community' )}">Browse by category</a></div>
-                <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='admin', action='browse_repositories', webapp='community' )}">Browse all repositories</a></div>
+                <div class="toolSectionTitle">
+                    Repositories
+                </div>
+                <div class="toolSectionBody">
+                    <div class="toolSectionBg">
+                        <div class="toolTitle">
+                            <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_categories', webapp='community' )}">Browse by category</a>
+                        </div>
+                        <div class="toolTitle">
+                            <a target="galaxy_main" href="${h.url_for( controller='admin', action='browse_repositories', webapp='community' )}">Browse all repositories</a>
+                        </div>
+                        <div class="toolTitle">
+                            <a target="galaxy_main" href="${h.url_for( controller='admin', action='browse_repository_metadata', webapp='community' )}">Browse metadata</a>
+                        </div>
+                    </div>
+                </div><div class="toolSectionTitle">
                     Categories
                 </div><div class="toolSectionBody"><div class="toolSectionBg">
-                        <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='admin', action='manage_categories', webapp='community' )}">Manage categories</a></div>
+                        <div class="toolTitle">
+                            <a target="galaxy_main" href="${h.url_for( controller='admin', action='manage_categories', webapp='community' )}">Manage categories</a>
+                        </div></div></div><div class="toolSectionPad"></div>
@@ -59,9 +75,15 @@
                 </div><div class="toolSectionBody"><div class="toolSectionBg">
-                        <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='admin', action='users', webapp='community' )}">Manage users</a></div>
-                        <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='admin', action='groups', webapp='community' )}">Manage groups</a></div>
-                        <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='admin', action='roles', webapp='community' )}">Manage roles</a></div>
+                        <div class="toolTitle">
+                            <a target="galaxy_main" href="${h.url_for( controller='admin', action='users', webapp='community' )}">Manage users</a>
+                        </div>
+                        <div class="toolTitle">
+                            <a target="galaxy_main" href="${h.url_for( controller='admin', action='groups', webapp='community' )}">Manage groups</a>
+                        </div>
+                        <div class="toolTitle">
+                            <a target="galaxy_main" href="${h.url_for( controller='admin', action='roles', webapp='community' )}">Manage roles</a>
+                        </div></div></div></div>


--- a/templates/webapps/community/index.mako	Mon Aug 15 13:28:40 2011 -0400
+++ b/templates/webapps/community/index.mako	Mon Aug 15 16:12:12 2011 -0400
@@ -45,13 +45,21 @@
         <div class="toolMenu"><div class="toolSectionList"><div class="toolSectionPad"></div>
-                <div class="toolSectionTitle">Repositories</div>
+                <div class="toolSectionTitle">
+                    Repositories
+                </div><div class="toolSectionBody"><div class="toolSectionBg">
-                        <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_categories', webapp='community' )}">Browse by category</a></div>
-                        <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_repositories', webapp='community' )}">Browse all repositories</a></div>
+                        <div class="toolTitle">
+                            <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_categories', webapp='community' )}">Browse by category</a>
+                        </div>
+                        <div class="toolTitle">
+                            <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_repositories', webapp='community' )}">Browse all repositories</a>
+                        </div>
                         %if trans.user:
-                            <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_repositories', operation='my_repositories', webapp='community' )}">Browse my repositories</a></div>
+                            <div class="toolTitle">
+                                <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_repositories', operation='my_repositories', webapp='community' )}">Browse my repositories</a>
+                            </div>
                         %endif
                     </div></div>

Repository URL: https://bitbucket.org/galaxy/galaxy-central/

--

This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.


More information about the galaxy-commits mailing list