[galaxy-commits] commit/galaxy-central: greg: Add the ability for a tool shed admin to browse repository metadata, and implement fixes for setting repository metadata for repositories with multiple tools.
Bitbucket
commits-noreply at bitbucket.org
Mon Aug 15 16:12:19 EDT 2011
1 new changeset in galaxy-central:
http://bitbucket.org/galaxy/galaxy-central/changeset/824c59db3c3a/
changeset: 824c59db3c3a
user: greg
date: 2011-08-15 22:12:12
summary: Add the ability for a tool shed admin to browse repository metadata, and implement fixes for setting repository metadata for repositories with multiple tools.
affected #: 4 files (5.8 KB)
--- a/lib/galaxy/webapps/community/controllers/admin.py Mon Aug 15 13:28:40 2011 -0400
+++ b/lib/galaxy/webapps/community/controllers/admin.py Mon Aug 15 16:12:12 2011 -0400
@@ -3,7 +3,7 @@
from galaxy.model.orm import *
from galaxy.web.framework.helpers import time_ago, iff, grids
from galaxy.util import inflector
-from common import get_category, get_repository
+from common import get_category, get_repository, get_repository_metadata_by_id
from repository import RepositoryListGrid, CategoryListGrid
import logging
log = logging.getLogger( __name__ )
@@ -308,6 +308,65 @@
async_compatible=False ) )
standard_filters = []
+class RepositoryMetadataListGrid( grids.Grid ):
+ class IdColumn( grids.IntegerColumn ):
+ def get_value( self, trans, grid, repository_metadata ):
+ return repository_metadata.id
+ class RepositoryIdColumn( grids.IntegerColumn ):
+ def get_value( self, trans, grid, repository_metadata ):
+ return repository_metadata.repository_id
+ class ChangesetRevisionColumn( grids.TextColumn ):
+ def get_value( self, trans, grid, repository_metadata ):
+ return repository_metadata.changeset_revision
+ class MetadataColumn( grids.TextColumn ):
+ def get_value( self, trans, grid, repository_metadata ):
+ metadata_str = ''
+ if repository_metadata:
+ metadata = repository_metadata.metadata
+ if metadata:
+ if 'tools' in metadata:
+ metadata_str += '<b>Tools:</b><br/>'
+ for tool_metadata_dict in metadata[ 'tools' ]:
+ metadata_str += '%s <b>%s</b><br/>' % \
+ ( tool_metadata_dict[ 'id' ], tool_metadata_dict[ 'version' ] )
+ if 'workflows' in metadata:
+ metadata_str += '<b>Workflows:</b><br/>'
+ for workflow_metadata_dict in metadata[ 'workflows' ]:
+ metadata_str += '%s <b>%s</b><br/>' % \
+ ( workflow_metadata_dict[ 'name' ], workflow_metadata_dict[ 'format-version' ] )
+ return metadata_str
+ class MaliciousColumn( grids.BooleanColumn ):
+ def get_value( self, trans, grid, repository_metadata ):
+ return repository_metadata.malicious
+ # Grid definition
+ title = "Repository Metadata"
+ model_class = model.RepositoryMetadata
+ template='/webapps/community/repository/grid.mako'
+ default_sort_key = "repository_id"
+ columns = [
+ IdColumn( "Id",
+ visible=False,
+ attach_popup=False ),
+ RepositoryIdColumn( "Repository Id",
+ key="repository_id",
+ attach_popup=False ),
+ ChangesetRevisionColumn( "Revision",
+ attach_popup=False ),
+ MetadataColumn( "Metadata",
+ attach_popup=False ),
+ MaliciousColumn( "Malicious",
+ attach_popup=False )
+ ]
+ operations = [ grids.GridOperation( "Delete",
+ allow_multiple=True ) ]
+ standard_filters = []
+ default_filter = {}
+ num_rows_per_page = 50
+ preserve_state = False
+ use_paging = True
+ def build_initial_query( self, trans, **kwd ):
+ return trans.sa_session.query( self.model_class )
+
class AdminController( BaseController, Admin ):
user_list_grid = UserListGrid()
@@ -315,9 +374,44 @@
group_list_grid = GroupListGrid()
manage_category_list_grid = ManageCategoryListGrid()
repository_list_grid = AdminRepositoryListGrid()
+ repository_metadata_list_grid = RepositoryMetadataListGrid()
@web.expose
@web.require_admin
+ def browse_repository_metadata( self, trans, **kwd ):
+ if 'operation' in kwd:
+ operation = kwd[ 'operation' ].lower()
+ if operation == "delete":
+ return self.delete_repository_metadata( trans, **kwd )
+ # Render the list view
+ return self.repository_metadata_list_grid( trans, **kwd )
+ @web.expose
+ @web.require_admin
+ def delete_repository_metadata( self, trans, **kwd ):
+ # TODO: Add a javascript confirm before this method executes....
+ params = util.Params( kwd )
+ message = util.restore_text( params.get( 'message', '' ) )
+ status = params.get( 'status', 'done' )
+ id = kwd.get( 'id', None )
+ if id:
+ ids = util.listify( id )
+ count = 0
+ for repository_metadata_id in ids:
+ repository_metadata = get_repository_metadata_by_id( trans, repository_metadata_id )
+ trans.sa_session.delete( repository_metadata )
+ trans.sa_session.flush()
+ count += 1
+ if count:
+ message = "Deleted %d repository metadata %s" % ( count, inflector.cond_plural( len( ids ), "record" ) )
+ else:
+ message = "No repository metadata ids received for deleting."
+ status = 'error'
+ trans.response.send_redirect( web.url_for( controller='admin',
+ action='browse_repository_metadata',
+ message=util.sanitize_text( message ),
+ status=status ) )
+ @web.expose
+ @web.require_admin
def browse_repositories( self, trans, **kwd ):
# We add params to the keyword dict in this method in order to rename the param
# with an "f-" prefix, simulating filtering by clicking a search link. We have
--- a/lib/galaxy/webapps/community/controllers/common.py Mon Aug 15 13:28:40 2011 -0400
+++ b/lib/galaxy/webapps/community/controllers/common.py Mon Aug 15 16:12:12 2011 -0400
@@ -78,52 +78,54 @@
.filter( and_( trans.model.RepositoryMetadata.table.c.repository_id == trans.security.decode_id( id ),
trans.model.RepositoryMetadata.table.c.changeset_revision == changeset_revision ) ) \
.first()
+def get_repository_metadata_by_id( trans, id ):
+ """Get repository metadata from the database"""
+ return trans.sa_session.query( trans.model.RepositoryMetadata ).get( trans.security.decode_id( id ) )
def get_latest_repository_metadata( trans, id ):
"""Get last metadata defined for a specified repository from the database"""
return trans.sa_session.query( trans.model.RepositoryMetadata ) \
.filter( trans.model.RepositoryMetadata.table.c.repository_id == trans.security.decode_id( id ) ) \
.order_by( trans.model.RepositoryMetadata.table.c.update_time.desc() ) \
.first()
-def set_workflowl_metadata( trans, id, change_set_revision, exported_workflow_dict, metadata_dict ):
- # We'll store everything except the workflow steps in the database.
- flush_needed = False
+def generate_workflow_metadata( trans, id, change_set_revision, exported_workflow_dict, metadata_dict ):
+ """
+ Update the received metadata_dict with changes that have been applied
+ to the received exported_workflow_dict. Store everything except the
+ workflow steps in the database.
+ """
workflow_dict = { 'a_galaxy_workflow' : exported_workflow_dict[ 'a_galaxy_workflow' ],
'name' :exported_workflow_dict[ 'name' ],
'annotation' : exported_workflow_dict[ 'annotation' ],
'format-version' : exported_workflow_dict[ 'format-version' ] }
- repository_metadata = get_latest_repository_metadata( trans, id )
- if repository_metadata:
- metadata = repository_metadata.metadata
- if metadata and 'workflows' in metadata:
- metadata_workflows = metadata[ 'workflows' ]
- found = False
- for workflow_metadata_dict in metadata_workflows:
- if 'a_galaxy_workflow' in workflow_metadata_dict and util.string_as_bool( workflow_metadata_dict[ 'a_galaxy_workflow' ] ) and \
- 'name' in workflow_metadata_dict and workflow_metadata_dict[ 'name' ] == exported_workflow_dict[ 'name' ] and \
- 'annotation' in workflow_metadata_dict and workflow_metadata_dict[ 'annotation' ] == exported_workflow_dict[ 'annotation' ] and \
- 'format-version' in workflow_metadata_dict and workflow_metadata_dict[ 'format-version' ] == exported_workflow_dict[ 'format-version' ]:
- found = True
- break
- if not found:
- metadata_workflows.append( workflow_dict )
+ if 'workflows' in metadata_dict:
+ metadata_dict[ 'workflows' ].append( workflow_dict )
+ else:
+ metadata_dict[ 'workflows' ] = [ workflow_dict ]
+ return metadata_dict
+def new_workflow_metadata_required( trans, id, metadata_dict ):
+ """
+ TODO: Currently everything about an exported workflow except the name is hard-coded, so
+ there's no real way to differentiate versions of exported workflows. If this changes at
+ some future time, this method should be enhanced accordingly...
+ """
+ if 'workflows' in metadata_dict:
+ repository_metadata = get_latest_repository_metadata( trans, id )
+ if repository_metadata:
+ if repository_metadata.metadata:
+ # The repository has metadata, so update the workflows value - no new record is needed.
+ return False
else:
- if metadata is None:
- repository_metadata.metadata = {}
- repository_metadata.metadata[ 'workflows' ] = workflow_dict
- repository_metadata.changeset_revision = change_set_revision
- trans.sa_session.add( repository_metadata )
- if not flush_needed:
- flush_needed = True
- else:
- if 'workflows' in metadata_dict:
- metadata_dict[ 'workflows' ].append( workflow_dict )
- else:
- metadata_dict[ 'workflows' ] = [ workflow_dict ]
- return flush_needed, metadata_dict
-
-def set_tool_metadata( trans, id, change_set_revision, root, name, tool, metadata_dict ):
+ # There is no saved repository metadata, so we need to create a new repository_metadata table record.
+ return True
+ # The received metadata_dict includes no metadata for workflows, so a new repository_metadata table
+ # record is not needed.
+ return False
+def generate_tool_metadata( trans, id, change_set_revision, root, name, tool, metadata_dict ):
+ """
+ Update the received metadata_dict with changes that have been
+ applied to the received tool.
+ """
tool_requirements = []
- flush_needed = False
for tr in tool.requirements:
requirement_dict = dict( name=tr.name,
type=tr.type,
@@ -145,54 +147,42 @@
tool_config=os.path.join( root, name ),
requirements=tool_requirements,
tests=tool_tests )
- repository_metadata = get_latest_repository_metadata( trans, id )
- if repository_metadata:
- # We have the last RepositoryMetadata record generated for this repository.
- metadata = repository_metadata.metadata
- if metadata and 'tools' in metadata:
- # The metadata for 1 or more tools was successfully generated in the past for this repository..
- metadata_tools = metadata[ 'tools' ]
- found = False
- for tool_metadata_dict in metadata_tools:
- if 'id' in tool_metadata_dict and tool_metadata_dict[ 'id' ] == tool.id and \
- 'version' in tool_metadata_dict and tool_metadata_dict[ 'version' ] == tool.version:
- # We've found a record for the current tool id and version, so we'll
- # update the metadata associated with the record.
- found = True
- tool_metadata_dict[ 'name' ] = tool.name
- tool_metadata_dict[ 'description' ] = tool.description
- tool_metadata_dict[ 'version_string_cmd' ] = tool.version_string_cmd
- tool_metadata_dict[ 'tool_config' ] = os.path.join( root, name )
- tool_metadata_dict[ 'requirements' ] = tool_requirements
- tool_metadata_dict[ 'tests' ] = tool_tests
- if not flush_needed:
- flush_needed = True
- if found:
- repository_metadata.changeset_revision = change_set_revision
+ if 'tools' in metadata_dict:
+ metadata_dict[ 'tools' ].append( tool_dict )
+ else:
+ metadata_dict[ 'tools' ] = [ tool_dict ]
+ return metadata_dict
+def new_tool_metadata_required( trans, id, metadata_dict ):
+ """
+ Compare the last saved metadata for each tool in the repository with the new metadata
+ in metadata_dict to determine if a new repository_metadata table record is required, or
+ if the last saved metadata record can updated instead.
+ """
+ if 'tools' in metadata_dict:
+ repository_metadata = get_latest_repository_metadata( trans, id )
+ if repository_metadata:
+ metadata = repository_metadata.metadata
+ if metadata and 'tools' in metadata:
+ # The metadata for one or more tools was successfully generated in the past
+ # for this repository, so we compare the version string for each tool id
+ # in metadata_dict with what was previously saved to see if we need to create
+ # a new table record or if we can simply update the existing record.
+ for new_tool_metadata_dict in metadata_dict[ 'tools' ]:
+ for saved_tool_metadata_dict in metadata[ 'tools' ]:
+ if new_tool_metadata_dict[ 'id' ] == saved_tool_metadata_dict[ 'id' ]:
+ if new_tool_metadata_dict[ 'version' ] != saved_tool_metadata_dict[ 'version' ]:
+ return True
else:
- if 'tools' in metadata_dict:
- metadata_dict[ 'tools' ].append( tool_dict )
- else:
- metadata_dict[ 'tools' ] = [ tool_dict ]
- if not flush_needed:
- flush_needed = True
- if not flush_needed:
- flush_needed = True
+ # We have repository metadata that does not include metadata for any tools in the
+ # repository, so we can update the existing repository metadata.
+ return False
else:
- # Either the metadata is Null, or it does not include the key 'tools'.
- if metadata is None:
- repository_metadata.metadata = {}
- repository_metadata.metadata[ 'tools' ] = [ tool_dict ]
- repository_metadata.changeset_revision = change_set_revision
- trans.sa_session.add( repository_metadata )
- if not flush_needed:
- flush_needed = True
- else:
- if 'tools' in metadata_dict:
- metadata_dict[ 'tools' ].append( tool_dict )
- else:
- metadata_dict[ 'tools' ] = [ tool_dict ]
- return flush_needed, metadata_dict
+ # There is no saved repository metadata, so we need to create a new repository_metadata
+ # table record.
+ return True
+ # The received metadata_dict includes no metadata for tools, so a new repository_metadata table
+ # record is not needed.
+ return False
def set_repository_metadata( trans, id, change_set_revision, **kwd ):
"""Set repository metadata"""
message = ''
@@ -202,7 +192,6 @@
repo = hg.repository( get_configured_ui(), repo_dir )
change_set = get_change_set( trans, repo, change_set_revision )
invalid_files = []
- flush_needed = False
if change_set is not None:
metadata_dict = {}
for root, dirs, files in os.walk( repo_dir ):
@@ -221,7 +210,8 @@
full_path = os.path.abspath( os.path.join( root, name ) )
tool = load_tool( trans, full_path )
if tool is not None:
- flush_needed, metadata_dict = set_tool_metadata( trans, id, change_set_revision, root, name, tool, metadata_dict )
+ # Update the list metadata dictionaries for tools in metadata_dict.
+ metadata_dict = generate_tool_metadata( trans, id, change_set_revision, root, name, tool, metadata_dict )
except Exception, e:
invalid_files.append( ( name, str( e ) ) )
# Find all exported workflows
@@ -233,16 +223,22 @@
workflow_text = fp.read()
fp.close()
exported_workflow_dict = from_json_string( workflow_text )
- flush_needed, metadata_dict = set_workflowl_metadata( trans, id, change_set_revision, exported_workflow_dict, metadata_dict )
+ # Update the list of metadata dictionaries for workflows in metadata_dict.
+ metadata_dict = generate_workflow_metadata( trans, id, change_set_revision, exported_workflow_dict, metadata_dict )
except Exception, e:
invalid_files.append( ( name, str( e ) ) )
if metadata_dict:
- # The metadata_dict dictionary will contain items only
- # if the repository did not already have metadata set.
- repository_metadata = trans.model.RepositoryMetadata( repository.id, change_set_revision, metadata_dict )
- trans.sa_session.add( repository_metadata )
- if not flush_needed:
- flush_needed = True
+ if new_tool_metadata_required( trans, id, metadata_dict ) or new_workflow_metadata_required( trans, id, metadata_dict ):
+ # Create a new repository_metadata table row.
+ repository_metadata = trans.model.RepositoryMetadata( repository.id, change_set_revision, metadata_dict )
+ trans.sa_session.add( repository_metadata )
+ trans.sa_session.flush()
+ else:
+ # Update the last saved repository_metadata table row.
+ repository_metadata = get_latest_repository_metadata( trans, id )
+ repository_metadata.metadata = metadata_dict
+ trans.sa_session.add( repository_metadata )
+ trans.sa_session.flush()
else:
# change_set is None
message = "Repository does not include changeset revision '%s'." % str( change_set_revision )
@@ -284,10 +280,6 @@
correction_msg = exception_msg
message += "<b>%s</b> - %s<br/>" % ( tool_file, correction_msg )
status = 'error'
- elif flush_needed:
- # We only flush if there are no tool config errors, so change sets will only have metadata
- # if everything in them is valid.
- trans.sa_session.flush()
return message, status
def get_repository_by_name( trans, name ):
"""Get a repository from the database via name"""
--- a/templates/webapps/community/admin/index.mako Mon Aug 15 13:28:40 2011 -0400
+++ b/templates/webapps/community/admin/index.mako Mon Aug 15 16:12:12 2011 -0400
@@ -43,14 +43,30 @@
<div class="page-container" style="padding: 10px;"><div class="toolMenu"><div class="toolSectionList">
- <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_categories', webapp='community' )}">Browse by category</a></div>
- <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='admin', action='browse_repositories', webapp='community' )}">Browse all repositories</a></div>
+ <div class="toolSectionTitle">
+ Repositories
+ </div>
+ <div class="toolSectionBody">
+ <div class="toolSectionBg">
+ <div class="toolTitle">
+ <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_categories', webapp='community' )}">Browse by category</a>
+ </div>
+ <div class="toolTitle">
+ <a target="galaxy_main" href="${h.url_for( controller='admin', action='browse_repositories', webapp='community' )}">Browse all repositories</a>
+ </div>
+ <div class="toolTitle">
+ <a target="galaxy_main" href="${h.url_for( controller='admin', action='browse_repository_metadata', webapp='community' )}">Browse metadata</a>
+ </div>
+ </div>
+ </div><div class="toolSectionTitle">
Categories
</div><div class="toolSectionBody"><div class="toolSectionBg">
- <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='admin', action='manage_categories', webapp='community' )}">Manage categories</a></div>
+ <div class="toolTitle">
+ <a target="galaxy_main" href="${h.url_for( controller='admin', action='manage_categories', webapp='community' )}">Manage categories</a>
+ </div></div></div><div class="toolSectionPad"></div>
@@ -59,9 +75,15 @@
</div><div class="toolSectionBody"><div class="toolSectionBg">
- <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='admin', action='users', webapp='community' )}">Manage users</a></div>
- <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='admin', action='groups', webapp='community' )}">Manage groups</a></div>
- <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='admin', action='roles', webapp='community' )}">Manage roles</a></div>
+ <div class="toolTitle">
+ <a target="galaxy_main" href="${h.url_for( controller='admin', action='users', webapp='community' )}">Manage users</a>
+ </div>
+ <div class="toolTitle">
+ <a target="galaxy_main" href="${h.url_for( controller='admin', action='groups', webapp='community' )}">Manage groups</a>
+ </div>
+ <div class="toolTitle">
+ <a target="galaxy_main" href="${h.url_for( controller='admin', action='roles', webapp='community' )}">Manage roles</a>
+ </div></div></div></div>
--- a/templates/webapps/community/index.mako Mon Aug 15 13:28:40 2011 -0400
+++ b/templates/webapps/community/index.mako Mon Aug 15 16:12:12 2011 -0400
@@ -45,13 +45,21 @@
<div class="toolMenu"><div class="toolSectionList"><div class="toolSectionPad"></div>
- <div class="toolSectionTitle">Repositories</div>
+ <div class="toolSectionTitle">
+ Repositories
+ </div><div class="toolSectionBody"><div class="toolSectionBg">
- <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_categories', webapp='community' )}">Browse by category</a></div>
- <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_repositories', webapp='community' )}">Browse all repositories</a></div>
+ <div class="toolTitle">
+ <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_categories', webapp='community' )}">Browse by category</a>
+ </div>
+ <div class="toolTitle">
+ <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_repositories', webapp='community' )}">Browse all repositories</a>
+ </div>
%if trans.user:
- <div class="toolTitle"><a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_repositories', operation='my_repositories', webapp='community' )}">Browse my repositories</a></div>
+ <div class="toolTitle">
+ <a target="galaxy_main" href="${h.url_for( controller='repository', action='browse_repositories', operation='my_repositories', webapp='community' )}">Browse my repositories</a>
+ </div>
%endif
</div></div>
Repository URL: https://bitbucket.org/galaxy/galaxy-central/
--
This is a commit notification from bitbucket.org. You are receiving
this because you have the service enabled, addressing the recipient of
this email.
More information about the galaxy-commits
mailing list