Skip to content

Commit

Permalink
Merge pull request #14 from whole-tale/master
Browse files Browse the repository at this point in the history
Merge master to stable for v0.2
  • Loading branch information
Xarthisius authored May 7, 2018
2 parents cb204b9 + 246b08d commit 3c2fa97
Show file tree
Hide file tree
Showing 7 changed files with 188 additions and 35 deletions.
12 changes: 10 additions & 2 deletions server/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from .resources.lock import Lock
from .resources.transfer import Transfer
from .resources.dm import DM
from .resources.fs import FS
from girder.models.setting import Setting
from girder.utility import setting_utilities
from girder.constants import SettingDefault
Expand Down Expand Up @@ -41,6 +42,7 @@ def load(info):
session = Session()
lock = Lock()
transfer = Transfer()
fs = FS()

pathMapper = path_mapper.PathMapper(settings)
# transferManager = transfer_manager.DelayingSimpleTransferManager(settings, pathMapper)
Expand Down Expand Up @@ -71,11 +73,17 @@ def load(info):
info['apiRoot'].dm.route('GET', ('session', ':id', 'lock',), lock.listLocksForSession)
info['apiRoot'].dm.route('GET', ('session', ':id', 'transfer'),
transfer.listTransfersForSession)
info['apiRoot'].dm.route('GET', ('session', ':id', 'item', ':itemId'),
session.getItemUnfiltered)


info['apiRoot'].dm.route('GET', ('transfer',), transfer.listTransfers)

info['apiRoot'].dm.route('GET', ('fs', 'item', ':itemId'), fs.getItemUnfiltered)
info['apiRoot'].dm.route('GET', ('fs', ':id', 'raw'), fs.getRawObject)
info['apiRoot'].dm.route('PUT', ('fs', ':id', 'setProperties'), fs.setProperties)
info['apiRoot'].dm.route('GET', ('fs', ':id', 'listing'), fs.getListing)
info['apiRoot'].dm.route('GET', ('fs', ':id', 'evict'), lock.evict)


def itemLocked(event):
dict = event.info
cacheManager.itemLocked(dict['user'], dict['itemId'], dict['sessionId'])
Expand Down
9 changes: 7 additions & 2 deletions server/lib/transfer_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
from girder.utility import assetstore_utilities
from girder.utility.model_importer import ModelImporter
from girder.models.model_base import ValidationException
from girder import logger


class TransferThread(threading.Thread):
Expand Down Expand Up @@ -98,7 +99,11 @@ def restartInterruptedTransfers(self):

for item in data:
print('Restarting transfer for item ' + str(item))
self.startTransfer(self.getUser(item['ownerId']), item['itemId'], item['sessionId'])
try:
self.startTransfer(self.getUser(item['ownerId']), item['itemId'], item['sessionId'])
except Exception as ex:
logger.warning('Failed to strart transfer for itemId %s. Reason: %s'
% (item['itemId'], ex.message))

def getUser(self, userId):
return Models.userModel.load(userId, force=True)
Expand Down Expand Up @@ -145,7 +150,7 @@ def getTransferHandler(self, transferId, itemId):
ModelImporter.model('assetstore').load(file['assetstoreId'])
adapter = assetstore_utilities.getAssetstoreAdapter(store)
url = adapter.fullPath(file)
except ValidationException:
except (AttributeError, ValidationException):
pass
if url:
try:
Expand Down
13 changes: 13 additions & 0 deletions server/models/lock.py
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,19 @@ def unlockForDeletion(self, itemId):
update={'$set': {Lock.FIELD_DELETE_IN_PROGRESS: False}},
multi=False)

def evict(self, itemId):
result = self.itemModel.update(
query={
'_id': objectid.ObjectId(itemId),
},
update={
'$set': {
Lock.FIELD_CACHED: False,
}
},
multi=False)
print('Evicting %s. Matched: %s.' % (itemId, result.matched_count))

def tryLock(self, user, sessionId, itemId, ownerId):
# Luckily, Mongo updates are atomic
result = self.itemModel.update(
Expand Down
15 changes: 9 additions & 6 deletions server/models/transfer.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from girder.models.model_base import AccessControlledModel
from girder.utility.model_importer import ModelImporter
from girder.utility import path as path_util
from girder.constants import AccessType
from ..constants import TransferStatus
from bson import objectid
Expand Down Expand Up @@ -28,7 +29,13 @@ def createTransfer(self, user, itemId, sessionId):
else:
transferId = objectid.ObjectId()

pathFromRoot = self.getPathFromRoot(user, itemId)
try:
pathFromRoot = self.getPathFromRoot(user, itemId)
except KeyError as ex:
# item does not exist any more, so delete existing transfer
if existing is not None:
self.remove(existing)
raise ex
transfer = {
'_id': transferId,
'ownerId': user['_id'],
Expand All @@ -48,11 +55,7 @@ def createTransfer(self, user, itemId, sessionId):

def getPathFromRoot(self, user, itemId):
item = self.itemModel.load(itemId, user=user, level=AccessType.READ)
dictPath = self.itemModel.parentsToRoot(item, user=user)
pathFromRoot = '/'
for dict in dictPath:
pathFromRoot = pathFromRoot + '/' + dict['object']['name']
return pathFromRoot
return path_util.getResourcePath('item', item, user=user)

def setStatus(self, transferId, status, error=None, size=0, transferred=0,
setTransferStartTime=False, setTransferEndTime=False):
Expand Down
130 changes: 130 additions & 0 deletions server/resources/fs.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
from girder.api.rest import Resource, RestException, loadmodel
from girder.constants import AccessType, TokenScope
from girder.api import access
from girder.api.describe import Description, describeRoute
from girder.utility import assetstore_utilities
from girder.models.model_base import ValidationException
from girder.utility.filesystem_assetstore_adapter import FilesystemAssetstoreAdapter
import bson

class FS(Resource):

ONE_ITEM_ONE_FILE = True

def __init__(self):
super(FS, self).__init__()
self.resourceName = 'fs'

@access.user
@describeRoute(
Description('Returns an unfiltered item')
.param('itemId', 'The ID of the item.', paramType='path')
.errorResponse('ID was invalid.')
.errorResponse('Object was not found.', 401)
)
def getItemUnfiltered(self, itemId, params):
user = self.getCurrentUser()
item = self.model('item').load(itemId, level=AccessType.READ, user=user)
return item

@access.public(scope=TokenScope.DATA_READ)
@describeRoute(
Description('Returns an unfiltered object')
.param('itemId', 'The ID of the object.', paramType='path')
.errorResponse('ID was invalid.')
.errorResponse('Object was not found.', 401)
)
def getRawObject(self, id, params):
user = self.getCurrentUser()
obj, type = self._discoverObject(id, user)
return obj

@access.public(scope=TokenScope.DATA_READ)
@describeRoute(
Description('List the content of a folder or item.')
.param('id', 'The ID of the folder/item.', paramType='path')
.errorResponse('ID was invalid.', 400)
.errorResponse('Read access was denied for the object.', 403)
)
def getListing(self, id, params):
user = self.getCurrentUser()
obj, type = self._discoverObject(id, user)
if type == 'item':
return self.listItem(obj, params, user)
if type == 'folder':
return self.listFolder(obj, params, user)
raise RestException('ID was invalid', code=400)

def listFolder(self, folder, params, user):
folders = list(
self.model('folder').childFolders(parentType='folder',
parent=folder, user=user))

files = []
for item in self.model('folder').childItems(folder=folder):
childFiles = list(self.model('item').childFiles(item))
nChildren = len(childFiles)
if nChildren == 1 or (FS.ONE_ITEM_ONE_FILE and nChildren > 1):
fileitem = childFiles[0]
fileitem['folderId'] = folder['_id']
if 'imported' not in fileitem and \
fileitem.get('assetstoreId') is not None:
try:
store = \
self.model('assetstore').load(fileitem['assetstoreId'])
adapter = assetstore_utilities.getAssetstoreAdapter(store)
fileitem["path"] = adapter.fullPath(fileitem)
except (ValidationException, AttributeError):
pass
files.append(item)
else:
if FS.ONE_ITEM_ONE_FILE:
# probably some form of brokenness. Put it there to allow deletion/fixing
files.append(item)
else:
folders.append(item)
return {'folders': folders, 'files': files}

def listItem(self, item, params, user):
files = []
for fileitem in self.model('item').childFiles(item):
if 'imported' not in fileitem and \
fileitem.get('assetstoreId') is not None:
try:
store = \
self.model('assetstore').load(fileitem['assetstoreId'])
adapter = assetstore_utilities.getAssetstoreAdapter(store)
fileitem["path"] = adapter.fullPath(fileitem)
except (ValidationException, AttributeError):
pass
files.append(fileitem)
return {'folders': [], 'files': files}

@access.user
@describeRoute(
Description('Set arbitrary property on folder/item/file')
.param('id', 'The ID of the object.', paramType='path')
.errorResponse('ID was invalid.', 400)
.errorResponse('Write access was denied.', 403)
)
def setProperties(self, id, params):
id = bson.ObjectId(id)
print("set props %s" % id)
user = self.getCurrentUser()
obj = None
type = None
if 'type' in params:
type = params['type']

if type is None:
obj, type = self._discoverObject(id, user, AccessType.ADMIN)
props = self.getBodyJson()

self.model(type).update(query={'_id': id}, update={'$set': props})

def _discoverObject(self, id, user, access=AccessType.READ):
for model in ['file', 'item', 'folder']:
obj = self.model(model).load(id, level=access, user=user)
if obj is not None:
return obj, model
raise RestException('ID was invalid.', code=400)
10 changes: 10 additions & 0 deletions server/resources/lock.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,3 +105,13 @@ def acquireLock(self, params):
)
def downloadItem(self, lock, params):
return self.model('lock', 'wt_data_manager').downloadItem(lock)

@access.user
@describeRoute(
Description('Evict an item from the cache.')
.param('id', 'The ID of the item.', paramType='path')
.errorResponse('ID was invalid.')
.errorResponse('Access was denied for the lock.', 403)
)
def evict(self, id, params):
return self.model('lock', 'wt_data_manager').evict(id)
34 changes: 9 additions & 25 deletions server/resources/session.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,16 +69,16 @@ def createSession(self, params):
return self.model('session', 'wt_data_manager').createSession(user, dataSet)

@access.user
@loadmodel(model='session', plugin='wt_data_manager', level=AccessType.READ)
@describeRoute(
Description('Get an object in a session using a path.')
.param('id', 'The ID of the session.', paramType='path')
.param('path', 'The path of the object, starting from the mount point.', paramType='query')
.param('children', 'Whether to also return a listing of all the children '
'of the object at the specified path', paramType='query')
.errorResponse('ID was invalid.')
.errorResponse('Read access was denied for the session.', 403)
.errorResponse('Object was not found.', 401)
.param('id', 'The ID of the session.', paramType='path')
.param('path', 'The path of the object, starting from the mount point.',
paramType='query')
.param('children', 'Whether to also return a listing of all the children '
'of the object at the specified path', paramType='query')
.errorResponse('ID was invalid.')
.errorResponse('Read access was denied for the session.', 403)
.errorResponse('Object was not found.', 401)
)
def getObject(self, session, params):
user = self.getCurrentUser()
Expand All @@ -89,20 +89,4 @@ def getObject(self, session, params):
return self.model('session', 'wt_data_manager').getObject(user, session,
params['path'], children)
except LookupError as ex:
raise RestException(ex.message, code=401)

@access.user
@loadmodel(model='session', plugin='wt_data_manager', level=AccessType.READ)
@describeRoute(
Description('Returns an unfiltered item in this session')
.param('id', 'The ID of the session.', paramType='path')
.param('itemId', 'The ID of the item.', paramType='path')
.errorResponse('ID was invalid.')
.errorResponse('Read access was denied for the session.', 403)
.errorResponse('Object was not found.', 401)
)
def getItemUnfiltered(self, session, itemId, params):
user = self.getCurrentUser()
self.model('session', 'wt_data_manager').checkOwnership(user, session)
item = self.model('item').load(itemId, level=AccessType.READ, user=user)
return item
raise RestException(ex.message, code=401)

0 comments on commit 3c2fa97

Please sign in to comment.