diff --git a/README.md b/README.md index f403c3df..cd5dbb9a 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ [License](https://github.com/jplusplus/detective.io/blob/master/LICENSE) • [Test coverage](https://coveralls.io/r/jplusplus/detective.io) • [Documentation](http://docs.detective.io/en/latest/) • -*Version 1.12.3 Frog* +*Version 1.12.4 Gorilla* ## Installation diff --git a/app/detective/bundle/bower.json b/app/detective/bundle/bower.json index d7a92453..d7880716 100644 --- a/app/detective/bundle/bower.json +++ b/app/detective/bundle/bower.json @@ -1,6 +1,6 @@ { "name": "detective", - "version": "1.12.3", + "version": "1.12.4", "dependencies": { "angular": "1.2.16", "angular-cookies": "1.2.16", diff --git a/app/detective/bundle/client/app/base.dj.html b/app/detective/bundle/client/app/base.dj.html index b4486dd0..b14a93fc 100644 --- a/app/detective/bundle/client/app/base.dj.html +++ b/app/detective/bundle/client/app/base.dj.html @@ -18,7 +18,16 @@ {% for picture_url in meta.pictures %} {% endfor %} - + + + diff --git a/app/detective/bundle/client/app/components/header/header.controller.coffee b/app/detective/bundle/client/app/components/header/header.controller.coffee index 72818c90..fdd46e55 100644 --- a/app/detective/bundle/client/app/components/header/header.controller.coffee +++ b/app/detective/bundle/client/app/components/header/header.controller.coffee @@ -7,33 +7,17 @@ class window.HeaderCtrl # Watch current topic @scope.$watch (=>@TopicsFactory.topic), (topic)=> @scope.topic = topic + @scope.shouldShowTopicSearch = @isInTopic @scope.shouldShowAddEntity = => - return false unless @isInTopic() - return @scope.user.hasAddPermission(@TopicsFactory.topic.ontology_as_mod) + @isInTopic() and @scope.user.hasAddPermission(@TopicsFactory.topic.ontology_as_mod) - @scope.shouldShowTopicSearch = => - in_topic = @isInTopic() - in_wrong_state = @isInEmptyState() or @isInInvite() or @isInHome() - in_topic and not in_wrong_state @scope.toggleUserMenu = @toggleUserMenu @scope.closeUserMenu = @closeUserMenu @scope.goToMyProfile = @goToMyProfile @scope.goToMySettings = @goToMySettings - isInTopic: => - topic = @TopicsFactory.topic - topic? and not _.isEmpty(topic) - - isInEmptyState: => - state = @state.current - not state? or _.isEmpty(state) or _.isEmpty(state.name) - - isInInvite: => - @state.current.name is 'user-topic-invite' - - isInHome: => - ((@state.current.name or '').match(/^home/) or []).length > 0 + isInTopic: => @state.params.topic? toggleUserMenu: => @scope.userMenuOpened = not @scope.userMenuOpened diff --git a/app/detective/bundle/client/app/components/header/header.dj.html b/app/detective/bundle/client/app/components/header/header.dj.html index a54a8543..5344d14c 100644 --- a/app/detective/bundle/client/app/components/header/header.dj.html +++ b/app/detective/bundle/client/app/components/header/header.dj.html @@ -54,7 +54,7 @@ diff --git a/app/detective/bundle/client/app/main/home/dashboard/create/customize-ontology/customize-ontology.less b/app/detective/bundle/client/app/main/home/dashboard/create/customize-ontology/customize-ontology.less index e6424131..55b19d33 100644 --- a/app/detective/bundle/client/app/main/home/dashboard/create/customize-ontology/customize-ontology.less +++ b/app/detective/bundle/client/app/main/home/dashboard/create/customize-ontology/customize-ontology.less @@ -96,6 +96,7 @@ } } + } ._jsPlumb_overlay.label { diff --git a/app/detective/bundle/client/app/main/home/dashboard/create/customize-ontology/model-form/model-form.directive.coffee b/app/detective/bundle/client/app/main/home/dashboard/create/customize-ontology/model-form/model-form.directive.coffee index 95314416..814a5640 100644 --- a/app/detective/bundle/client/app/main/home/dashboard/create/customize-ontology/model-form/model-form.directive.coffee +++ b/app/detective/bundle/client/app/main/home/dashboard/create/customize-ontology/model-form/model-form.directive.coffee @@ -6,8 +6,10 @@ angular.module('detective').directive "modelForm", ()-> modelForm: "=" submit: "&" cancel: "&" - controller: [ '$scope', ($scope)-> - FIELD_TYPES = ['string', 'richtext', 'float', 'datetime', 'url'] + mayLostFieldData: "&" + mayLostModelData: "&" + controller: [ '$scope', 'Modal', ($scope, Modal)-> + FIELD_TYPES = ['string', 'richtext', 'float', 'datetime', 'url', 'boolean'] # Transform the given string into a valid model name toModelName = (verbose_name)-> verbose_name = getSlug verbose_name, titleCase: yes @@ -17,8 +19,11 @@ angular.module('detective').directive "modelForm", ()-> # Sanitize the model to make it ready to be inserted $scope.sanitizeModel = (remove_empty_field=no, populate_empty=no)-> if $scope.model.verbose_name? - # Generate model name - $scope.model.name = toModelName $scope.model.verbose_name + # You may be allowed to change the name of the model + # with no risk of loosing data + if not $scope.mayLostModelData({model: $scope.master}) + # Generate model name + $scope.model.name = toModelName $scope.model.verbose_name else $scope.model.verbose_name = "" # Add field array @@ -29,20 +34,43 @@ angular.module('detective').directive "modelForm", ()-> continue unless $scope.isAllowedType(field) # Use name as default verbose name field.verbose_name = field.verbose_name or field.name if populate_empty - # Generate fields name - field.name = toFieldName field.verbose_name - # Field name exists? - unless field.name? and field.name isnt '' - # Should we remove empty field? - if remove_empty_field - delete $scope.model.fields[index] - $scope.model.fields.splice index, 1 - continue - # Lowercase first letter - if field.name.length < 2 - field.name = do field.name.toLowerCase + # You may be allowed to change the name of the field + # with no risk of loosing data + if not $scope.mayLostFieldData({field: field, model: $scope.master}) + # Generate fields name + field.name = toFieldName field.verbose_name + # Field name exists? + unless field.name? and field.name isnt '' + # Should we remove empty field? + if remove_empty_field + delete $scope.model.fields[index] + $scope.model.fields.splice index, 1 + continue + # Lowercase first letter + if field.name.length < 2 + field.name = do field.name.toLowerCase + else + field.name = field.name.substring(0, 1).toLowerCase() + field.name.substring(1) + # This field might not be changeable without risk else - field.name = field.name.substring(0, 1).toLowerCase() + field.name.substring(1) + # Original field + masterField = _.find($scope.master.fields, { name: field.name }) + # Type changed + if field.type isnt masterField.type + # Closure function to transmit the field type + resetType = (field, masterField)-> + # User cancel the change + (isYes)-> + if isYes + # Update the master to ask the question once + masterField.type = field.type + else + # Restore the field type + field.type = masterField.type + # Ask confirmation + m = Modal("Unconvertible data will be lost. Are you sure?", "Yes, change the type") + # Reset (or no the field's type) + m.then resetType field, masterField # Overide verbose_name_plural value if $scope.model.verbose_name.substr(-1) is 'y' # Name finishing by an y must finish by "ies" in there pluaral form @@ -85,11 +113,11 @@ angular.module('detective').directive "modelForm", ()-> $scope.model.fields.splice(index, 1) # True if the given type is allowed $scope.isAllowedType = (field)-> FIELD_TYPES.indexOf( do field.type.toLowerCase ) > -1 + # Original model + $scope.master = $scope.modelForm # Shortcut to the model object $scope.model = angular.copy $scope.modelForm or {} $scope.model = $scope.sanitizeModel yes, yes - # Original model - $scope.master = $scope.modelForm # Add default fields $scope.model.fields = [] unless $scope.model.fields? # Field that will be added to the list diff --git a/app/detective/bundle/client/app/main/home/dashboard/create/customize-ontology/model-form/model-form.dj.html b/app/detective/bundle/client/app/main/home/dashboard/create/customize-ontology/model-form/model-form.dj.html index bcba1dbe..8f32278d 100644 --- a/app/detective/bundle/client/app/main/home/dashboard/create/customize-ontology/model-form/model-form.dj.html +++ b/app/detective/bundle/client/app/main/home/dashboard/create/customize-ontology/model-form/model-form.dj.html @@ -55,6 +55,7 @@
+
diff --git a/app/detective/bundle/client/app/main/home/dashboard/create/customize-ontology/relationship-form/relationship-form.directive.coffee b/app/detective/bundle/client/app/main/home/dashboard/create/customize-ontology/relationship-form/relationship-form.directive.coffee index 8c040dbe..741b5c9a 100644 --- a/app/detective/bundle/client/app/main/home/dashboard/create/customize-ontology/relationship-form/relationship-form.directive.coffee +++ b/app/detective/bundle/client/app/main/home/dashboard/create/customize-ontology/relationship-form/relationship-form.directive.coffee @@ -7,15 +7,22 @@ angular.module('detective').directive "relationshipForm", ()-> changeBounds: "=" submit: "&" cancel: "&" - controller: [ '$scope', ($scope)-> - FIELD_TYPES = ['string', 'float', 'date', 'url'] + mayLostFieldData: "&" + controller: [ '$scope', '$state', 'Modal', ($scope, $state, Modal)-> + FIELD_TYPES = ['string', 'richtext', 'float', 'datetime', 'url', 'boolean'] + $scope.isEditing = -> + # Some field may be disable in edit mode + $state.includes("user-topic-edit") and $scope.mayLostFieldData field: $scope.relationship # Transform the given string into a valid field name toFieldName = (verbose_name)-> getSlug verbose_name, separator: '_' # Sanitize the model to make it ready to be inserted $scope.sanitizeRelationship = (remove_empty_field=no, populate_empty=no)-> if $scope.relationship.verbose_name? - # Generate model name - $scope.relationship.name = toFieldName $scope.relationship.verbose_name + # You may be allowed to change the name of the relationship + # with no risk of loosing data + if not $scope.mayLostFieldData({field: $scope.relationship}) + # Generate model name + $scope.relationship.name = toFieldName $scope.relationship.verbose_name # Complete missing data $scope.relationship.fields or= [] $scope.relationship.type = "relationship" @@ -33,20 +40,43 @@ angular.module('detective').directive "relationshipForm", ()-> continue unless $scope.isAllowedType(field) # Use name as default verbose name field.verbose_name = field.verbose_name or field.name if populate_empty - # Generate fields name - field.name = toFieldName field.verbose_name - # Field name exists? - if not field.name? or field.name is '' - # Should we remove empty field? - if remove_empty_field - delete $scope.relationship.fields[index] - $scope.relationship.fields.splice index, 1 - continue - # Lowercase first letter - if field.name.length < 2 - field.name = do field.name.toLowerCase + # You may be allowed to change the name of the field + # with no risk of loosing data + if not $scope.mayLostFieldData({field: field, model: $scope.master}) + # Generate fields name + field.name = toFieldName field.verbose_name + # Field name exists? + if not field.name? or field.name is '' + # Should we remove empty field? + if remove_empty_field + delete $scope.relationship.fields[index] + $scope.relationship.fields.splice index, 1 + continue + # Lowercase first letter + if field.name.length < 2 + field.name = do field.name.toLowerCase + else + field.name = field.name.substring(0, 1).toLowerCase() + field.name.substring(1) + # This field might not be changeable without risk else - field.name = field.name.substring(0, 1).toLowerCase() + field.name.substring(1) + # Original field + masterField = _.find($scope.master.fields, { name: field.name }) + # Type changed + if field.type isnt masterField.type + # Closure function to transmit the field type + resetType = (field, masterField)-> + # User cancel the change + (isYes)-> + if isYes + # Update the master to ask the question once + masterField.type = field.type + else + # Restore the field type + field.type = masterField.type + # Ask confirmation + m = Modal("Unconvertible data will be lost. Are you sure?", "Yes, change the type") + # Reset (or no the field's type) + m.then resetType field, masterField # Remove empty field if needed delete $scope.relationship.fields if $scope.relationship.fields.length is 0 and remove_empty_field # Returns the relationship after sanitzing @@ -75,11 +105,11 @@ angular.module('detective').directive "relationshipForm", ()-> $scope.relationship.fields.splice(index, 1) # True if the given type is allowed $scope.isAllowedType = (field)-> FIELD_TYPES.indexOf( do field.type.toLowerCase ) > -1 + # Original model + $scope.master = $scope.relationshipForm # Shortcut to the model object $scope.relationship = angular.copy($scope.relationshipForm or {}) $scope.relationship = $scope.sanitizeRelationship no, yes - # Original model - $scope.master = $scope.relationshipForm # Add default fields $scope.relationship.fields = [] unless $scope.relationship.fields? # Field that will be added to the list diff --git a/app/detective/bundle/client/app/main/home/dashboard/create/customize-ontology/relationship-form/relationship-form.dj.html b/app/detective/bundle/client/app/main/home/dashboard/create/customize-ontology/relationship-form/relationship-form.dj.html index 6b0a0c2c..8fca6588 100644 --- a/app/detective/bundle/client/app/main/home/dashboard/create/customize-ontology/relationship-form/relationship-form.dj.html +++ b/app/detective/bundle/client/app/main/home/dashboard/create/customize-ontology/relationship-form/relationship-form.dj.html @@ -31,12 +31,14 @@
+
This name already exists. @@ -68,6 +70,7 @@
+
@@ -96,4 +99,4 @@
Cancel
- \ No newline at end of file + diff --git a/app/detective/bundle/client/app/main/home/dashboard/create/customize-ontology/remove-model/remove-model.dj.html b/app/detective/bundle/client/app/main/home/dashboard/create/customize-ontology/remove-model/remove-model.dj.html index 0d9141b8..a1648a0e 100644 --- a/app/detective/bundle/client/app/main/home/dashboard/create/customize-ontology/remove-model/remove-model.dj.html +++ b/app/detective/bundle/client/app/main/home/dashboard/create/customize-ontology/remove-model/remove-model.dj.html @@ -2,8 +2,12 @@

[[model.verbose_name || model.name]]

- [[resource.description]] + [[resource.help_text]]
field.rules.is_rich or no isEditable: (field)=> @isAllowedType(field.type) and # We must say explicitely if this field is not editable diff --git a/app/detective/bundle/client/app/main/user/topic/contribute/type-datetime/type-datetime.dj.html b/app/detective/bundle/client/app/main/user/topic/contribute/type-datetime/type-datetime.dj.html index de54dff4..c6946adb 100644 --- a/app/detective/bundle/client/app/main/user/topic/contribute/type-datetime/type-datetime.dj.html +++ b/app/detective/bundle/client/app/main/user/topic/contribute/type-datetime/type-datetime.dj.html @@ -5,7 +5,7 @@ - @state.is("user-topic-edit") + isEditing: => @state.includes("user-topic-edit") isCreating: => @state.includes("user-topic-create") diff --git a/app/detective/bundle/client/app/main/user/topic/settings/customize-ontology/customize-ontology.coffee b/app/detective/bundle/client/app/main/user/topic/settings/customize-ontology/customize-ontology.coffee new file mode 100644 index 00000000..fd3fd3fa --- /dev/null +++ b/app/detective/bundle/client/app/main/user/topic/settings/customize-ontology/customize-ontology.coffee @@ -0,0 +1,7 @@ +angular.module('detective').config ["$stateProvider", ($stateProvider)-> + $stateProvider.state('user-topic-edit.customize-ontology', + controller: EditTopicOntologyCtrl + url: "structure/" + templateUrl: '/partial/main/home/dashboard/create/customize-ontology/customize-ontology.html' + ) +] \ No newline at end of file diff --git a/app/detective/bundle/client/app/main/user/topic/settings/settings.controller.coffee b/app/detective/bundle/client/app/main/user/topic/settings/settings.controller.coffee index 0ea577e0..72d9ec7e 100644 --- a/app/detective/bundle/client/app/main/user/topic/settings/settings.controller.coffee +++ b/app/detective/bundle/client/app/main/user/topic/settings/settings.controller.coffee @@ -20,11 +20,9 @@ class window.EditTopicCtrl extends window.TopicFormCtrl @init = no if @init @scope.$on @EVENTS.topic.updated, (e, topic)=> - @topic = topic - @scope.topic = @topic # avoid reference binding, otherwise @topicChanges will return an # empty object everytime. - @master = angular.copy @topic + @master = angular.copy topic @Page.title "Settings of #{@topic.title}" @@ -49,12 +47,12 @@ class window.EditTopicCtrl extends window.TopicFormCtrl changes[prop] = now_val changes - edit: (panel)=> + edit: (panel='main')=> @scope.loading[panel] = yes @scope.saved = no changes = @topicChanges @scope.topic - @TopicsFactory.update({id: @scope.topic.id}, changes, (data)=> + @TopicsFactory.update id: @scope.topic.id, changes, (data)=> @scope.$broadcast @EVENTS.topic.updated, data @scope.saved = yes @scope.loading[panel] = no @@ -63,7 +61,6 @@ class window.EditTopicCtrl extends window.TopicFormCtrl @scope.saved = no if response.status is 400 @scope.error = response.data.topic - ) diff --git a/app/detective/bundle/client/app/main/user/topic/topic.dj.html b/app/detective/bundle/client/app/main/user/topic/topic.dj.html index 48a02555..af733ad0 100644 --- a/app/detective/bundle/client/app/main/user/topic/topic.dj.html +++ b/app/detective/bundle/client/app/main/user/topic/topic.dj.html @@ -54,10 +54,17 @@

[[meta.title]]

- + + Add items - + + + Settings + + Explore

diff --git a/app/detective/bundle/client/app/main/user/topic/type/entity/card/card.directive.coffee b/app/detective/bundle/client/app/main/user/topic/type/entity/card/card.directive.coffee index b67bc45a..49b7d2b6 100644 --- a/app/detective/bundle/client/app/main/user/topic/type/entity/card/card.directive.coffee +++ b/app/detective/bundle/client/app/main/user/topic/type/entity/card/card.directive.coffee @@ -26,12 +26,10 @@ angular.module('detective').directive "card", ['Summary', 'Individual', '$sce', id: scope.individual.id # Get the value for the given field name with the current individual scope.get = (name, isrel=no)-> - unless isrel - scope.individual[name] or false - else - scope.relIndividual[name] or false + scope[ if isrel then "relIndividual" else "individual" ][name] or no + scope.getTrusted = (n) -> - val = scope.get n + val = scope.get n, yes if val? and val.length > 0 then ($sce.trustAsHtml val) else "" # True if the given property is a string scope.isString = (f)-> ["CharField", "URLField"].indexOf(f.type) > -1 diff --git a/app/detective/bundle/package.json b/app/detective/bundle/package.json index ff8b32a2..56d76720 100644 --- a/app/detective/bundle/package.json +++ b/app/detective/bundle/package.json @@ -1,6 +1,6 @@ { "name": "detective", - "version": "1.12.3", + "version": "1.12.4", "dependencies": { "bower": "latest", "coffee-script": "1.6.2", diff --git a/app/detective/individual.py b/app/detective/individual.py index 0422f908..e0aedb7b 100644 --- a/app/detective/individual.py +++ b/app/detective/individual.py @@ -11,11 +11,13 @@ from app.detective.topics.common.user import UserNestedResource from app.detective.models import Topic from app.detective.exceptions import UnavailableImage, NotAnImage, OversizedFile +from app.detective.paginator import Paginator, resource_paginator +from django import forms from django.conf import settings from django.conf.urls import url from django.contrib.auth.models import User from django.core.exceptions import ObjectDoesNotExist, ValidationError -from django.core.paginator import Paginator, InvalidPage +from django.core.paginator import InvalidPage from django.core.urlresolvers import reverse from django.core.files.storage import default_storage from django.db.models.query import QuerySet @@ -24,7 +26,6 @@ from neo4jrestclient.request import TransactionException from neo4django.db import connection from neo4django.db.models import NodeModel -from neo4django.db.models.properties import DateProperty, BoundProperty from neo4django.db.models.relationships import MultipleNodes from tastypie import fields from tastypie.authentication import Authentication, SessionAuthentication, BasicAuthentication, MultiAuthentication @@ -118,6 +119,7 @@ def dehydrate(self, bundle): del bundle.data["resource_uri"] return bundle + class IndividualResource(ModelResource): field_sources = fields.ToManyField( @@ -128,8 +130,11 @@ class IndividualResource(ModelResource): use_in='detail' ) + def __init__(self, api_name=None): super(IndividualResource, self).__init__(api_name) + # Pass the current instance of the resource to the paginator + self._meta.paginator_class = resource_paginator(self) # By default, tastypie detects detail mode globally: it means that # even into an embeded resource (through a relationship), Tastypie will # serialize it as if we are in it's detail view. @@ -271,6 +276,7 @@ def use_in(self, bundle=None): # Use in detail return self.get_resource_uri(bundle) == bundle.request.path + def dehydrate(self, bundle): # Get the request from the bundle request = bundle.request @@ -337,7 +343,6 @@ def dehydrate(self, bundle): for key, size in settings.THUMBNAIL_SIZES.items() } except InvalidImageFormatError as e: - print e to_add[field + '_thumbnail'] = '' # Convert tuple to array for better serialization @@ -398,13 +403,16 @@ def obj_get(self, **kwargs): request = bundle.request # Current model model = self.get_model() - # Get the node's data using the rest API try: node = connection.nodes.get(pk) # Node not found except client.NotFoundError: raise Http404("Not found.") + # Convert existing properties. + # Since we allow the user to change her data structure we must be able + # to convert the data she already put into the database. + node.properties = self.convert(node.properties) # Create a model istance from the node - return model._neo4j_instance(node) + return model._neo4j_instance( node ) def get_detail(self, request, **kwargs): basic_bundle = self.build_bundle(request=request) @@ -478,38 +486,24 @@ def get_search(self, request, **kwargs): query = request.GET.get('q', '').lower() query = re.sub("\"|'|`|;|:|{|}|\|(|\|)|\|", '', query).strip() limit = int( request.GET.get('limit', 20)) - exclude = int( request.GET.get('exclude', -1) ) + p = int(request.GET.get('page', 1)) # Do the query. results = self._meta.queryset.filter(name__icontains=query) - # Quicker than query exclude - results = [r for r in results if r.id != exclude] - paginator = Paginator(results, limit) + # For retro compatibility we use the django paginator + paginator = resource_paginator(self)(request.GET, results, resource_uri=self.get_resource_uri(), limit=limit, collection_name=self._meta.collection_name) + to_be_serialized = paginator.page() - try: - p = int(request.GET.get('page', 1)) - page = paginator.page(p) - except InvalidPage: - raise Http404("Sorry, no results on that page.") - - objects = [] - - for result in page.object_list: - bundle = self.build_bundle(obj=result, request=request) - bundle = self.full_dehydrate(bundle, for_list=True) - objects.append(bundle) - - object_list = { - 'objects': objects, - 'meta': { - 'q': query, - 'page': p, - 'limit': limit, - 'total_count': paginator.count - } - } + # Dehydrate the bundles in preparation for serialization. + bundles = [] - self.log_throttled_access(request) - return self.create_response(request, object_list) + for obj in to_be_serialized[self._meta.collection_name]: + bundle = self.build_bundle(obj=obj, request=request) + bundles.append(self.full_dehydrate(bundle, for_list=True)) + + to_be_serialized[self._meta.collection_name] = bundles + to_be_serialized = self.alter_list_data_to_serialize(request, to_be_serialized) + + return self.create_response(request, to_be_serialized) def get_mine(self, request, **kwargs): self.method_check(request, allowed=['get']) @@ -518,45 +512,72 @@ def get_mine(self, request, **kwargs): limit = int(request.GET.get('limit', 20)) if request.user.id is None: - object_list = { + return self.create_response(request, { 'objects': [], 'meta': { 'author': request.user, - 'page': 1, 'limit': limit, 'total_count': 0 } - } + }) else: # Do the query. results = self._meta.queryset.filter(_author__contains=request.user.id) - paginator = Paginator(results, limit) + # For retro compatibility we use the django paginator + paginator = resource_paginator(self)(request.GET, results, resource_uri=self.get_resource_uri(), limit=limit, collection_name=self._meta.collection_name) + to_be_serialized = paginator.page() - try: - p = int(request.GET.get('page', 1)) - page = paginator.page(p) - except InvalidPage: - raise Http404("Sorry, no results on that page.") + # Dehydrate the bundles in preparation for serialization. + bundles = [] - objects = [] + for obj in to_be_serialized[self._meta.collection_name]: + bundle = self.build_bundle(obj=obj, request=request) + bundles.append(self.full_dehydrate(bundle, for_list=True)) - for result in page.object_list: - bundle = self.build_bundle(obj=result, request=request) - bundle = self.full_dehydrate(bundle, for_list=True) - objects.append(bundle) + to_be_serialized[self._meta.collection_name] = bundles + to_be_serialized = self.alter_list_data_to_serialize(request, to_be_serialized) - object_list = { - 'objects': objects, - 'meta': { - 'author': request.user, - 'page': p, - 'limit': limit, - 'total_count': paginator.count - } - } + return self.create_response(request, to_be_serialized) - self.log_throttled_access(request) - return self.create_response(request, object_list) + def convert(self, properties, model=None): + if model is None: model = self.get_model() + validate = False + # Iterate until the whole properties object validates + while not validate: + try: + self.validate(properties, model=model) + validate = True + except ValidationError as e: + # Convert each key + for key in e.message_dict.keys(): + value = self.convert_field(key, properties[key], model=model) + # Skip unconvertible values + if value is None: del properties[key] + # Save the value + else: properties[key] = value + return properties + + def convert_field(self, name, value, model=None): + if model is None: model = self.get_model() + # Find the model's field + field = self.get_model_field(name, model) + # Find the field type + fieldtype = field._property.get_internal_type() + # Get the field widget + formfield = field._property.formfield() + # Choose the best way to convert + try: + if fieldtype == 'BooleanField': + return bool(value) + elif fieldtype == 'CharField': + return str(value) + elif fieldtype == 'DateTimeField': + return forms.DateTimeField().clean(value) + else: + return formfield.clean(value) + # Wrong convettion result to a None value + except (ValueError, TypeError, ValidationError): + return None def validate(self, data, model=None, allow_missing=False): if model is None: model = self.get_model() @@ -573,6 +594,18 @@ def validate(self, data, model=None, allow_missing=False): # Skip this field else: continue cleaned_data[field_name] = data[field_name] + # DateTime field must be validate manually + elif field.get_internal_type() == 'DateTimeField': + # Create a native datetimefield + formfield = forms.DateTimeField(input_formats=settings.DATETIME_FORMATS, required=False) + try: + # Validate and clean the data + cleaned_data[field_name] = formfield.clean(data[field_name]) + except ValidationError as e: + # Raise the same error the field name as key + if not allow_missing: raise ValidationError({field_name: 'Must be a valid date/time'}) + # Skip this field + else: continue # Only literal values have a _property attribute elif hasattr(field, "_property"): try: @@ -590,7 +623,7 @@ def validate(self, data, model=None, allow_missing=False): # @warning: this will validate the data for # array of values but not clean them cleaned_data[field_name] = data[field_name] - except ValidationError as e: + except ValidationError: # Raise the same error the field name as key if not allow_missing: raise ValidationError({field_name: e.messages}) # The given value is a relationship @@ -848,7 +881,6 @@ def create_source(individual, data): elif request.method == 'DELETE': delete_source(source_id) - print "Took %f to patch sources" % (time.time() - start_time) return self.create_response(request, source) def get_authors(self, request, **kwargs): diff --git a/app/detective/models.py b/app/detective/models.py index 431b0782..ee398a39 100644 --- a/app/detective/models.py +++ b/app/detective/models.py @@ -205,9 +205,16 @@ def app_label(self): def get_module_token(size=10, chars=string.ascii_uppercase + string.digits): return "topic%s" % ''.join(random.choice(chars) for x in range(size)) - def get_module(self): - from app.detective import topics - return getattr(topics, self.app_label()) + def get_module_path(self): + return "app.detective.topics.%s" % self.module + + def get_module(self, reload_module=True): + if reload_module and self.ontology_as_json: + module = self.reload() + else: + from app.detective import topics + module = getattr(topics, self.app_label()) + return module def get_models_module(self): """ return the module topic_module.models """ @@ -223,26 +230,38 @@ def get_models(self): if inspect.isclass(klass) and issubclass(klass, models.Model): yield klass + def get_model(self, name): + model = None + for m in self.get_models(): + if m.__name__.lower() == name.lower(): + model = m + return model + def clean(self): models.Model.clean(self) def save(self, *args, **kwargs): + try: + # Original model before saving + orig = Topic.objects.get(pk=self.pk) + except Topic.DoesNotExist: + orig = None # Ensure that the module field is populated with app_label() self.ontology_as_mod = self.app_label() - # For automatic slug generation. if not self.slug: self.slug = slugify(self.title)[:50] - # Call the parent save method super(Topic, self).save(*args, **kwargs) - # Refresh the API - #self.reload() + # Ontology changed + if orig is not None and orig.ontology_as_json != self.ontology_as_json: + # Refresh the API + self.reload() def reload(self): from app.detective.register import topic_models # Register the topic's models again - topic_models(self.get_module().__name__, force=True) + return topic_models(self.get_module_path(), force=True) def has_default_ontology(self): try: @@ -310,7 +329,7 @@ def entities_count(self): if response is None: query = """ START a = node(0) - MATCH a-[`<>`]->(b)--> c + MATCH a-[:`<>`]->(b)-[:`<>`]->(c) WHERE b.app_label = "{app_label}" AND not(has(c._relationship)) RETURN count(c) as count; @@ -380,7 +399,7 @@ def rdf_search_query(self, subject, predicate, obj): model=subject["name"], app=self.app_label() ) - + # If the received identifier describe a literal value elif self.is_registered_relationship(predicate["name"]): fields = utils.iterate_model_fields( all_models[predicate["subject"]] ) @@ -404,7 +423,6 @@ def rdf_search_query(self, subject, predicate, obj): is_out='<' if relationships[0]['direction'] == 'out' else '', is_in='>' if relationships[0]['direction'] == 'in' else '' ) - print query else: return {'errors': 'Unkown predicate type: %s' % predicate["name"]} return connection.cypher(query).to_dicts() diff --git a/app/detective/paginator.py b/app/detective/paginator.py new file mode 100644 index 00000000..70a36204 --- /dev/null +++ b/app/detective/paginator.py @@ -0,0 +1,39 @@ +from tastypie.paginator import Paginator as TastypiePaginator +from django.core.exceptions import ValidationError + +# We use a custom Paginator embeded into a close that receive a resource class. +# That way we can pass to tastypie's resource a Paginator awares of the resource +# it might have to use. Since Detective implements models with field types that +# can change over time, it is important to be able to convert old data types into +# the new one during runtime. +def resource_paginator(resource=None, base=TastypiePaginator): + class Paginator(base): + # Closure function to receive the model used to convert a node to an instance + def get_converter(self, model): + def neo4j_instance(node): + try: + resource.validate(node.properties, model=model) + # The given node properties don't validate with the resource model + except ValidationError as e: + node.properties = resource.convert(node.properties, model=model) + # No resource given to make the convertion + except NameError: pass + # Return the model instance + return model._neo4j_instance(node) + return neo4j_instance + # Override the get_slice method of the paginator to allow + # dynamic node convertion. Since every model instance is sent to the user + # using the paginator, we use it an interface to convert data. + def get_slice(self, limit, offset): + # Override the query function that transforms node into neo4django model. + # We pass the current model through a closure. + self.objects.query.model_from_node = self.get_converter(self.objects.model) + # Iterate over the objects using the modified query + subset = [o for o in self.objects.query.execute(self.objects.db) ] + # No limit parameter, we return the whole subset from the given offset + if limit == 0: return subset[offset:] + return subset[offset:offset + limit] + return Paginator + +# Allows paginator exportation without auto convertion +Paginator = resource_paginator() \ No newline at end of file diff --git a/app/detective/register.py b/app/detective/register.py index 6e369a78..d7404ab1 100644 --- a/app/detective/register.py +++ b/app/detective/register.py @@ -133,18 +133,22 @@ def topics_rules(): def import_or_create(path, register=True, force=False): try: - # For the new module to be written - if force: - if path in sys.modules: del( sys.modules[path] ) - raise ImportError - # Import the models.py file + # Import the module once module = importlib.import_module(path) + # If it doesn't raise an Importerror, + # we may need to reload it + if force: + # The module isn't a file + if getattr(module, "__file__", None) is None and path in sys.modules: + del( sys.modules[path] ) + # Reimport the module + raise ImportError # File dosen't exist, we create it virtually! except ImportError: - path_parts = path.split(".") - module = imp.new_module(path) - module.__name__ = path - name = path_parts[-1] + path_parts = path.split(".") + module = imp.new_module(path) + module.__name__ = path + name = path_parts[-1] # Register the new module in the global scope if register: # Get the parent module @@ -165,6 +169,14 @@ def reload_urlconf(urlconf=None): if urlconf in sys.modules: reload(sys.modules[urlconf]) +def clean_topic(path): + mod_to_delete = [] + for mod_name in sys.modules: + if mod_name.startswith(path): + mod_to_delete.append(mod_name) + for mod_name in mod_to_delete: + del sys.modules[mod_name] + def topic_models(path, force=False): """ Auto-discover topic-related model by looking into @@ -178,6 +190,8 @@ def topic_models(path, force=False): {path}.summary {path}.urls """ + # Clean the topic virtual instances from sys.module + if force: clean_topic(path) topic_module = import_or_create(path, force=force) topic_name = path.split(".")[-1] # Ensure that the topic's model exist @@ -223,7 +237,6 @@ def topic_models(path, force=False): # * as an attribute of `resources` # * as a module setattr(resources, resource_name, Resource) - sys.modules[resource_path] = Resource # And register it into the API instance api.register(Resource()) # Every app have to instance a SummaryResource class @@ -262,4 +275,6 @@ def topic_models(path, force=False): # At last, force the url resolver to reload (because we update it) clear_url_caches() reload_urlconf() - return topic_module \ No newline at end of file + topic_module.__name__ = path + sys.modules[path] = topic_module + return topic_module diff --git a/app/detective/tests/api.py b/app/detective/tests/api.py index 962cbfc8..b1030480 100644 --- a/app/detective/tests/api.py +++ b/app/detective/tests/api.py @@ -605,10 +605,6 @@ def test_search_organization(self): # At least 2 results self.assertGreater( len(data.items()), 1 ) - def test_search_organization_wrong_page(self): - resp = self.api_client.get('/api/detective/energy/v1/organization/search/?q=Roméra&page=10000', format='json', authentication=self.get_super_credentials()) - self.assertEqual(resp.status_code in [302, 404], True) - def test_cypher_detail(self): resp = self.api_client.get('/api/detective/common/v1/cypher/111/', format='json', authentication=self.get_super_credentials()) self.assertTrue(resp.status_code in [302, 404]) @@ -882,12 +878,33 @@ def add_properties(pilule_id, molecule_id, property): self.assertIn("quantity_(in_milligrams).", relation.keys() , relation) self.assertEqual(relation["quantity_(in_milligrams)."], property , relation) + # Get orphan count + def count_orphans(): + orphans_count = 0 + for Model in topic.get_models(): + for field in utils.iterate_model_fields(Model): + if field["rel_type"] and "through" in field["rules"]: + ids= [] + for entity in Model.objects.all(): + ids.extend([_.id for _ in entity.node.relationships.all()]) + Properties = field["rules"]["through"] + for info in Properties.objects.all(): + if info._relationship not in ids: + orphans_count += 1 + return orphans_count + + # Check if orphans exist. It shouldn't ! + def has_more_orphans(orphans_count=0): + self.assertEqual(orphans_count, count_orphans()) + topic = Topic.objects.get(slug='test-pillen') models = topic.get_models_module() # get models PillMoleculesContainedMoleculeProperties = models.PillMoleculesContainedMoleculeProperties Molecule = models.Molecule Pill = models.Pill + # Initial orphan count + orphans_count = count_orphans() # create entities pilulea = Pill .objects.create(name='pilule A') mola = Molecule.objects.create(name="molecule A") @@ -936,28 +953,14 @@ def patch_mol_b_c_d(): self.assertNotIn("quantity_(in_milligrams).", relation_pamd.keys() , relation_pamd) self.assertTrue(int(relation_pamd["_relationship"]) > 0 , relation_pamd) self.assertEquals(relation_pamd["_endnodes"], [pilulea.id, mold_wo_infos.id]) - # check if orphans exist. It shouldn't ! - def check_if_orphans_exist(): - orphans_count = 0 - for Model in topic.get_models(): - for field in utils.iterate_model_fields(Model): - if field["rel_type"] and "through" in field["rules"]: - ids= [] - for entity in Model.objects.all(): - ids.extend([_.id for _ in entity.node.relationships.all()]) - Properties = field["rules"]["through"] - for info in Properties.objects.all(): - if info._relationship not in ids: - orphans_count += 1 - self.assertEqual(orphans_count, 0) # remove one molecule molb.delete() - check_if_orphans_exist() + has_more_orphans(orphans_count) pilulea.delete() mola.delete() molc.delete() mold_wo_infos.delete() - check_if_orphans_exist() + has_more_orphans(orphans_count) def test_patch_relations(self): """ @@ -1158,6 +1161,8 @@ def test_topic_entities_count(self): PillMoleculesContainedMoleculeProperties = models.PillMoleculesContainedMoleculeProperties Molecule = models.Molecule Pill = models.Pill + # Original count + original_count = topic.entities_count() # create entities pilulea = Pill .objects.create(name='pilule A') mola = Molecule.objects.create(name="molecule A") @@ -1167,16 +1172,16 @@ def test_topic_entities_count(self): "quantity_(in_milligrams)." : "12" } PillMoleculesContainedMoleculeProperties.objects.create(**relation_args) - self.assertEqual(topic.entities_count(), 2) + self.assertEqual(topic.entities_count(), original_count + 2) pilulea.molecules_contained.add(mola) - self.assertEqual(topic.entities_count(), 2) + self.assertEqual(topic.entities_count(), original_count + 2) molb = Molecule.objects.create(name="molecule B") - self.assertEqual(topic.entities_count(), 3) + self.assertEqual(topic.entities_count(), original_count + 3) molb.delete() - self.assertEqual(topic.entities_count(), 2) + self.assertEqual(topic.entities_count(), original_count + 2) mola.delete() pilulea.delete() - self.assertEqual(topic.entities_count(), 0) + self.assertEqual(topic.entities_count(), original_count) def test_topic_endpoint_exists(self): resp = self.api_client.get('/api/detective/common/v1/topic/?slug=christmas', follow=True, format='json') diff --git a/app/detective/topics/common/summary.py b/app/detective/topics/common/summary.py index 94e47ae4..696c95aa 100644 --- a/app/detective/topics/common/summary.py +++ b/app/detective/topics/common/summary.py @@ -146,19 +146,23 @@ def summary_types(self, bundle, request): del t["name"] return obj + + def sanitize_field(self, field): + if "through" in field["rules"]: + field["rules"]["through"] = getattr(field["rules"]["through"], "__name__") + return field + def summary_forms(self, bundle, request): available_resources = {} # Get the model's rules manager - rulesManager = request.current_topic.get_rules() + rulesManager = self.topic.get_rules() # Fetch every registered model # to print out its rules for model in self.topic.get_models(): name = model.__name__.lower() rules = rulesManager.model(model).all() - fields = utils.get_model_fields(model) verbose_name = getattr(model._meta, "verbose_name", name) verbose_name_plural = getattr(model._meta, "verbose_name_plural", verbose_name + "s") - for key in rules: # Filter rules to keep only Neomatch if isinstance(rules[key], Neomatch): @@ -170,20 +174,11 @@ def summary_forms(self, bundle, request): "related_model": rules[key].target_model.__name__ }) - for field in fields: - # Create a copy of the rule to avoid compromize the rules singleton - field["rules"] = field["rules"].copy() - for key, rule in field["rules"].items(): - # Convert class to model name - if inspect.isclass(rule): - field["rules"][key] = getattr(rule, "__name__", rule) + fields = [ field.copy() for field in utils.iterate_model_fields(model) ] + fields = [ self.sanitize_field(field) for field in fields ] - try: - idx = model.__idx__ - except AttributeError: - idx = 0 available_resources[name] = { - 'description' : getattr(model, "_description", None), + 'help_text' : getattr(model, "_description", None), 'topic' : getattr(model, "_topic", self.topic.slug) or self.topic.slug, 'model' : getattr(model, "__name__", ""), 'verbose_name' : verbose_name, @@ -191,11 +186,13 @@ def summary_forms(self, bundle, request): 'name' : name, 'fields' : fields, 'rules' : rules, - 'index' : idx + 'index' : getattr(model, "__idx__", 0) } + return available_resources + def summary_mine(self, bundle, request): app_label = self.topic.app_label() self.method_check(request, allowed=['get']) diff --git a/app/detective/utils.py b/app/detective/utils.py index 371bf08a..0c9c394a 100644 --- a/app/detective/utils.py +++ b/app/detective/utils.py @@ -215,10 +215,10 @@ def iterate_model_fields(model, order_by='name'): models_rules = register.topics_rules().model(model) if hasattr(model, '__fields_order__'): _len = len(model._meta.fields) - model_fileds = sorted(model._meta.fields, key=lambda x: model.__fields_order__.index(x.name) if x.name in model.__fields_order__ else _len) + model_fields = sorted(model._meta.fields, key=lambda x: model.__fields_order__.index(x.name) if x.name in model.__fields_order__ else _len) else: - model_fileds = sorted(model._meta.fields, key=lambda el: getattr(el, order_by)) - for f in model_fileds: + model_fields = sorted(model._meta.fields, key=lambda el: getattr(el, order_by)) + for f in model_fields: # Ignores field terminating by + or begining by _ if not f.name.endswith("+") and not f.name.endswith("_set") and not f.name.startswith("_"): try: @@ -256,7 +256,7 @@ def iterate_model_fields(model, order_by='name'): 'verbose_name' : verbose_name, 'related_model': related_model, 'model' : model.__name__, - 'rules' : field_rules + 'rules' : field_rules.copy() } def get_model_nodes(): diff --git a/app/settings/common.py b/app/settings/common.py index cffeaf95..985db77e 100644 --- a/app/settings/common.py +++ b/app/settings/common.py @@ -65,6 +65,22 @@ # If you set this to False, Django will not use timezone-aware datetimes. USE_TZ = False +DATETIME_FORMATS = [ +'%Y-%m-%dT%H:%M:%S.%fZ', # '2006-10-25T14:30:59.000Z' +'%Y-%m-%d %H:%M:%S.%fZ', # '2006-10-25T14:30:59.000Z' +'%Y-%m-%dT%H:%M:%S.%f', # '2006-10-25T14:30:59.000' +'%Y-%m-%d %H:%M:%S.%f', # '2006-10-25 14:30:59.000' +'%Y-%m-%d %H:%M:%S', # '2006-10-25 14:30:59' +'%Y-%m-%d %H:%M', # '2006-10-25 14:30' +'%Y-%m-%d', # '2006-10-25' +'%m/%d/%Y %H:%M:%S', # '10/25/2006 14:30:59' +'%m/%d/%Y %H:%M', # '10/25/2006 14:30' +'%m/%d/%Y', # '10/25/2006' +'%m/%d/%y %H:%M:%S', # '10/25/06 14:30:59' +'%m/%d/%y %H:%M', # '10/25/06 14:30' +'%m/%d/%y' +] # '10/25/06' + # Absolute filesystem path to the directory that will hold user-uploaded files. # Example: "/home/media/media.lawrence.com/media/" MEDIA_ROOT = root('media') @@ -124,7 +140,8 @@ CACHE_MIDDLEWARE_ANONYMOUS_ONLY = True CACHE_BYPASS_URLS = ( r"/api/(?P[\w\-\.]+)/(?P[\w\-]+)/v1/summary/graph/", - r"/api/(?P[\w\-\.]+)/(?P[\w\-]+)/v1/summary/export/" + r"/api/(?P[\w\-\.]+)/(?P[\w\-]+)/v1/summary/export/", + r"/api/(?P[\w\-\.]+)/(?P[\w\-]+)/v1/summary/forms/" ) MIDDLEWARE_CLASSES = [ @@ -244,8 +261,8 @@ def get_cache(): # Use django local development cache (for local development). return { 'default': { - 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', - #'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', + #'BACKEND': 'django.core.cache.backends.dummy.DummyCache', + 'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', 'LOCATION': '/tmp/django_cache', } } diff --git a/app/urls.py b/app/urls.py index 8bc9f85b..a3e440b8 100644 --- a/app/urls.py +++ b/app/urls.py @@ -42,6 +42,8 @@ url(r'^(?P[\w\-\.]+)/$', 'app.detective.views.profile', name='user'), url(r'^(?P[\w\-\.]+)/(?P[\w\-]+)/$', 'app.detective.views.topic', name='explore'), url(r'^(?P[\w\-\.]+)/(?P[\w\-]+)/graph/$', 'app.detective.views.topic', name='explore'), + url(r'^(?P[\w\-\.]+)/(?P[\w\-]+)/settings/$', 'app.detective.views.topic', name='topic-settings'), + url(r'^(?P[\w\-\.]+)/(?P[\w\-]+)/settings/structure/$', 'app.detective.views.topic', name='topic-settings'), url(r'^(?P[\w\-\.]+)/(?P[\w\-]+)/(?P[\w-]+)/$', 'app.detective.views.entity_list', name='list'), url(r'^(?P[\w\-\.]+)/(?P[\w\-]+)/(?P\w+)/(?P\d+)/$', 'app.detective.views.entity_details', name='single'), url(r'^(?P[\w\-\.]+)/(?P[\w\-]+)/(?P\w+)/(?P\d+)/network/$', 'app.detective.views.entity_details', name='single_network'), diff --git a/docs/conf.py b/docs/conf.py index fac2c639..95c9ff51 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,7 +58,7 @@ # The short X.Y version. version = '1.12' # The full version, including alpha/beta/rc tags. -release = '1.12.3 Frog' +release = '1.12.4 Gorilla' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages.